diff --git "a/checkpoint-40000/trainer_state.json" "b/checkpoint-40000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-40000/trainer_state.json"
@@ -0,0 +1,288872 @@
+{
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.7943758192000635,
+ "global_step": 40000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.5873015873015873e-06,
+ "loss": 10.5641,
+ "theoretical_loss": 20.84128112621979,
+ "tokens_seen": 65536
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.1746031746031746e-06,
+ "loss": 10.5513,
+ "theoretical_loss": 17.594664429001284,
+ "tokens_seen": 131072
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.7619047619047615e-06,
+ "loss": 10.4631,
+ "theoretical_loss": 15.967940417509208,
+ "tokens_seen": 196608
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.349206349206349e-06,
+ "loss": 10.2495,
+ "theoretical_loss": 14.92078008066121,
+ "tokens_seen": 262144
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.936507936507936e-06,
+ "loss": 10.0077,
+ "theoretical_loss": 14.164680262435617,
+ "tokens_seen": 327680
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.523809523809523e-06,
+ "loss": 9.7256,
+ "theoretical_loss": 13.581024797222863,
+ "tokens_seen": 393216
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.1111111111111112e-05,
+ "loss": 9.5124,
+ "theoretical_loss": 13.11027232607383,
+ "tokens_seen": 458752
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.2698412698412699e-05,
+ "loss": 9.3381,
+ "theoretical_loss": 12.718592950155966,
+ "tokens_seen": 524288
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.4285714285714285e-05,
+ "loss": 9.1669,
+ "theoretical_loss": 12.385055788546264,
+ "tokens_seen": 589824
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.5873015873015872e-05,
+ "loss": 8.9763,
+ "theoretical_loss": 12.09587593170772,
+ "tokens_seen": 655360
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.746031746031746e-05,
+ "loss": 8.8295,
+ "theoretical_loss": 11.841529555774319,
+ "tokens_seen": 720896
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 1.9047619047619046e-05,
+ "loss": 8.8699,
+ "theoretical_loss": 11.615182533379372,
+ "tokens_seen": 786432
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.0634920634920636e-05,
+ "loss": 8.7295,
+ "theoretical_loss": 11.41177746485943,
+ "tokens_seen": 851968
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.2222222222222223e-05,
+ "loss": 8.6618,
+ "theoretical_loss": 11.227475026784514,
+ "tokens_seen": 917504
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.380952380952381e-05,
+ "loss": 8.6244,
+ "theoretical_loss": 11.059297242650555,
+ "tokens_seen": 983040
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.5396825396825397e-05,
+ "loss": 8.5977,
+ "theoretical_loss": 10.90489141112959,
+ "tokens_seen": 1048576
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.6984126984126984e-05,
+ "loss": 8.4756,
+ "theoretical_loss": 10.762369064735367,
+ "tokens_seen": 1114112
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 2.857142857142857e-05,
+ "loss": 8.4981,
+ "theoretical_loss": 10.63019320090292,
+ "tokens_seen": 1179648
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.0158730158730158e-05,
+ "loss": 8.3172,
+ "theoretical_loss": 10.507097497029282,
+ "tokens_seen": 1245184
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.1746031746031745e-05,
+ "loss": 8.176,
+ "theoretical_loss": 10.392027268435971,
+ "tokens_seen": 1310720
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.3333333333333335e-05,
+ "loss": 8.1964,
+ "theoretical_loss": 10.284095545569386,
+ "tokens_seen": 1376256
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.492063492063492e-05,
+ "loss": 8.1379,
+ "theoretical_loss": 10.182549877942659,
+ "tokens_seen": 1441792
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.650793650793651e-05,
+ "loss": 7.9969,
+ "theoretical_loss": 10.08674688640524,
+ "tokens_seen": 1507328
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.809523809523809e-05,
+ "loss": 8.1059,
+ "theoretical_loss": 9.99613250351292,
+ "tokens_seen": 1572864
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 3.968253968253968e-05,
+ "loss": 7.9094,
+ "theoretical_loss": 9.91022645106575,
+ "tokens_seen": 1638400
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.126984126984127e-05,
+ "loss": 7.7211,
+ "theoretical_loss": 9.828609916213201,
+ "tokens_seen": 1703936
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.2857142857142856e-05,
+ "loss": 7.7885,
+ "theoretical_loss": 9.750915671580087,
+ "tokens_seen": 1769472
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.4444444444444447e-05,
+ "loss": 7.7711,
+ "theoretical_loss": 9.676820083754187,
+ "tokens_seen": 1835008
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.603174603174603e-05,
+ "loss": 7.6633,
+ "theoretical_loss": 9.606036595818281,
+ "tokens_seen": 1900544
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.761904761904762e-05,
+ "loss": 7.5596,
+ "theoretical_loss": 9.538310371437493,
+ "tokens_seen": 1966080
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 4.9206349206349204e-05,
+ "loss": 7.5956,
+ "theoretical_loss": 9.473413862304575,
+ "tokens_seen": 2031616
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.0793650793650794e-05,
+ "loss": 7.4655,
+ "theoretical_loss": 9.411143115583098,
+ "tokens_seen": 2097152
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.2380952380952384e-05,
+ "loss": 7.5066,
+ "theoretical_loss": 9.351314678906622,
+ "tokens_seen": 2162688
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.396825396825397e-05,
+ "loss": 7.3266,
+ "theoretical_loss": 9.293762991332915,
+ "tokens_seen": 2228224
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.555555555555556e-05,
+ "loss": 7.2918,
+ "theoretical_loss": 9.238338172117398,
+ "tokens_seen": 2293760
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.714285714285714e-05,
+ "loss": 7.3421,
+ "theoretical_loss": 9.184904137180933,
+ "tokens_seen": 2359296
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 5.873015873015873e-05,
+ "loss": 7.2943,
+ "theoretical_loss": 9.133336987085992,
+ "tokens_seen": 2424832
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.0317460317460316e-05,
+ "loss": 7.277,
+ "theoretical_loss": 9.083523621206375,
+ "tokens_seen": 2490368
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.19047619047619e-05,
+ "loss": 7.1754,
+ "theoretical_loss": 9.03536054131647,
+ "tokens_seen": 2555904
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.349206349206349e-05,
+ "loss": 7.1903,
+ "theoretical_loss": 8.988752814581996,
+ "tokens_seen": 2621440
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.507936507936509e-05,
+ "loss": 7.1207,
+ "theoretical_loss": 8.943613171313253,
+ "tokens_seen": 2686976
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.666666666666667e-05,
+ "loss": 7.0332,
+ "theoretical_loss": 8.899861217150864,
+ "tokens_seen": 2752512
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.825396825396825e-05,
+ "loss": 6.8774,
+ "theoretical_loss": 8.85742274282614,
+ "tokens_seen": 2818048
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 6.984126984126984e-05,
+ "loss": 6.9871,
+ "theoretical_loss": 8.816229117451055,
+ "tokens_seen": 2883584
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.142857142857143e-05,
+ "loss": 6.7777,
+ "theoretical_loss": 8.776216753584098,
+ "tokens_seen": 2949120
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.301587301587302e-05,
+ "loss": 6.8747,
+ "theoretical_loss": 8.737326634193474,
+ "tokens_seen": 3014656
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.460317460317461e-05,
+ "loss": 6.6371,
+ "theoretical_loss": 8.699503893181488,
+ "tokens_seen": 3080192
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.619047619047618e-05,
+ "loss": 6.5967,
+ "theoretical_loss": 8.662697442408115,
+ "tokens_seen": 3145728
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.777777777777778e-05,
+ "loss": 6.7174,
+ "theoretical_loss": 8.626859639208966,
+ "tokens_seen": 3211264
+ },
+ {
+ "epoch": 0.0,
+ "objective/train/docs_used": 7703,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 6.697691917419434,
+ "objective/train/theoretical_loss": 8.591945989283708,
+ "objective/train/tokens_used": 23736800,
+ "theoretical_loss": 8.591945989283708,
+ "tokens_seen": 3276800
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 7.936507936507937e-05,
+ "loss": 6.7105,
+ "theoretical_loss": 8.591945989283708,
+ "tokens_seen": 3276800
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.095238095238096e-05,
+ "loss": 6.599,
+ "theoretical_loss": 8.557914880567758,
+ "tokens_seen": 3342336
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.253968253968255e-05,
+ "loss": 6.4888,
+ "theoretical_loss": 8.524727344318643,
+ "tokens_seen": 3407872
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.412698412698413e-05,
+ "loss": 6.4092,
+ "theoretical_loss": 8.492346840169828,
+ "tokens_seen": 3473408
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.571428571428571e-05,
+ "loss": 6.5305,
+ "theoretical_loss": 8.460739062345422,
+ "tokens_seen": 3538944
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.730158730158731e-05,
+ "loss": 6.3927,
+ "theoretical_loss": 8.429871764603359,
+ "tokens_seen": 3604480
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 8.888888888888889e-05,
+ "loss": 6.2062,
+ "theoretical_loss": 8.399714601792851,
+ "tokens_seen": 3670016
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.047619047619048e-05,
+ "loss": 6.2861,
+ "theoretical_loss": 8.370238986183718,
+ "tokens_seen": 3735552
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.206349206349206e-05,
+ "loss": 6.2594,
+ "theoretical_loss": 8.34141795695797,
+ "tokens_seen": 3801088
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.365079365079366e-05,
+ "loss": 6.0057,
+ "theoretical_loss": 8.313226061453822,
+ "tokens_seen": 3866624
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.523809523809524e-05,
+ "loss": 6.1933,
+ "theoretical_loss": 8.285639246924504,
+ "tokens_seen": 3932160
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.682539682539682e-05,
+ "loss": 6.1101,
+ "theoretical_loss": 8.258634761722755,
+ "tokens_seen": 3997696
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 9.841269841269841e-05,
+ "loss": 6.0988,
+ "theoretical_loss": 8.232191064950612,
+ "tokens_seen": 4063232
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001,
+ "loss": 6.0056,
+ "theoretical_loss": 8.206287743725687,
+ "tokens_seen": 4128768
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010158730158730159,
+ "loss": 6.0406,
+ "theoretical_loss": 8.180905437312258,
+ "tokens_seen": 4194304
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010317460317460319,
+ "loss": 5.9078,
+ "theoretical_loss": 8.15602576745014,
+ "tokens_seen": 4259840
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010476190476190477,
+ "loss": 5.7249,
+ "theoretical_loss": 8.13163127428835,
+ "tokens_seen": 4325376
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010634920634920635,
+ "loss": 5.847,
+ "theoretical_loss": 8.107705357395282,
+ "tokens_seen": 4390912
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010793650793650794,
+ "loss": 5.7916,
+ "theoretical_loss": 8.084232221374057,
+ "tokens_seen": 4456448
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00010952380952380953,
+ "loss": 5.7336,
+ "theoretical_loss": 8.06119682566165,
+ "tokens_seen": 4521984
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00011111111111111112,
+ "loss": 5.7012,
+ "theoretical_loss": 8.038584838134476,
+ "tokens_seen": 4587520
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001126984126984127,
+ "loss": 5.6223,
+ "theoretical_loss": 8.016382592182019,
+ "tokens_seen": 4653056
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00011428571428571428,
+ "loss": 5.6187,
+ "theoretical_loss": 7.994577046944442,
+ "tokens_seen": 4718592
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001158730158730159,
+ "loss": 5.5594,
+ "theoretical_loss": 7.973155750440629,
+ "tokens_seen": 4784128
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00011746031746031746,
+ "loss": 5.5114,
+ "theoretical_loss": 7.95210680534016,
+ "tokens_seen": 4849664
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00011904761904761905,
+ "loss": 5.7208,
+ "theoretical_loss": 7.931418837156708,
+ "tokens_seen": 4915200
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00012063492063492063,
+ "loss": 5.6176,
+ "theoretical_loss": 7.911080964661844,
+ "tokens_seen": 4980736
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00012222222222222224,
+ "loss": 5.4612,
+ "theoretical_loss": 7.891082772337269,
+ "tokens_seen": 5046272
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001238095238095238,
+ "loss": 5.4758,
+ "theoretical_loss": 7.871414284700578,
+ "tokens_seen": 5111808
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001253968253968254,
+ "loss": 5.4206,
+ "theoretical_loss": 7.852065942354954,
+ "tokens_seen": 5177344
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00012698412698412698,
+ "loss": 5.4236,
+ "theoretical_loss": 7.833028579626806,
+ "tokens_seen": 5242880
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00012857142857142858,
+ "loss": 5.526,
+ "theoretical_loss": 7.814293403667714,
+ "tokens_seen": 5308416
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00013015873015873017,
+ "loss": 5.4883,
+ "theoretical_loss": 7.795851974908005,
+ "tokens_seen": 5373952
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00013174603174603174,
+ "loss": 5.3321,
+ "theoretical_loss": 7.77769618875926,
+ "tokens_seen": 5439488
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00013333333333333334,
+ "loss": 5.3773,
+ "theoretical_loss": 7.7598182584719595,
+ "tokens_seen": 5505024
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00013492063492063494,
+ "loss": 5.3975,
+ "theoretical_loss": 7.742210699062561,
+ "tokens_seen": 5570560
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001365079365079365,
+ "loss": 5.301,
+ "theoretical_loss": 7.724866312231575,
+ "tokens_seen": 5636096
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001380952380952381,
+ "loss": 5.3225,
+ "theoretical_loss": 7.707778172200822,
+ "tokens_seen": 5701632
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00013968253968253967,
+ "loss": 5.4532,
+ "theoretical_loss": 7.6909396124039695,
+ "tokens_seen": 5767168
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001412698412698413,
+ "loss": 5.3501,
+ "theoretical_loss": 7.67434421296992,
+ "tokens_seen": 5832704
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00014285714285714287,
+ "loss": 5.3752,
+ "theoretical_loss": 7.657985788943502,
+ "tokens_seen": 5898240
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00014444444444444444,
+ "loss": 5.1601,
+ "theoretical_loss": 7.641858379192337,
+ "tokens_seen": 5963776
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00014603174603174603,
+ "loss": 5.1855,
+ "theoretical_loss": 7.625956235952921,
+ "tokens_seen": 6029312
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00014761904761904763,
+ "loss": 5.2364,
+ "theoretical_loss": 7.610273814972529,
+ "tokens_seen": 6094848
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00014920634920634923,
+ "loss": 5.1903,
+ "theoretical_loss": 7.594805766207012,
+ "tokens_seen": 6160384
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001507936507936508,
+ "loss": 5.2898,
+ "theoretical_loss": 7.57954692503759,
+ "tokens_seen": 6225920
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00015238095238095237,
+ "loss": 5.1998,
+ "theoretical_loss": 7.5644923039725604,
+ "tokens_seen": 6291456
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.000153968253968254,
+ "loss": 4.9998,
+ "theoretical_loss": 7.549637084802403,
+ "tokens_seen": 6356992
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00015555555555555556,
+ "loss": 5.2391,
+ "theoretical_loss": 7.534976611179138,
+ "tokens_seen": 6422528
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00015714285714285716,
+ "loss": 5.1035,
+ "theoretical_loss": 7.52050638159289,
+ "tokens_seen": 6488064
+ },
+ {
+ "epoch": 0.0,
+ "objective/train/docs_used": 8901,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 4.883451461791992,
+ "objective/train/theoretical_loss": 7.506222042720672,
+ "objective/train/tokens_used": 27013600,
+ "theoretical_loss": 7.506222042720672,
+ "tokens_seen": 6553600
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00015873015873015873,
+ "loss": 5.0335,
+ "theoretical_loss": 7.506222042720672,
+ "tokens_seen": 6553600
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00016031746031746033,
+ "loss": 5.1059,
+ "theoretical_loss": 7.492119383124103,
+ "tokens_seen": 6619136
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00016190476190476192,
+ "loss": 5.2313,
+ "theoretical_loss": 7.478194327274557,
+ "tokens_seen": 6684672
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001634920634920635,
+ "loss": 5.0888,
+ "theoretical_loss": 7.46444292988566,
+ "tokens_seen": 6750208
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001650793650793651,
+ "loss": 4.9131,
+ "theoretical_loss": 7.450861370534546,
+ "tokens_seen": 6815744
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001666666666666667,
+ "loss": 5.1414,
+ "theoretical_loss": 7.437445948554516,
+ "tokens_seen": 6881280
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00016825396825396826,
+ "loss": 5.0188,
+ "theoretical_loss": 7.424193078182986,
+ "tokens_seen": 6946816
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00016984126984126986,
+ "loss": 5.0327,
+ "theoretical_loss": 7.411099283949674,
+ "tokens_seen": 7012352
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00017142857142857143,
+ "loss": 5.1783,
+ "theoretical_loss": 7.3981611962910145,
+ "tokens_seen": 7077888
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00017301587301587302,
+ "loss": 5.1053,
+ "theoretical_loss": 7.38537554737773,
+ "tokens_seen": 7143424
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00017460317460317462,
+ "loss": 4.9697,
+ "theoretical_loss": 7.372739167143329,
+ "tokens_seen": 7208960
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0001761904761904762,
+ "loss": 5.009,
+ "theoretical_loss": 7.36024897950216,
+ "tokens_seen": 7274496
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00017777777777777779,
+ "loss": 5.0185,
+ "theoretical_loss": 7.347901998746318,
+ "tokens_seen": 7340032
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00017936507936507938,
+ "loss": 5.0807,
+ "theoretical_loss": 7.335695326111465,
+ "tokens_seen": 7405568
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018095238095238095,
+ "loss": 4.9311,
+ "theoretical_loss": 7.323626146502205,
+ "tokens_seen": 7471104
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018253968253968255,
+ "loss": 5.051,
+ "theoretical_loss": 7.311691725368293,
+ "tokens_seen": 7536640
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018412698412698412,
+ "loss": 5.0423,
+ "theoretical_loss": 7.299889405723473,
+ "tokens_seen": 7602176
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018571428571428572,
+ "loss": 4.9575,
+ "theoretical_loss": 7.288216605299264,
+ "tokens_seen": 7667712
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018730158730158731,
+ "loss": 4.9026,
+ "theoretical_loss": 7.276670813826511,
+ "tokens_seen": 7733248
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00018888888888888888,
+ "loss": 4.9363,
+ "theoretical_loss": 7.265249590437903,
+ "tokens_seen": 7798784
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019047619047619048,
+ "loss": 4.9141,
+ "theoretical_loss": 7.253950561185138,
+ "tokens_seen": 7864320
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019206349206349208,
+ "loss": 4.8576,
+ "theoretical_loss": 7.242771416664761,
+ "tokens_seen": 7929856
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019365079365079365,
+ "loss": 4.8336,
+ "theoretical_loss": 7.231709909747042,
+ "tokens_seen": 7995392
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019523809523809525,
+ "loss": 4.9158,
+ "theoretical_loss": 7.220763853402656,
+ "tokens_seen": 8060928
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019682539682539682,
+ "loss": 4.8969,
+ "theoretical_loss": 7.209931118622157,
+ "tokens_seen": 8126464
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00019841269841269844,
+ "loss": 5.0328,
+ "theoretical_loss": 7.199209632423606,
+ "tokens_seen": 8192000
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002,
+ "loss": 4.7254,
+ "theoretical_loss": 7.188597375943913,
+ "tokens_seen": 8257536
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020158730158730158,
+ "loss": 4.8566,
+ "theoretical_loss": 7.178092382609779,
+ "tokens_seen": 8323072
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020317460317460318,
+ "loss": 4.9601,
+ "theoretical_loss": 7.1676927363842795,
+ "tokens_seen": 8388608
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020476190476190475,
+ "loss": 4.8234,
+ "theoretical_loss": 7.157396570085435,
+ "tokens_seen": 8454144
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020634920634920637,
+ "loss": 4.9181,
+ "theoretical_loss": 7.147202063773257,
+ "tokens_seen": 8519680
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020793650793650797,
+ "loss": 4.8418,
+ "theoretical_loss": 7.137107443201986,
+ "tokens_seen": 8585216
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00020952380952380954,
+ "loss": 4.819,
+ "theoretical_loss": 7.127110978334411,
+ "tokens_seen": 8650752
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00021111111111111113,
+ "loss": 4.9044,
+ "theoretical_loss": 7.117210981915334,
+ "tokens_seen": 8716288
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002126984126984127,
+ "loss": 4.8086,
+ "theoretical_loss": 7.10740580810139,
+ "tokens_seen": 8781824
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002142857142857143,
+ "loss": 4.7956,
+ "theoretical_loss": 7.097693851144607,
+ "tokens_seen": 8847360
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00021587301587301587,
+ "loss": 4.8153,
+ "theoretical_loss": 7.088073544127209,
+ "tokens_seen": 8912896
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00021746031746031744,
+ "loss": 4.8548,
+ "theoretical_loss": 7.07854335774531,
+ "tokens_seen": 8978432
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00021904761904761907,
+ "loss": 4.6907,
+ "theoretical_loss": 7.069101799139266,
+ "tokens_seen": 9043968
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00022063492063492066,
+ "loss": 4.7805,
+ "theoretical_loss": 7.0597474107685745,
+ "tokens_seen": 9109504
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00022222222222222223,
+ "loss": 4.7233,
+ "theoretical_loss": 7.050478769329319,
+ "tokens_seen": 9175040
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00022380952380952383,
+ "loss": 4.8346,
+ "theoretical_loss": 7.0412944847122505,
+ "tokens_seen": 9240576
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002253968253968254,
+ "loss": 4.7503,
+ "theoretical_loss": 7.032193198999712,
+ "tokens_seen": 9306112
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.000226984126984127,
+ "loss": 4.6828,
+ "theoretical_loss": 7.023173585499698,
+ "tokens_seen": 9371648
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00022857142857142857,
+ "loss": 4.6974,
+ "theoretical_loss": 7.014234347815421,
+ "tokens_seen": 9437184
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00023015873015873016,
+ "loss": 4.8072,
+ "theoretical_loss": 7.00537421894883,
+ "tokens_seen": 9502720
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002317460317460318,
+ "loss": 4.692,
+ "theoretical_loss": 6.996591960436656,
+ "tokens_seen": 9568256
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00023333333333333336,
+ "loss": 4.7647,
+ "theoretical_loss": 6.9878863615175355,
+ "tokens_seen": 9633792
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00023492063492063493,
+ "loss": 4.7072,
+ "theoretical_loss": 6.97925623832895,
+ "tokens_seen": 9699328
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00023650793650793653,
+ "loss": 4.8046,
+ "theoretical_loss": 6.97070043313267,
+ "tokens_seen": 9764864
+ },
+ {
+ "epoch": 0.0,
+ "objective/train/docs_used": 9988,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 4.5934672355651855,
+ "objective/train/theoretical_loss": 6.962217813567541,
+ "objective/train/tokens_used": 30290400,
+ "theoretical_loss": 6.962217813567541,
+ "tokens_seen": 9830400
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002380952380952381,
+ "loss": 4.7083,
+ "theoretical_loss": 6.962217813567541,
+ "tokens_seen": 9830400
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002396825396825397,
+ "loss": 4.6527,
+ "theoretical_loss": 6.953807271928458,
+ "tokens_seen": 9895936
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00024126984126984126,
+ "loss": 4.4851,
+ "theoretical_loss": 6.945467724470439,
+ "tokens_seen": 9961472
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00024285714285714286,
+ "loss": 4.4885,
+ "theoretical_loss": 6.937198110736788,
+ "tokens_seen": 10027008
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002444444444444445,
+ "loss": 4.5112,
+ "theoretical_loss": 6.928997392910329,
+ "tokens_seen": 10092544
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00024603174603174605,
+ "loss": 4.7377,
+ "theoretical_loss": 6.920864555186801,
+ "tokens_seen": 10158080
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002476190476190476,
+ "loss": 4.6674,
+ "theoretical_loss": 6.912798603169512,
+ "tokens_seen": 10223616
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00024920634920634925,
+ "loss": 4.5936,
+ "theoretical_loss": 6.904798563284391,
+ "tokens_seen": 10289152
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002507936507936508,
+ "loss": 4.8009,
+ "theoretical_loss": 6.896863482214638,
+ "tokens_seen": 10354688
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002523809523809524,
+ "loss": 4.7057,
+ "theoretical_loss": 6.888992426354192,
+ "tokens_seen": 10420224
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00025396825396825396,
+ "loss": 4.7217,
+ "theoretical_loss": 6.881184481279271,
+ "tokens_seen": 10485760
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00025555555555555553,
+ "loss": 4.5575,
+ "theoretical_loss": 6.873438751237283,
+ "tokens_seen": 10551296
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00025714285714285715,
+ "loss": 4.6313,
+ "theoretical_loss": 6.865754358652436,
+ "tokens_seen": 10616832
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002587301587301588,
+ "loss": 4.5677,
+ "theoretical_loss": 6.85813044364739,
+ "tokens_seen": 10682368
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00026031746031746035,
+ "loss": 4.5783,
+ "theoretical_loss": 6.85056616358036,
+ "tokens_seen": 10747904
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002619047619047619,
+ "loss": 4.7549,
+ "theoretical_loss": 6.843060692597036,
+ "tokens_seen": 10813440
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002634920634920635,
+ "loss": 4.5659,
+ "theoretical_loss": 6.835613221196822,
+ "tokens_seen": 10878976
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002650793650793651,
+ "loss": 4.6755,
+ "theoretical_loss": 6.8282229558127865,
+ "tokens_seen": 10944512
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002666666666666667,
+ "loss": 4.5563,
+ "theoretical_loss": 6.820889118404866,
+ "tokens_seen": 11010048
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00026825396825396825,
+ "loss": 4.5433,
+ "theoretical_loss": 6.813610946065789,
+ "tokens_seen": 11075584
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002698412698412699,
+ "loss": 4.5981,
+ "theoretical_loss": 6.806387690639279,
+ "tokens_seen": 11141120
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00027142857142857144,
+ "loss": 4.5952,
+ "theoretical_loss": 6.799218618350064,
+ "tokens_seen": 11206656
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.000273015873015873,
+ "loss": 4.6503,
+ "theoretical_loss": 6.792103009445266,
+ "tokens_seen": 11272192
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00027460317460317464,
+ "loss": 4.4218,
+ "theoretical_loss": 6.7850401578467645,
+ "tokens_seen": 11337728
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002761904761904762,
+ "loss": 4.5615,
+ "theoretical_loss": 6.778029370814127,
+ "tokens_seen": 11403264
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002777777777777778,
+ "loss": 4.535,
+ "theoretical_loss": 6.771069968617732,
+ "tokens_seen": 11468800
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00027936507936507935,
+ "loss": 4.6074,
+ "theoretical_loss": 6.764161284221716,
+ "tokens_seen": 11534336
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00028095238095238097,
+ "loss": 4.7694,
+ "theoretical_loss": 6.757302662976402,
+ "tokens_seen": 11599872
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002825396825396826,
+ "loss": 4.5586,
+ "theoretical_loss": 6.750493462319867,
+ "tokens_seen": 11665408
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00028412698412698417,
+ "loss": 4.4784,
+ "theoretical_loss": 6.74373305148833,
+ "tokens_seen": 11730944
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00028571428571428574,
+ "loss": 4.4651,
+ "theoretical_loss": 6.737020811235054,
+ "tokens_seen": 11796480
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002873015873015873,
+ "loss": 4.3919,
+ "theoretical_loss": 6.730356133557465,
+ "tokens_seen": 11862016
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002888888888888889,
+ "loss": 4.5578,
+ "theoretical_loss": 6.7237384214322,
+ "tokens_seen": 11927552
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0002904761904761905,
+ "loss": 4.4157,
+ "theoretical_loss": 6.717167088557821,
+ "tokens_seen": 11993088
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00029206349206349207,
+ "loss": 4.534,
+ "theoretical_loss": 6.710641559104915,
+ "tokens_seen": 12058624
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00029365079365079364,
+ "loss": 4.5115,
+ "theoretical_loss": 6.704161267473356,
+ "tokens_seen": 12124160
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00029523809523809526,
+ "loss": 4.5263,
+ "theoretical_loss": 6.697725658056456,
+ "tokens_seen": 12189696
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00029682539682539683,
+ "loss": 4.623,
+ "theoretical_loss": 6.691334185011799,
+ "tokens_seen": 12255232
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00029841269841269846,
+ "loss": 4.4268,
+ "theoretical_loss": 6.68498631203852,
+ "tokens_seen": 12320768
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00030000000000000003,
+ "loss": 4.4161,
+ "theoretical_loss": 6.678681512160812,
+ "tokens_seen": 12386304
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003015873015873016,
+ "loss": 4.3853,
+ "theoretical_loss": 6.672419267517472,
+ "tokens_seen": 12451840
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00030317460317460317,
+ "loss": 4.4433,
+ "theoretical_loss": 6.666199069157267,
+ "tokens_seen": 12517376
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00030476190476190474,
+ "loss": 4.5607,
+ "theoretical_loss": 6.660020416839945,
+ "tokens_seen": 12582912
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00030634920634920636,
+ "loss": 4.4346,
+ "theoretical_loss": 6.65388281884269,
+ "tokens_seen": 12648448
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.000307936507936508,
+ "loss": 4.4253,
+ "theoretical_loss": 6.647785791771864,
+ "tokens_seen": 12713984
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00030952380952380956,
+ "loss": 4.423,
+ "theoretical_loss": 6.641728860379852,
+ "tokens_seen": 12779520
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003111111111111111,
+ "loss": 4.5214,
+ "theoretical_loss": 6.635711557386854,
+ "tokens_seen": 12845056
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003126984126984127,
+ "loss": 4.5018,
+ "theoretical_loss": 6.629733423307451,
+ "tokens_seen": 12910592
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003142857142857143,
+ "loss": 4.5074,
+ "theoretical_loss": 6.6237940062818215,
+ "tokens_seen": 12976128
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003158730158730159,
+ "loss": 4.5088,
+ "theoretical_loss": 6.617892861911426,
+ "tokens_seen": 13041664
+ },
+ {
+ "epoch": 0.0,
+ "objective/train/docs_used": 11319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 4.34834098815918,
+ "objective/train/theoretical_loss": 6.612029553099055,
+ "objective/train/tokens_used": 33567200,
+ "theoretical_loss": 6.612029553099055,
+ "tokens_seen": 13107200
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00031746031746031746,
+ "loss": 4.4598,
+ "theoretical_loss": 6.612029553099055,
+ "tokens_seen": 13107200
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003190476190476191,
+ "loss": 4.5264,
+ "theoretical_loss": 6.606203649893072,
+ "tokens_seen": 13172736
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00032063492063492065,
+ "loss": 4.4953,
+ "theoretical_loss": 6.600414729335739,
+ "tokens_seen": 13238272
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003222222222222222,
+ "loss": 4.5654,
+ "theoretical_loss": 6.5946623753155045,
+ "tokens_seen": 13303808
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00032380952380952385,
+ "loss": 4.4299,
+ "theoretical_loss": 6.588946178423104,
+ "tokens_seen": 13369344
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003253968253968254,
+ "loss": 4.5707,
+ "theoretical_loss": 6.583265735811386,
+ "tokens_seen": 13434880
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.000326984126984127,
+ "loss": 4.4346,
+ "theoretical_loss": 6.577620651058732,
+ "tokens_seen": 13500416
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00032857142857142856,
+ "loss": 4.4908,
+ "theoretical_loss": 6.572010534035971,
+ "tokens_seen": 13565952
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003301587301587302,
+ "loss": 4.5556,
+ "theoretical_loss": 6.566435000776667,
+ "tokens_seen": 13631488
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003317460317460318,
+ "loss": 4.5845,
+ "theoretical_loss": 6.5608936733507095,
+ "tokens_seen": 13697024
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003333333333333334,
+ "loss": 4.5322,
+ "theoretical_loss": 6.555386179741061,
+ "tokens_seen": 13762560
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00033492063492063495,
+ "loss": 4.4791,
+ "theoretical_loss": 6.549912153723619,
+ "tokens_seen": 13828096
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003365079365079365,
+ "loss": 4.4018,
+ "theoretical_loss": 6.544471234750057,
+ "tokens_seen": 13893632
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003380952380952381,
+ "loss": 4.451,
+ "theoretical_loss": 6.539063067833591,
+ "tokens_seen": 13959168
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003396825396825397,
+ "loss": 4.2116,
+ "theoretical_loss": 6.533687303437551,
+ "tokens_seen": 14024704
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003412698412698413,
+ "loss": 4.462,
+ "theoretical_loss": 6.528343597366709,
+ "tokens_seen": 14090240
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00034285714285714285,
+ "loss": 4.405,
+ "theoretical_loss": 6.523031610661259,
+ "tokens_seen": 14155776
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003444444444444445,
+ "loss": 4.5102,
+ "theoretical_loss": 6.517751009493386,
+ "tokens_seen": 14221312
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00034603174603174604,
+ "loss": 4.3705,
+ "theoretical_loss": 6.51250146506634,
+ "tokens_seen": 14286848
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00034761904761904767,
+ "loss": 4.4277,
+ "theoretical_loss": 6.507282653515952,
+ "tokens_seen": 14352384
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00034920634920634924,
+ "loss": 4.384,
+ "theoretical_loss": 6.5020942558145105,
+ "tokens_seen": 14417920
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003507936507936508,
+ "loss": 4.4511,
+ "theoretical_loss": 6.496935957676949,
+ "tokens_seen": 14483456
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003523809523809524,
+ "loss": 4.4827,
+ "theoretical_loss": 6.491807449469257,
+ "tokens_seen": 14548992
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00035396825396825395,
+ "loss": 4.4038,
+ "theoretical_loss": 6.486708426119071,
+ "tokens_seen": 14614528
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00035555555555555557,
+ "loss": 4.4651,
+ "theoretical_loss": 6.481638587028371,
+ "tokens_seen": 14680064
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003571428571428572,
+ "loss": 4.2654,
+ "theoretical_loss": 6.4765976359882345,
+ "tokens_seen": 14745600
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00035873015873015877,
+ "loss": 4.3412,
+ "theoretical_loss": 6.47158528109558,
+ "tokens_seen": 14811136
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00036031746031746034,
+ "loss": 4.4479,
+ "theoretical_loss": 6.466601234671855,
+ "tokens_seen": 14876672
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003619047619047619,
+ "loss": 4.2494,
+ "theoretical_loss": 6.461645213183605,
+ "tokens_seen": 14942208
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00036349206349206353,
+ "loss": 4.3764,
+ "theoretical_loss": 6.45671693716488,
+ "tokens_seen": 15007744
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003650793650793651,
+ "loss": 4.2567,
+ "theoretical_loss": 6.451816131141427,
+ "tokens_seen": 15073280
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00036666666666666667,
+ "loss": 4.3653,
+ "theoretical_loss": 6.4469425235566105,
+ "tokens_seen": 15138816
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00036825396825396824,
+ "loss": 4.462,
+ "theoretical_loss": 6.442095846699032,
+ "tokens_seen": 15204352
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00036984126984126986,
+ "loss": 4.4221,
+ "theoretical_loss": 6.437275836631781,
+ "tokens_seen": 15269888
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00037142857142857143,
+ "loss": 4.2723,
+ "theoretical_loss": 6.432482233123297,
+ "tokens_seen": 15335424
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00037301587301587306,
+ "loss": 4.2973,
+ "theoretical_loss": 6.427714779579773,
+ "tokens_seen": 15400960
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00037460317460317463,
+ "loss": 4.2562,
+ "theoretical_loss": 6.422973222979093,
+ "tokens_seen": 15466496
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003761904761904762,
+ "loss": 4.3652,
+ "theoretical_loss": 6.418257313806215,
+ "tokens_seen": 15532032
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00037777777777777777,
+ "loss": 4.3504,
+ "theoretical_loss": 6.413566805990028,
+ "tokens_seen": 15597568
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003793650793650794,
+ "loss": 4.2707,
+ "theoretical_loss": 6.408901456841564,
+ "tokens_seen": 15663104
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00038095238095238096,
+ "loss": 4.2359,
+ "theoretical_loss": 6.404261026993607,
+ "tokens_seen": 15728640
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003825396825396826,
+ "loss": 4.3195,
+ "theoretical_loss": 6.399645280341607,
+ "tokens_seen": 15794176
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00038412698412698416,
+ "loss": 4.3046,
+ "theoretical_loss": 6.395053983985889,
+ "tokens_seen": 15859712
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003857142857142857,
+ "loss": 4.1676,
+ "theoretical_loss": 6.390486908175121,
+ "tokens_seen": 15925248
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003873015873015873,
+ "loss": 4.3473,
+ "theoretical_loss": 6.385943826251012,
+ "tokens_seen": 15990784
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003888888888888889,
+ "loss": 4.3593,
+ "theoretical_loss": 6.381424514594194,
+ "tokens_seen": 16056320
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003904761904761905,
+ "loss": 4.2621,
+ "theoretical_loss": 6.376928752571271,
+ "tokens_seen": 16121856
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00039206349206349206,
+ "loss": 4.2769,
+ "theoretical_loss": 6.372456322483007,
+ "tokens_seen": 16187392
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00039365079365079363,
+ "loss": 4.3993,
+ "theoretical_loss": 6.368007009513611,
+ "tokens_seen": 16252928
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00039523809523809526,
+ "loss": 4.3462,
+ "theoretical_loss": 6.363580601681101,
+ "tokens_seen": 16318464
+ },
+ {
+ "epoch": 0.0,
+ "objective/train/docs_used": 12210,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 4.425621032714844,
+ "objective/train/theoretical_loss": 6.359176889788724,
+ "objective/train/tokens_used": 36844000,
+ "theoretical_loss": 6.359176889788724,
+ "tokens_seen": 16384000
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.0003968253968253969,
+ "loss": 4.3752,
+ "theoretical_loss": 6.359176889788724,
+ "tokens_seen": 16384000
+ },
+ {
+ "epoch": 0.0,
+ "learning_rate": 0.00039841269841269845,
+ "loss": 4.2801,
+ "theoretical_loss": 6.354795667377392,
+ "tokens_seen": 16449536
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004,
+ "loss": 4.282,
+ "theoretical_loss": 6.350436730679111,
+ "tokens_seen": 16515072
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004015873015873016,
+ "loss": 4.3637,
+ "theoretical_loss": 6.346099878571405,
+ "tokens_seen": 16580608
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00040317460317460316,
+ "loss": 4.2939,
+ "theoretical_loss": 6.341784912532662,
+ "tokens_seen": 16646144
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004047619047619048,
+ "loss": 4.3405,
+ "theoretical_loss": 6.337491636598426,
+ "tokens_seen": 16711680
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00040634920634920635,
+ "loss": 4.1746,
+ "theoretical_loss": 6.33321985731858,
+ "tokens_seen": 16777216
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004079365079365079,
+ "loss": 4.2457,
+ "theoretical_loss": 6.328969383715416,
+ "tokens_seen": 16842752
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004095238095238095,
+ "loss": 4.254,
+ "theoretical_loss": 6.324740027242559,
+ "tokens_seen": 16908288
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004111111111111111,
+ "loss": 4.1043,
+ "theoretical_loss": 6.32053160174474,
+ "tokens_seen": 16973824
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00041269841269841274,
+ "loss": 4.1842,
+ "theoretical_loss": 6.316343923418371,
+ "tokens_seen": 17039360
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00041428571428571437,
+ "loss": 4.0588,
+ "theoretical_loss": 6.312176810772938,
+ "tokens_seen": 17104896
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00041587301587301594,
+ "loss": 4.1163,
+ "theoretical_loss": 6.308030084593148,
+ "tokens_seen": 17170432
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004174603174603175,
+ "loss": 4.2286,
+ "theoretical_loss": 6.30390356790187,
+ "tokens_seen": 17235968
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004190476190476191,
+ "loss": 4.2368,
+ "theoretical_loss": 6.2997970859237835,
+ "tokens_seen": 17301504
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00042063492063492065,
+ "loss": 4.2891,
+ "theoretical_loss": 6.295710466049778,
+ "tokens_seen": 17367040
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00042222222222222227,
+ "loss": 4.3256,
+ "theoretical_loss": 6.29164353780205,
+ "tokens_seen": 17432576
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00042380952380952384,
+ "loss": 4.2051,
+ "theoretical_loss": 6.287596132799887,
+ "tokens_seen": 17498112
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004253968253968254,
+ "loss": 4.2521,
+ "theoretical_loss": 6.283568084726138,
+ "tokens_seen": 17563648
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000426984126984127,
+ "loss": 4.214,
+ "theoretical_loss": 6.27955922929433,
+ "tokens_seen": 17629184
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004285714285714286,
+ "loss": 4.1344,
+ "theoretical_loss": 6.2755694042164425,
+ "tokens_seen": 17694720
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004301587301587302,
+ "loss": 4.2665,
+ "theoretical_loss": 6.271598449171293,
+ "tokens_seen": 17760256
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00043174603174603174,
+ "loss": 4.1745,
+ "theoretical_loss": 6.267646205773557,
+ "tokens_seen": 17825792
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004333333333333333,
+ "loss": 4.1578,
+ "theoretical_loss": 6.263712517543373,
+ "tokens_seen": 17891328
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004349206349206349,
+ "loss": 4.0887,
+ "theoretical_loss": 6.25979722987654,
+ "tokens_seen": 17956864
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004365079365079365,
+ "loss": 4.1966,
+ "theoretical_loss": 6.255900190015284,
+ "tokens_seen": 18022400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00043809523809523813,
+ "loss": 4.2064,
+ "theoretical_loss": 6.252021247019596,
+ "tokens_seen": 18087936
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00043968253968253976,
+ "loss": 4.153,
+ "theoretical_loss": 6.2481602517390975,
+ "tokens_seen": 18153472
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004412698412698413,
+ "loss": 4.1275,
+ "theoretical_loss": 6.244317056785462,
+ "tokens_seen": 18219008
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004428571428571429,
+ "loss": 4.1322,
+ "theoretical_loss": 6.240491516505343,
+ "tokens_seen": 18284544
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00044444444444444447,
+ "loss": 4.1679,
+ "theoretical_loss": 6.236683486953819,
+ "tokens_seen": 18350080
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004460317460317461,
+ "loss": 4.0966,
+ "theoretical_loss": 6.232892825868342,
+ "tokens_seen": 18415616
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00044761904761904766,
+ "loss": 4.1138,
+ "theoretical_loss": 6.229119392643171,
+ "tokens_seen": 18481152
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00044920634920634923,
+ "loss": 4.1271,
+ "theoretical_loss": 6.225363048304281,
+ "tokens_seen": 18546688
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004507936507936508,
+ "loss": 4.2432,
+ "theoretical_loss": 6.221623655484739,
+ "tokens_seen": 18612224
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00045238095238095237,
+ "loss": 4.2264,
+ "theoretical_loss": 6.217901078400542,
+ "tokens_seen": 18677760
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000453968253968254,
+ "loss": 4.2129,
+ "theoretical_loss": 6.214195182826902,
+ "tokens_seen": 18743296
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00045555555555555556,
+ "loss": 4.105,
+ "theoretical_loss": 6.2105058360749545,
+ "tokens_seen": 18808832
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00045714285714285713,
+ "loss": 4.0347,
+ "theoretical_loss": 6.206832906968911,
+ "tokens_seen": 18874368
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004587301587301587,
+ "loss": 4.0975,
+ "theoretical_loss": 6.203176265823624,
+ "tokens_seen": 18939904
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00046031746031746033,
+ "loss": 4.2128,
+ "theoretical_loss": 6.1995357844225545,
+ "tokens_seen": 19005440
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004619047619047619,
+ "loss": 4.0438,
+ "theoretical_loss": 6.195911335996151,
+ "tokens_seen": 19070976
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004634920634920636,
+ "loss": 4.2097,
+ "theoretical_loss": 6.192302795200606,
+ "tokens_seen": 19136512
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00046507936507936515,
+ "loss": 4.1158,
+ "theoretical_loss": 6.188710038097013,
+ "tokens_seen": 19202048
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004666666666666667,
+ "loss": 4.1666,
+ "theoretical_loss": 6.185132942130884,
+ "tokens_seen": 19267584
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004682539682539683,
+ "loss": 4.1723,
+ "theoretical_loss": 6.181571386112038,
+ "tokens_seen": 19333120
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00046984126984126986,
+ "loss": 4.1159,
+ "theoretical_loss": 6.178025250194852,
+ "tokens_seen": 19398656
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004714285714285715,
+ "loss": 4.2241,
+ "theoretical_loss": 6.174494415858863,
+ "tokens_seen": 19464192
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00047301587301587305,
+ "loss": 4.0897,
+ "theoretical_loss": 6.170978765889705,
+ "tokens_seen": 19529728
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004746031746031746,
+ "loss": 4.1264,
+ "theoretical_loss": 6.167478184360402,
+ "tokens_seen": 19595264
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 13501,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 4.086112976074219,
+ "objective/train/theoretical_loss": 6.163992556612967,
+ "objective/train/tokens_used": 40120800,
+ "theoretical_loss": 6.163992556612967,
+ "tokens_seen": 19660800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004761904761904762,
+ "loss": 4.1434,
+ "theoretical_loss": 6.163992556612967,
+ "tokens_seen": 19660800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004777777777777778,
+ "loss": 4.0815,
+ "theoretical_loss": 6.160521769240346,
+ "tokens_seen": 19726336
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004793650793650794,
+ "loss": 4.1549,
+ "theoretical_loss": 6.157065710068663,
+ "tokens_seen": 19791872
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00048095238095238095,
+ "loss": 4.0909,
+ "theoretical_loss": 6.1536242681397795,
+ "tokens_seen": 19857408
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004825396825396825,
+ "loss": 3.918,
+ "theoretical_loss": 6.150197333694162,
+ "tokens_seen": 19922944
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004841269841269841,
+ "loss": 4.0747,
+ "theoretical_loss": 6.146784798154039,
+ "tokens_seen": 19988480
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004857142857142857,
+ "loss": 4.1139,
+ "theoretical_loss": 6.143386554106854,
+ "tokens_seen": 20054016
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004873015873015873,
+ "loss": 3.9063,
+ "theoretical_loss": 6.140002495289,
+ "tokens_seen": 20119552
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000488888888888889,
+ "loss": 4.0389,
+ "theoretical_loss": 6.136632516569836,
+ "tokens_seen": 20185088
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004904761904761905,
+ "loss": 4.0393,
+ "theoretical_loss": 6.133276513935977,
+ "tokens_seen": 20250624
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004920634920634921,
+ "loss": 4.0666,
+ "theoretical_loss": 6.129934384475845,
+ "tokens_seen": 20316160
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004936507936507937,
+ "loss": 4.0097,
+ "theoretical_loss": 6.126606026364497,
+ "tokens_seen": 20381696
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004952380952380952,
+ "loss": 4.0515,
+ "theoretical_loss": 6.123291338848694,
+ "tokens_seen": 20447232
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004968253968253969,
+ "loss": 4.0653,
+ "theoretical_loss": 6.119990222232234,
+ "tokens_seen": 20512768
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0004984126984126985,
+ "loss": 4.0558,
+ "theoretical_loss": 6.116702577861526,
+ "tokens_seen": 20578304
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005,
+ "loss": 4.1177,
+ "theoretical_loss": 6.113428308111404,
+ "tokens_seen": 20643840
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005015873015873016,
+ "loss": 4.1428,
+ "theoretical_loss": 6.110167316371188,
+ "tokens_seen": 20709376
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005031746031746032,
+ "loss": 3.9055,
+ "theoretical_loss": 6.106919507030966,
+ "tokens_seen": 20774912
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005047619047619048,
+ "loss": 4.1163,
+ "theoretical_loss": 6.103684785468113,
+ "tokens_seen": 20840448
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005063492063492064,
+ "loss": 3.929,
+ "theoretical_loss": 6.100463058034027,
+ "tokens_seen": 20905984
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005079365079365079,
+ "loss": 4.0137,
+ "theoretical_loss": 6.097254232041089,
+ "tokens_seen": 20971520
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005095238095238095,
+ "loss": 3.9822,
+ "theoretical_loss": 6.094058215749833,
+ "tokens_seen": 21037056
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005111111111111111,
+ "loss": 4.1239,
+ "theoretical_loss": 6.090874918356328,
+ "tokens_seen": 21102592
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005126984126984128,
+ "loss": 3.888,
+ "theoretical_loss": 6.087704249979775,
+ "tokens_seen": 21168128
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005142857142857143,
+ "loss": 3.9305,
+ "theoretical_loss": 6.084546121650289,
+ "tokens_seen": 21233664
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005158730158730159,
+ "loss": 4.0623,
+ "theoretical_loss": 6.081400445296899,
+ "tokens_seen": 21299200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005174603174603176,
+ "loss": 4.0524,
+ "theoretical_loss": 6.078267133735724,
+ "tokens_seen": 21364736
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005190476190476191,
+ "loss": 4.0179,
+ "theoretical_loss": 6.075146100658356,
+ "tokens_seen": 21430272
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005206349206349207,
+ "loss": 3.8274,
+ "theoretical_loss": 6.072037260620421,
+ "tokens_seen": 21495808
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005222222222222222,
+ "loss": 3.9728,
+ "theoretical_loss": 6.0689405290303275,
+ "tokens_seen": 21561344
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005238095238095238,
+ "loss": 4.0168,
+ "theoretical_loss": 6.065855822138193,
+ "tokens_seen": 21626880
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005253968253968255,
+ "loss": 3.9993,
+ "theoretical_loss": 6.06278305702495,
+ "tokens_seen": 21692416
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000526984126984127,
+ "loss": 3.9977,
+ "theoretical_loss": 6.05972215159162,
+ "tokens_seen": 21757952
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005285714285714286,
+ "loss": 4.0765,
+ "theoretical_loss": 6.056673024548766,
+ "tokens_seen": 21823488
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005301587301587302,
+ "loss": 3.9494,
+ "theoretical_loss": 6.053635595406105,
+ "tokens_seen": 21889024
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005317460317460317,
+ "loss": 3.8884,
+ "theoretical_loss": 6.050609784462289,
+ "tokens_seen": 21954560
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005333333333333334,
+ "loss": 3.9952,
+ "theoretical_loss": 6.047595512794841,
+ "tokens_seen": 22020096
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005349206349206349,
+ "loss": 3.8465,
+ "theoretical_loss": 6.044592702250255,
+ "tokens_seen": 22085632
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005365079365079365,
+ "loss": 3.8355,
+ "theoretical_loss": 6.041601275434251,
+ "tokens_seen": 22151168
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005380952380952381,
+ "loss": 3.9425,
+ "theoretical_loss": 6.038621155702174,
+ "tokens_seen": 22216704
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005396825396825397,
+ "loss": 3.9988,
+ "theoretical_loss": 6.0356522671495485,
+ "tokens_seen": 22282240
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005412698412698414,
+ "loss": 3.9923,
+ "theoretical_loss": 6.032694534602784,
+ "tokens_seen": 22347776
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005428571428571429,
+ "loss": 3.8449,
+ "theoretical_loss": 6.029747883610009,
+ "tokens_seen": 22413312
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005444444444444445,
+ "loss": 3.8843,
+ "theoretical_loss": 6.026812240432067,
+ "tokens_seen": 22478848
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000546031746031746,
+ "loss": 3.9525,
+ "theoretical_loss": 6.023887532033628,
+ "tokens_seen": 22544384
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005476190476190477,
+ "loss": 3.9851,
+ "theoretical_loss": 6.020973686074459,
+ "tokens_seen": 22609920
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005492063492063493,
+ "loss": 3.8682,
+ "theoretical_loss": 6.018070630900809,
+ "tokens_seen": 22675456
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005507936507936508,
+ "loss": 3.9546,
+ "theoretical_loss": 6.015178295536933,
+ "tokens_seen": 22740992
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005523809523809524,
+ "loss": 3.8417,
+ "theoretical_loss": 6.012296609676753,
+ "tokens_seen": 22806528
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005539682539682539,
+ "loss": 3.9796,
+ "theoretical_loss": 6.009425503675624,
+ "tokens_seen": 22872064
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 14491,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 3.740633487701416,
+ "objective/train/theoretical_loss": 6.006564908542249,
+ "objective/train/tokens_used": 43397600,
+ "theoretical_loss": 6.006564908542249,
+ "tokens_seen": 22937600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005555555555555556,
+ "loss": 3.8125,
+ "theoretical_loss": 6.006564908542249,
+ "tokens_seen": 22937600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005571428571428572,
+ "loss": 4.0247,
+ "theoretical_loss": 6.003714755930698,
+ "tokens_seen": 23003136
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005587301587301587,
+ "loss": 3.8499,
+ "theoretical_loss": 6.000874978132554,
+ "tokens_seen": 23068672
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005603174603174603,
+ "loss": 3.7935,
+ "theoretical_loss": 5.998045508069175,
+ "tokens_seen": 23134208
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005619047619047619,
+ "loss": 3.6955,
+ "theoretical_loss": 5.995226279284073,
+ "tokens_seen": 23199744
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005634920634920636,
+ "loss": 3.848,
+ "theoretical_loss": 5.992417225935405,
+ "tokens_seen": 23265280
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005650793650793652,
+ "loss": 4.0086,
+ "theoretical_loss": 5.989618282788578,
+ "tokens_seen": 23330816
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005666666666666667,
+ "loss": 4.0299,
+ "theoretical_loss": 5.9868293852089565,
+ "tokens_seen": 23396352
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005682539682539683,
+ "loss": 3.7135,
+ "theoretical_loss": 5.984050469154685,
+ "tokens_seen": 23461888
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005698412698412698,
+ "loss": 3.9052,
+ "theoretical_loss": 5.981281471169619,
+ "tokens_seen": 23527424
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005714285714285715,
+ "loss": 3.916,
+ "theoretical_loss": 5.978522328376346,
+ "tokens_seen": 23592960
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005730158730158731,
+ "loss": 3.9566,
+ "theoretical_loss": 5.975772978469321,
+ "tokens_seen": 23658496
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005746031746031746,
+ "loss": 3.861,
+ "theoretical_loss": 5.973033359708099,
+ "tokens_seen": 23724032
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005761904761904762,
+ "loss": 4.0453,
+ "theoretical_loss": 5.9703034109106685,
+ "tokens_seen": 23789568
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005777777777777778,
+ "loss": 3.8675,
+ "theoretical_loss": 5.967583071446869,
+ "tokens_seen": 23855104
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005793650793650794,
+ "loss": 3.8817,
+ "theoretical_loss": 5.9648722812319255,
+ "tokens_seen": 23920640
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000580952380952381,
+ "loss": 3.9538,
+ "theoretical_loss": 5.962170980720055,
+ "tokens_seen": 23986176
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005825396825396825,
+ "loss": 3.8526,
+ "theoretical_loss": 5.959479110898181,
+ "tokens_seen": 24051712
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005841269841269841,
+ "loss": 3.8755,
+ "theoretical_loss": 5.956796613279727,
+ "tokens_seen": 24117248
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005857142857142857,
+ "loss": 3.9102,
+ "theoretical_loss": 5.954123429898508,
+ "tokens_seen": 24182784
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005873015873015873,
+ "loss": 3.721,
+ "theoretical_loss": 5.951459503302701,
+ "tokens_seen": 24248320
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005888888888888889,
+ "loss": 3.8645,
+ "theoretical_loss": 5.948804776548908,
+ "tokens_seen": 24313856
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005904761904761905,
+ "loss": 4.0393,
+ "theoretical_loss": 5.9461591931963,
+ "tokens_seen": 24379392
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005920634920634922,
+ "loss": 3.9327,
+ "theoretical_loss": 5.943522697300843,
+ "tokens_seen": 24444928
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005936507936507937,
+ "loss": 3.6637,
+ "theoretical_loss": 5.940895233409613,
+ "tokens_seen": 24510464
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005952380952380953,
+ "loss": 3.8462,
+ "theoretical_loss": 5.938276746555175,
+ "tokens_seen": 24576000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005968253968253969,
+ "loss": 3.8563,
+ "theoretical_loss": 5.9356671822500635,
+ "tokens_seen": 24641536
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0005984126984126984,
+ "loss": 3.6734,
+ "theoretical_loss": 5.933066486481318,
+ "tokens_seen": 24707072
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006000000000000001,
+ "loss": 3.9587,
+ "theoretical_loss": 5.930474605705113,
+ "tokens_seen": 24772608
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006015873015873016,
+ "loss": 3.8129,
+ "theoretical_loss": 5.927891486841449,
+ "tokens_seen": 24838144
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006031746031746032,
+ "loss": 3.8233,
+ "theoretical_loss": 5.925317077268927,
+ "tokens_seen": 24903680
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006047619047619048,
+ "loss": 3.8697,
+ "theoretical_loss": 5.92275132481959,
+ "tokens_seen": 24969216
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006063492063492063,
+ "loss": 3.7977,
+ "theoretical_loss": 5.9201941777738405,
+ "tokens_seen": 25034752
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000607936507936508,
+ "loss": 3.8119,
+ "theoretical_loss": 5.917645584855421,
+ "tokens_seen": 25100288
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006095238095238095,
+ "loss": 3.8489,
+ "theoretical_loss": 5.915105495226474,
+ "tokens_seen": 25165824
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006111111111111111,
+ "loss": 3.7896,
+ "theoretical_loss": 5.912573858482657,
+ "tokens_seen": 25231360
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006126984126984127,
+ "loss": 3.7893,
+ "theoretical_loss": 5.910050624648335,
+ "tokens_seen": 25296896
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006142857142857143,
+ "loss": 3.6833,
+ "theoretical_loss": 5.907535744171835,
+ "tokens_seen": 25362432
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000615873015873016,
+ "loss": 3.7628,
+ "theoretical_loss": 5.905029167920765,
+ "tokens_seen": 25427968
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006174603174603175,
+ "loss": 3.7026,
+ "theoretical_loss": 5.902530847177394,
+ "tokens_seen": 25493504
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006190476190476191,
+ "loss": 3.8071,
+ "theoretical_loss": 5.900040733634098,
+ "tokens_seen": 25559040
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006206349206349206,
+ "loss": 3.8134,
+ "theoretical_loss": 5.897558779388872,
+ "tokens_seen": 25624576
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006222222222222223,
+ "loss": 3.6805,
+ "theoretical_loss": 5.895084936940887,
+ "tokens_seen": 25690112
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006238095238095239,
+ "loss": 3.7538,
+ "theoretical_loss": 5.892619159186127,
+ "tokens_seen": 25755648
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006253968253968254,
+ "loss": 3.7874,
+ "theoretical_loss": 5.890161399413071,
+ "tokens_seen": 25821184
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000626984126984127,
+ "loss": 3.7056,
+ "theoretical_loss": 5.887711611298428,
+ "tokens_seen": 25886720
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006285714285714286,
+ "loss": 3.7162,
+ "theoretical_loss": 5.885269748902955,
+ "tokens_seen": 25952256
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006301587301587302,
+ "loss": 3.6735,
+ "theoretical_loss": 5.882835766667295,
+ "tokens_seen": 26017792
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006317460317460318,
+ "loss": 3.721,
+ "theoretical_loss": 5.880409619407899,
+ "tokens_seen": 26083328
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006333333333333333,
+ "loss": 3.618,
+ "theoretical_loss": 5.877991262312986,
+ "tokens_seen": 26148864
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 15706,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 3.5033531188964844,
+ "objective/train/theoretical_loss": 5.875580650938565,
+ "objective/train/tokens_used": 46674400,
+ "theoretical_loss": 5.875580650938565,
+ "tokens_seen": 26214400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006349206349206349,
+ "loss": 3.6113,
+ "theoretical_loss": 5.875580650938565,
+ "tokens_seen": 26214400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006365079365079365,
+ "loss": 3.7019,
+ "theoretical_loss": 5.873177741204501,
+ "tokens_seen": 26279936
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006380952380952382,
+ "loss": 3.644,
+ "theoretical_loss": 5.8707824893906455,
+ "tokens_seen": 26345472
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006396825396825398,
+ "loss": 3.6657,
+ "theoretical_loss": 5.868394852133001,
+ "tokens_seen": 26411008
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006412698412698413,
+ "loss": 3.7231,
+ "theoretical_loss": 5.866014786419951,
+ "tokens_seen": 26476544
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006428571428571429,
+ "loss": 3.8293,
+ "theoretical_loss": 5.86364224958853,
+ "tokens_seen": 26542080
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006444444444444444,
+ "loss": 3.7392,
+ "theoretical_loss": 5.861277199320743,
+ "tokens_seen": 26607616
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006460317460317461,
+ "loss": 3.7438,
+ "theoretical_loss": 5.858919593639941,
+ "tokens_seen": 26673152
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006476190476190477,
+ "loss": 3.7997,
+ "theoretical_loss": 5.856569390907229,
+ "tokens_seen": 26738688
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006492063492063492,
+ "loss": 3.8363,
+ "theoretical_loss": 5.854226549817936,
+ "tokens_seen": 26804224
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006507936507936508,
+ "loss": 3.6657,
+ "theoretical_loss": 5.851891029398114,
+ "tokens_seen": 26869760
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006523809523809524,
+ "loss": 3.7011,
+ "theoretical_loss": 5.8495627890011015,
+ "tokens_seen": 26935296
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000653968253968254,
+ "loss": 3.7339,
+ "theoretical_loss": 5.847241788304114,
+ "tokens_seen": 27000832
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006555555555555556,
+ "loss": 3.77,
+ "theoretical_loss": 5.844927987304884,
+ "tokens_seen": 27066368
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006571428571428571,
+ "loss": 3.5868,
+ "theoretical_loss": 5.842621346318348,
+ "tokens_seen": 27131904
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006587301587301587,
+ "loss": 3.709,
+ "theoretical_loss": 5.84032182597337,
+ "tokens_seen": 27197440
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006603174603174604,
+ "loss": 3.6199,
+ "theoretical_loss": 5.8380293872095095,
+ "tokens_seen": 27262976
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006619047619047619,
+ "loss": 3.6197,
+ "theoretical_loss": 5.835743991273832,
+ "tokens_seen": 27328512
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006634920634920636,
+ "loss": 3.507,
+ "theoretical_loss": 5.833465599717752,
+ "tokens_seen": 27394048
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006650793650793651,
+ "loss": 3.7224,
+ "theoretical_loss": 5.831194174393929,
+ "tokens_seen": 27459584
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006666666666666668,
+ "loss": 3.5851,
+ "theoretical_loss": 5.828929677453187,
+ "tokens_seen": 27525120
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006682539682539683,
+ "loss": 3.5854,
+ "theoretical_loss": 5.826672071341488,
+ "tokens_seen": 27590656
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006698412698412699,
+ "loss": 3.6313,
+ "theoretical_loss": 5.824421318796932,
+ "tokens_seen": 27656192
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006714285714285715,
+ "loss": 3.6213,
+ "theoretical_loss": 5.822177382846799,
+ "tokens_seen": 27721728
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000673015873015873,
+ "loss": 3.6204,
+ "theoretical_loss": 5.8199402268046265,
+ "tokens_seen": 27787264
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006746031746031747,
+ "loss": 3.6112,
+ "theoretical_loss": 5.817709814267329,
+ "tokens_seen": 27852800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006761904761904762,
+ "loss": 3.7146,
+ "theoretical_loss": 5.815486109112342,
+ "tokens_seen": 27918336
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006777777777777778,
+ "loss": 3.5925,
+ "theoretical_loss": 5.813269075494812,
+ "tokens_seen": 27983872
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006793650793650794,
+ "loss": 3.5826,
+ "theoretical_loss": 5.811058677844817,
+ "tokens_seen": 28049408
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006809523809523809,
+ "loss": 3.6308,
+ "theoretical_loss": 5.808854880864618,
+ "tokens_seen": 28114944
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006825396825396826,
+ "loss": 3.5636,
+ "theoretical_loss": 5.8066576495259525,
+ "tokens_seen": 28180480
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006841269841269841,
+ "loss": 3.5684,
+ "theoretical_loss": 5.804466949067352,
+ "tokens_seen": 28246016
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006857142857142857,
+ "loss": 3.6715,
+ "theoretical_loss": 5.8022827449914995,
+ "tokens_seen": 28311552
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006873015873015873,
+ "loss": 3.6888,
+ "theoretical_loss": 5.800105003062612,
+ "tokens_seen": 28377088
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000688888888888889,
+ "loss": 3.5941,
+ "theoretical_loss": 5.797933689303866,
+ "tokens_seen": 28442624
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006904761904761906,
+ "loss": 3.6386,
+ "theoretical_loss": 5.795768769994842,
+ "tokens_seen": 28508160
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006920634920634921,
+ "loss": 3.5843,
+ "theoretical_loss": 5.793610211669004,
+ "tokens_seen": 28573696
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006936507936507937,
+ "loss": 3.5697,
+ "theoretical_loss": 5.79145798111122,
+ "tokens_seen": 28639232
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006952380952380953,
+ "loss": 3.6252,
+ "theoretical_loss": 5.789312045355292,
+ "tokens_seen": 28704768
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006968253968253969,
+ "loss": 3.6311,
+ "theoretical_loss": 5.787172371681535,
+ "tokens_seen": 28770304
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0006984126984126985,
+ "loss": 3.7606,
+ "theoretical_loss": 5.785038927614373,
+ "tokens_seen": 28835840
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007,
+ "loss": 3.3781,
+ "theoretical_loss": 5.782911680919971,
+ "tokens_seen": 28901376
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007015873015873016,
+ "loss": 3.6269,
+ "theoretical_loss": 5.78079059960389,
+ "tokens_seen": 28966912
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007031746031746032,
+ "loss": 3.5331,
+ "theoretical_loss": 5.77867565190877,
+ "tokens_seen": 29032448
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007047619047619048,
+ "loss": 3.5922,
+ "theoretical_loss": 5.776566806312052,
+ "tokens_seen": 29097984
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007063492063492064,
+ "loss": 3.572,
+ "theoretical_loss": 5.774464031523705,
+ "tokens_seen": 29163520
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007079365079365079,
+ "loss": 3.5933,
+ "theoretical_loss": 5.772367296484001,
+ "tokens_seen": 29229056
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007095238095238095,
+ "loss": 3.624,
+ "theoretical_loss": 5.770276570361309,
+ "tokens_seen": 29294592
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007111111111111111,
+ "loss": 3.5538,
+ "theoretical_loss": 5.768191822549908,
+ "tokens_seen": 29360128
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007126984126984128,
+ "loss": 3.5156,
+ "theoretical_loss": 5.766113022667838,
+ "tokens_seen": 29425664
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 16666,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 3.571195363998413,
+ "objective/train/theoretical_loss": 5.764040140554766,
+ "objective/train/tokens_used": 49951200,
+ "theoretical_loss": 5.764040140554766,
+ "tokens_seen": 29491200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007142857142857144,
+ "loss": 3.5886,
+ "theoretical_loss": 5.764040140554766,
+ "tokens_seen": 29491200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007158730158730159,
+ "loss": 3.6281,
+ "theoretical_loss": 5.761973146269881,
+ "tokens_seen": 29556736
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007174603174603175,
+ "loss": 3.3317,
+ "theoretical_loss": 5.759912010089819,
+ "tokens_seen": 29622272
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000719047619047619,
+ "loss": 3.5542,
+ "theoretical_loss": 5.757856702506597,
+ "tokens_seen": 29687808
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007206349206349207,
+ "loss": 3.584,
+ "theoretical_loss": 5.755807194225595,
+ "tokens_seen": 29753344
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007222222222222223,
+ "loss": 3.4624,
+ "theoretical_loss": 5.753763456163529,
+ "tokens_seen": 29818880
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007238095238095238,
+ "loss": 3.5575,
+ "theoretical_loss": 5.751725459446487,
+ "tokens_seen": 29884416
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007253968253968254,
+ "loss": 3.5754,
+ "theoretical_loss": 5.7496931754079466,
+ "tokens_seen": 29949952
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007269841269841271,
+ "loss": 3.6041,
+ "theoretical_loss": 5.747666575586849,
+ "tokens_seen": 30015488
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007285714285714286,
+ "loss": 3.5614,
+ "theoretical_loss": 5.7456456317256785,
+ "tokens_seen": 30081024
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007301587301587302,
+ "loss": 3.68,
+ "theoretical_loss": 5.743630315768563,
+ "tokens_seen": 30146560
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007317460317460317,
+ "loss": 3.5685,
+ "theoretical_loss": 5.741620599859402,
+ "tokens_seen": 30212096
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007333333333333333,
+ "loss": 3.646,
+ "theoretical_loss": 5.739616456340021,
+ "tokens_seen": 30277632
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000734920634920635,
+ "loss": 3.5125,
+ "theoretical_loss": 5.737617857748329,
+ "tokens_seen": 30343168
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007365079365079365,
+ "loss": 3.5059,
+ "theoretical_loss": 5.73562477681652,
+ "tokens_seen": 30408704
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007380952380952382,
+ "loss": 3.6117,
+ "theoretical_loss": 5.733637186469274,
+ "tokens_seen": 30474240
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007396825396825397,
+ "loss": 3.5285,
+ "theoretical_loss": 5.731655059821993,
+ "tokens_seen": 30539776
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007412698412698414,
+ "loss": 3.5112,
+ "theoretical_loss": 5.729678370179052,
+ "tokens_seen": 30605312
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007428571428571429,
+ "loss": 3.3423,
+ "theoretical_loss": 5.727707091032062,
+ "tokens_seen": 30670848
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007444444444444445,
+ "loss": 3.5549,
+ "theoretical_loss": 5.725741196058175,
+ "tokens_seen": 30736384
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007460317460317461,
+ "loss": 3.5069,
+ "theoretical_loss": 5.723780659118377,
+ "tokens_seen": 30801920
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007476190476190476,
+ "loss": 3.5296,
+ "theoretical_loss": 5.721825454255825,
+ "tokens_seen": 30867456
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007492063492063493,
+ "loss": 3.6073,
+ "theoretical_loss": 5.71987555569419,
+ "tokens_seen": 30932992
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007507936507936508,
+ "loss": 3.4977,
+ "theoretical_loss": 5.717930937836029,
+ "tokens_seen": 30998528
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007523809523809524,
+ "loss": 3.5041,
+ "theoretical_loss": 5.715991575261159,
+ "tokens_seen": 31064064
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000753968253968254,
+ "loss": 3.4418,
+ "theoretical_loss": 5.714057442725068,
+ "tokens_seen": 31129600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007555555555555555,
+ "loss": 3.5898,
+ "theoretical_loss": 5.712128515157324,
+ "tokens_seen": 31195136
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007571428571428572,
+ "loss": 3.5238,
+ "theoretical_loss": 5.710204767660022,
+ "tokens_seen": 31260672
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007587301587301588,
+ "loss": 3.4737,
+ "theoretical_loss": 5.708286175506232,
+ "tokens_seen": 31326208
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007603174603174603,
+ "loss": 3.5078,
+ "theoretical_loss": 5.706372714138472,
+ "tokens_seen": 31391744
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007619047619047619,
+ "loss": 3.3246,
+ "theoretical_loss": 5.704464359167195,
+ "tokens_seen": 31457280
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007634920634920635,
+ "loss": 3.5155,
+ "theoretical_loss": 5.702561086369294,
+ "tokens_seen": 31522816
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007650793650793652,
+ "loss": 3.6084,
+ "theoretical_loss": 5.7006628716866246,
+ "tokens_seen": 31588352
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007666666666666667,
+ "loss": 3.54,
+ "theoretical_loss": 5.698769691224539,
+ "tokens_seen": 31653888
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007682539682539683,
+ "loss": 3.5811,
+ "theoretical_loss": 5.696881521250441,
+ "tokens_seen": 31719424
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007698412698412699,
+ "loss": 3.4774,
+ "theoretical_loss": 5.694998338192356,
+ "tokens_seen": 31784960
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007714285714285715,
+ "loss": 3.3984,
+ "theoretical_loss": 5.693120118637513,
+ "tokens_seen": 31850496
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007730158730158731,
+ "loss": 3.495,
+ "theoretical_loss": 5.691246839330946,
+ "tokens_seen": 31916032
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007746031746031746,
+ "loss": 3.3938,
+ "theoretical_loss": 5.689378477174106,
+ "tokens_seen": 31981568
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007761904761904762,
+ "loss": 3.4542,
+ "theoretical_loss": 5.687515009223498,
+ "tokens_seen": 32047104
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007777777777777778,
+ "loss": 3.5173,
+ "theoretical_loss": 5.685656412689318,
+ "tokens_seen": 32112640
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007793650793650794,
+ "loss": 3.4633,
+ "theoretical_loss": 5.683802664934115,
+ "tokens_seen": 32178176
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000780952380952381,
+ "loss": 3.3623,
+ "theoretical_loss": 5.681953743471466,
+ "tokens_seen": 32243712
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007825396825396825,
+ "loss": 3.5352,
+ "theoretical_loss": 5.680109625964663,
+ "tokens_seen": 32309248
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007841269841269841,
+ "loss": 3.5366,
+ "theoretical_loss": 5.678270290225415,
+ "tokens_seen": 32374784
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007857142857142857,
+ "loss": 3.4648,
+ "theoretical_loss": 5.676435714212562,
+ "tokens_seen": 32440320
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007873015873015873,
+ "loss": 3.3286,
+ "theoretical_loss": 5.674605876030808,
+ "tokens_seen": 32505856
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000788888888888889,
+ "loss": 3.4303,
+ "theoretical_loss": 5.672780753929461,
+ "tokens_seen": 32571392
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007904761904761905,
+ "loss": 3.4569,
+ "theoretical_loss": 5.67096032630119,
+ "tokens_seen": 32636928
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007920634920634921,
+ "loss": 3.4918,
+ "theoretical_loss": 5.669144571680797,
+ "tokens_seen": 32702464
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 17795,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 3.261939525604248,
+ "objective/train/theoretical_loss": 5.667333468743994,
+ "objective/train/tokens_used": 53228000,
+ "theoretical_loss": 5.667333468743994,
+ "tokens_seen": 32768000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007936507936507938,
+ "loss": 3.3689,
+ "theoretical_loss": 5.667333468743994,
+ "tokens_seen": 32768000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007952380952380953,
+ "loss": 3.3711,
+ "theoretical_loss": 5.665526996306205,
+ "tokens_seen": 32833536
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968253968253969,
+ "loss": 3.5329,
+ "theoretical_loss": 5.663725133321369,
+ "tokens_seen": 32899072
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984126984126984,
+ "loss": 3.5592,
+ "theoretical_loss": 5.661927858880761,
+ "tokens_seen": 32964608
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0008,
+ "loss": 3.3173,
+ "theoretical_loss": 5.6601351522118275,
+ "tokens_seen": 33030144
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999839518555668,
+ "loss": 3.5531,
+ "theoretical_loss": 5.65834699267703,
+ "tokens_seen": 33095680
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999679037111334,
+ "loss": 3.2348,
+ "theoretical_loss": 5.6565633597727025,
+ "tokens_seen": 33161216
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999518555667002,
+ "loss": 3.4544,
+ "theoretical_loss": 5.65478423312792,
+ "tokens_seen": 33226752
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999358074222668,
+ "loss": 3.4522,
+ "theoretical_loss": 5.65300959250338,
+ "tokens_seen": 33292288
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999197592778336,
+ "loss": 3.4293,
+ "theoretical_loss": 5.651239417790293,
+ "tokens_seen": 33357824
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007999037111334002,
+ "loss": 3.3407,
+ "theoretical_loss": 5.649473689009292,
+ "tokens_seen": 33423360
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799887662988967,
+ "loss": 3.3672,
+ "theoretical_loss": 5.6477123863093395,
+ "tokens_seen": 33488896
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007998716148445336,
+ "loss": 3.403,
+ "theoretical_loss": 5.645955489966661,
+ "tokens_seen": 33554432
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007998555667001004,
+ "loss": 3.3867,
+ "theoretical_loss": 5.644202980383678,
+ "tokens_seen": 33619968
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799839518555667,
+ "loss": 3.351,
+ "theoretical_loss": 5.6424548380879616,
+ "tokens_seen": 33685504
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007998234704112338,
+ "loss": 3.4848,
+ "theoretical_loss": 5.640711043731184,
+ "tokens_seen": 33751040
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007998074222668004,
+ "loss": 3.4619,
+ "theoretical_loss": 5.6389715780880945,
+ "tokens_seen": 33816576
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007997913741223672,
+ "loss": 3.4355,
+ "theoretical_loss": 5.6372364220555,
+ "tokens_seen": 33882112
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007997753259779338,
+ "loss": 3.4325,
+ "theoretical_loss": 5.635505556651251,
+ "tokens_seen": 33947648
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007997592778335006,
+ "loss": 3.4346,
+ "theoretical_loss": 5.633778963013247,
+ "tokens_seen": 34013184
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007997432296890672,
+ "loss": 3.3384,
+ "theoretical_loss": 5.632056622398448,
+ "tokens_seen": 34078720
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799727181544634,
+ "loss": 3.4516,
+ "theoretical_loss": 5.63033851618189,
+ "tokens_seen": 34144256
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007997111334002006,
+ "loss": 3.2575,
+ "theoretical_loss": 5.628624625855725,
+ "tokens_seen": 34209792
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007996950852557674,
+ "loss": 3.3118,
+ "theoretical_loss": 5.626914933028255,
+ "tokens_seen": 34275328
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799679037111334,
+ "loss": 3.3996,
+ "theoretical_loss": 5.625209419422983,
+ "tokens_seen": 34340864
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007996629889669008,
+ "loss": 3.4081,
+ "theoretical_loss": 5.623508066877678,
+ "tokens_seen": 34406400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007996469408224675,
+ "loss": 3.3567,
+ "theoretical_loss": 5.621810857343441,
+ "tokens_seen": 34471936
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007996308926780342,
+ "loss": 3.2694,
+ "theoretical_loss": 5.620117772883784,
+ "tokens_seen": 34537472
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007996148445336009,
+ "loss": 3.4846,
+ "theoretical_loss": 5.618428795673718,
+ "tokens_seen": 34603008
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995987963891676,
+ "loss": 3.4082,
+ "theoretical_loss": 5.616743907998853,
+ "tokens_seen": 34668544
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995827482447343,
+ "loss": 3.3935,
+ "theoretical_loss": 5.6150630922545,
+ "tokens_seen": 34734080
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799566700100301,
+ "loss": 3.3234,
+ "theoretical_loss": 5.613386330944792,
+ "tokens_seen": 34799616
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995506519558677,
+ "loss": 3.3006,
+ "theoretical_loss": 5.6117136066818,
+ "tokens_seen": 34865152
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995346038114344,
+ "loss": 3.3978,
+ "theoretical_loss": 5.610044902184672,
+ "tokens_seen": 34930688
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995185556670011,
+ "loss": 3.3917,
+ "theoretical_loss": 5.608380200278775,
+ "tokens_seen": 34996224
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007995025075225678,
+ "loss": 3.2776,
+ "theoretical_loss": 5.606719483894837,
+ "tokens_seen": 35061760
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007994864593781345,
+ "loss": 3.429,
+ "theoretical_loss": 5.605062736068113,
+ "tokens_seen": 35127296
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007994704112337012,
+ "loss": 3.3658,
+ "theoretical_loss": 5.603409939937547,
+ "tokens_seen": 35192832
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007994543630892679,
+ "loss": 3.3409,
+ "theoretical_loss": 5.601761078744952,
+ "tokens_seen": 35258368
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007994383149448346,
+ "loss": 3.3312,
+ "theoretical_loss": 5.600116135834183,
+ "tokens_seen": 35323904
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007994222668004013,
+ "loss": 3.1932,
+ "theoretical_loss": 5.598475094650334,
+ "tokens_seen": 35389440
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799406218655968,
+ "loss": 3.2632,
+ "theoretical_loss": 5.59683793873894,
+ "tokens_seen": 35454976
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993901705115347,
+ "loss": 3.3286,
+ "theoretical_loss": 5.595204651745176,
+ "tokens_seen": 35520512
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993741223671014,
+ "loss": 3.1714,
+ "theoretical_loss": 5.593575217413074,
+ "tokens_seen": 35586048
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993580742226681,
+ "loss": 3.2249,
+ "theoretical_loss": 5.591949619584746,
+ "tokens_seen": 35651584
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993420260782347,
+ "loss": 3.1943,
+ "theoretical_loss": 5.590327842199611,
+ "tokens_seen": 35717120
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993259779338015,
+ "loss": 3.261,
+ "theoretical_loss": 5.588709869293634,
+ "tokens_seen": 35782656
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007993099297893681,
+ "loss": 3.3474,
+ "theoretical_loss": 5.5870956849985705,
+ "tokens_seen": 35848192
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992938816449349,
+ "loss": 3.4015,
+ "theoretical_loss": 5.585485273541212,
+ "tokens_seen": 35913728
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992778335005015,
+ "loss": 3.3678,
+ "theoretical_loss": 5.583878619242652,
+ "tokens_seen": 35979264
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 18676,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 3.2103042602539062,
+ "objective/train/theoretical_loss": 5.582275706517549,
+ "objective/train/tokens_used": 56504800,
+ "theoretical_loss": 5.582275706517549,
+ "tokens_seen": 36044800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992617853560683,
+ "loss": 3.3616,
+ "theoretical_loss": 5.582275706517549,
+ "tokens_seen": 36044800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992457372116349,
+ "loss": 3.3332,
+ "theoretical_loss": 5.580676519873396,
+ "tokens_seen": 36110336
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992296890672017,
+ "loss": 3.5113,
+ "theoretical_loss": 5.579081043909808,
+ "tokens_seen": 36175872
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007992136409227683,
+ "loss": 3.4334,
+ "theoretical_loss": 5.577489263317796,
+ "tokens_seen": 36241408
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799197592778335,
+ "loss": 3.3125,
+ "theoretical_loss": 5.5759011628790764,
+ "tokens_seen": 36306944
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991815446339017,
+ "loss": 3.4013,
+ "theoretical_loss": 5.574316727465358,
+ "tokens_seen": 36372480
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991654964894684,
+ "loss": 3.2732,
+ "theoretical_loss": 5.572735942037655,
+ "tokens_seen": 36438016
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991494483450351,
+ "loss": 3.1625,
+ "theoretical_loss": 5.571158791645597,
+ "tokens_seen": 36503552
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991334002006018,
+ "loss": 3.284,
+ "theoretical_loss": 5.569585261426752,
+ "tokens_seen": 36569088
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991173520561685,
+ "loss": 3.4507,
+ "theoretical_loss": 5.568015336605953,
+ "tokens_seen": 36634624
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007991013039117352,
+ "loss": 3.3153,
+ "theoretical_loss": 5.566449002494627,
+ "tokens_seen": 36700160
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007990852557673019,
+ "loss": 3.135,
+ "theoretical_loss": 5.564886244490136,
+ "tokens_seen": 36765696
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007990692076228686,
+ "loss": 3.3133,
+ "theoretical_loss": 5.563327048075124,
+ "tokens_seen": 36831232
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007990531594784353,
+ "loss": 3.3034,
+ "theoretical_loss": 5.5617713988168616,
+ "tokens_seen": 36896768
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000799037111334002,
+ "loss": 3.2984,
+ "theoretical_loss": 5.560219282366614,
+ "tokens_seen": 36962304
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007990210631895687,
+ "loss": 3.3318,
+ "theoretical_loss": 5.558670684458994,
+ "tokens_seen": 37027840
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007990050150451354,
+ "loss": 3.3465,
+ "theoretical_loss": 5.557125590911339,
+ "tokens_seen": 37093376
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989889669007021,
+ "loss": 3.3319,
+ "theoretical_loss": 5.5555839876230815,
+ "tokens_seen": 37158912
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989729187562688,
+ "loss": 3.2583,
+ "theoretical_loss": 5.55404586057513,
+ "tokens_seen": 37224448
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989568706118355,
+ "loss": 3.1394,
+ "theoretical_loss": 5.5525111958292594,
+ "tokens_seen": 37289984
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989408224674022,
+ "loss": 3.3485,
+ "theoretical_loss": 5.550979979527501,
+ "tokens_seen": 37355520
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989247743229689,
+ "loss": 3.3191,
+ "theoretical_loss": 5.549452197891538,
+ "tokens_seen": 37421056
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007989087261785356,
+ "loss": 3.243,
+ "theoretical_loss": 5.547927837222119,
+ "tokens_seen": 37486592
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007988926780341023,
+ "loss": 3.128,
+ "theoretical_loss": 5.546406883898451,
+ "tokens_seen": 37552128
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000798876629889669,
+ "loss": 3.2572,
+ "theoretical_loss": 5.544889324377628,
+ "tokens_seen": 37617664
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007988605817452357,
+ "loss": 3.1857,
+ "theoretical_loss": 5.543375145194048,
+ "tokens_seen": 37683200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007988445336008024,
+ "loss": 3.1529,
+ "theoretical_loss": 5.541864332958831,
+ "tokens_seen": 37748736
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007988284854563691,
+ "loss": 3.2037,
+ "theoretical_loss": 5.540356874359257,
+ "tokens_seen": 37814272
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007988124373119358,
+ "loss": 3.2591,
+ "theoretical_loss": 5.538852756158201,
+ "tokens_seen": 37879808
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987963891675025,
+ "loss": 3.1802,
+ "theoretical_loss": 5.537351965193569,
+ "tokens_seen": 37945344
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987803410230693,
+ "loss": 3.1964,
+ "theoretical_loss": 5.535854488377751,
+ "tokens_seen": 38010880
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987642928786359,
+ "loss": 3.346,
+ "theoretical_loss": 5.534360312697064,
+ "tokens_seen": 38076416
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987482447342027,
+ "loss": 3.1631,
+ "theoretical_loss": 5.532869425211214,
+ "tokens_seen": 38141952
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987321965897693,
+ "loss": 3.2718,
+ "theoretical_loss": 5.531381813052757,
+ "tokens_seen": 38207488
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987161484453361,
+ "loss": 3.17,
+ "theoretical_loss": 5.529897463426559,
+ "tokens_seen": 38273024
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007987001003009027,
+ "loss": 3.2378,
+ "theoretical_loss": 5.528416363609276,
+ "tokens_seen": 38338560
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986840521564695,
+ "loss": 3.1696,
+ "theoretical_loss": 5.526938500948817,
+ "tokens_seen": 38404096
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986680040120361,
+ "loss": 3.2103,
+ "theoretical_loss": 5.525463862863836,
+ "tokens_seen": 38469632
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986519558676029,
+ "loss": 3.0849,
+ "theoretical_loss": 5.523992436843209,
+ "tokens_seen": 38535168
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986359077231695,
+ "loss": 3.2743,
+ "theoretical_loss": 5.522524210445528,
+ "tokens_seen": 38600704
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986198595787363,
+ "loss": 3.272,
+ "theoretical_loss": 5.521059171298592,
+ "tokens_seen": 38666240
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007986038114343029,
+ "loss": 3.3046,
+ "theoretical_loss": 5.519597307098907,
+ "tokens_seen": 38731776
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985877632898697,
+ "loss": 3.2986,
+ "theoretical_loss": 5.518138605611188,
+ "tokens_seen": 38797312
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985717151454363,
+ "loss": 3.2102,
+ "theoretical_loss": 5.516683054667868,
+ "tokens_seen": 38862848
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985556670010031,
+ "loss": 3.3232,
+ "theoretical_loss": 5.515230642168607,
+ "tokens_seen": 38928384
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985396188565697,
+ "loss": 3.2207,
+ "theoretical_loss": 5.513781356079811,
+ "tokens_seen": 38993920
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985235707121365,
+ "loss": 3.1524,
+ "theoretical_loss": 5.512335184434152,
+ "tokens_seen": 39059456
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007985075225677031,
+ "loss": 3.3457,
+ "theoretical_loss": 5.5108921153300905,
+ "tokens_seen": 39124992
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984914744232699,
+ "loss": 3.1989,
+ "theoretical_loss": 5.5094521369314045,
+ "tokens_seen": 39190528
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984754262788365,
+ "loss": 3.0462,
+ "theoretical_loss": 5.5080152374667275,
+ "tokens_seen": 39256064
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 19810,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.971580982208252,
+ "objective/train/theoretical_loss": 5.506581405229079,
+ "objective/train/tokens_used": 59781600,
+ "theoretical_loss": 5.506581405229079,
+ "tokens_seen": 39321600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984593781344033,
+ "loss": 3.0874,
+ "theoretical_loss": 5.506581405229079,
+ "tokens_seen": 39321600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984433299899699,
+ "loss": 3.2052,
+ "theoretical_loss": 5.505150628575409,
+ "tokens_seen": 39387136
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984272818455367,
+ "loss": 3.164,
+ "theoretical_loss": 5.503722895926143,
+ "tokens_seen": 39452672
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007984112337011033,
+ "loss": 3.0422,
+ "theoretical_loss": 5.50229819576473,
+ "tokens_seen": 39518208
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983951855566701,
+ "loss": 3.0343,
+ "theoretical_loss": 5.500876516637199,
+ "tokens_seen": 39583744
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983791374122367,
+ "loss": 3.1188,
+ "theoretical_loss": 5.49945784715171,
+ "tokens_seen": 39649280
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983630892678035,
+ "loss": 3.0288,
+ "theoretical_loss": 5.498042175978123,
+ "tokens_seen": 39714816
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983470411233701,
+ "loss": 3.2065,
+ "theoretical_loss": 5.496629491847553,
+ "tokens_seen": 39780352
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983309929789369,
+ "loss": 3.1265,
+ "theoretical_loss": 5.495219783551947,
+ "tokens_seen": 39845888
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007983149448345036,
+ "loss": 3.1179,
+ "theoretical_loss": 5.493813039943654,
+ "tokens_seen": 39911424
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982988966900703,
+ "loss": 2.9897,
+ "theoretical_loss": 5.492409249934996,
+ "tokens_seen": 39976960
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000798282848545637,
+ "loss": 3.1839,
+ "theoretical_loss": 5.491008402497855,
+ "tokens_seen": 40042496
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982668004012037,
+ "loss": 3.1042,
+ "theoretical_loss": 5.489610486663252,
+ "tokens_seen": 40108032
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982507522567704,
+ "loss": 3.0255,
+ "theoretical_loss": 5.488215491520933,
+ "tokens_seen": 40173568
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982347041123371,
+ "loss": 3.0889,
+ "theoretical_loss": 5.486823406218962,
+ "tokens_seen": 40239104
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982186559679038,
+ "loss": 3.2075,
+ "theoretical_loss": 5.485434219963315,
+ "tokens_seen": 40304640
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007982026078234705,
+ "loss": 3.0804,
+ "theoretical_loss": 5.484047922017476,
+ "tokens_seen": 40370176
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007981865596790372,
+ "loss": 3.199,
+ "theoretical_loss": 5.482664501702036,
+ "tokens_seen": 40435712
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007981705115346039,
+ "loss": 3.0804,
+ "theoretical_loss": 5.481283948394303,
+ "tokens_seen": 40501248
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007981544633901706,
+ "loss": 3.1914,
+ "theoretical_loss": 5.479906251527901,
+ "tokens_seen": 40566784
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007981384152457373,
+ "loss": 3.1122,
+ "theoretical_loss": 5.478531400592393,
+ "tokens_seen": 40632320
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000798122367101304,
+ "loss": 3.0153,
+ "theoretical_loss": 5.4771593851328815,
+ "tokens_seen": 40697856
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007981063189568707,
+ "loss": 3.0745,
+ "theoretical_loss": 5.475790194749637,
+ "tokens_seen": 40763392
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980902708124374,
+ "loss": 3.2291,
+ "theoretical_loss": 5.474423819097713,
+ "tokens_seen": 40828928
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980742226680041,
+ "loss": 3.072,
+ "theoretical_loss": 5.473060247886572,
+ "tokens_seen": 40894464
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980581745235708,
+ "loss": 3.0078,
+ "theoretical_loss": 5.471699470879715,
+ "tokens_seen": 40960000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980421263791375,
+ "loss": 3.0983,
+ "theoretical_loss": 5.470341477894303,
+ "tokens_seen": 41025536
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980260782347042,
+ "loss": 3.1138,
+ "theoretical_loss": 5.468986258800805,
+ "tokens_seen": 41091072
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007980100300902709,
+ "loss": 3.0204,
+ "theoretical_loss": 5.4676338035226175,
+ "tokens_seen": 41156608
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007979939819458376,
+ "loss": 3.2892,
+ "theoretical_loss": 5.466284102035719,
+ "tokens_seen": 41222144
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007979779338014043,
+ "loss": 3.1621,
+ "theoretical_loss": 5.464937144368307,
+ "tokens_seen": 41287680
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797961885656971,
+ "loss": 3.0173,
+ "theoretical_loss": 5.463592920600435,
+ "tokens_seen": 41353216
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007979458375125376,
+ "loss": 3.0774,
+ "theoretical_loss": 5.46225142086368,
+ "tokens_seen": 41418752
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007979297893681044,
+ "loss": 3.1238,
+ "theoretical_loss": 5.460912635340775,
+ "tokens_seen": 41484288
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797913741223671,
+ "loss": 3.0618,
+ "theoretical_loss": 5.459576554265276,
+ "tokens_seen": 41549824
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978976930792378,
+ "loss": 3.1313,
+ "theoretical_loss": 5.458243167921212,
+ "tokens_seen": 41615360
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978816449348044,
+ "loss": 3.2165,
+ "theoretical_loss": 5.456912466642748,
+ "tokens_seen": 41680896
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978655967903712,
+ "loss": 2.9085,
+ "theoretical_loss": 5.455584440813853,
+ "tokens_seen": 41746432
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978495486459378,
+ "loss": 2.9859,
+ "theoretical_loss": 5.454259080867951,
+ "tokens_seen": 41811968
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978335005015046,
+ "loss": 2.9411,
+ "theoretical_loss": 5.4529363772876085,
+ "tokens_seen": 41877504
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978174523570712,
+ "loss": 3.0094,
+ "theoretical_loss": 5.451616320604193,
+ "tokens_seen": 41943040
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007978014042126379,
+ "loss": 2.9767,
+ "theoretical_loss": 5.450298901397547,
+ "tokens_seen": 42008576
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007977853560682046,
+ "loss": 3.1042,
+ "theoretical_loss": 5.448984110295672,
+ "tokens_seen": 42074112
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007977693079237713,
+ "loss": 3.0762,
+ "theoretical_loss": 5.447671937974404,
+ "tokens_seen": 42139648
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797753259779338,
+ "loss": 2.9742,
+ "theoretical_loss": 5.446362375157092,
+ "tokens_seen": 42205184
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007977372116349047,
+ "loss": 3.2779,
+ "theoretical_loss": 5.445055412614287,
+ "tokens_seen": 42270720
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007977211634904714,
+ "loss": 2.9849,
+ "theoretical_loss": 5.443751041163428,
+ "tokens_seen": 42336256
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007977051153460381,
+ "loss": 3.0884,
+ "theoretical_loss": 5.442449251668533,
+ "tokens_seen": 42401792
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976890672016048,
+ "loss": 3.0079,
+ "theoretical_loss": 5.441150035039886,
+ "tokens_seen": 42467328
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976730190571715,
+ "loss": 3.0901,
+ "theoretical_loss": 5.439853382233738,
+ "tokens_seen": 42532864
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 20768,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.992009401321411,
+ "objective/train/theoretical_loss": 5.438559284252003,
+ "objective/train/tokens_used": 63058400,
+ "theoretical_loss": 5.438559284252003,
+ "tokens_seen": 42598400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976569709127382,
+ "loss": 2.9284,
+ "theoretical_loss": 5.438559284252003,
+ "tokens_seen": 42598400
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976409227683049,
+ "loss": 2.9027,
+ "theoretical_loss": 5.437267732141949,
+ "tokens_seen": 42663936
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976248746238716,
+ "loss": 2.9383,
+ "theoretical_loss": 5.435978716995917,
+ "tokens_seen": 42729472
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007976088264794383,
+ "loss": 3.0424,
+ "theoretical_loss": 5.434692229951009,
+ "tokens_seen": 42795008
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797592778335005,
+ "loss": 3.0061,
+ "theoretical_loss": 5.433408262188802,
+ "tokens_seen": 42860544
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007975767301905718,
+ "loss": 3.0298,
+ "theoretical_loss": 5.432126804935059,
+ "tokens_seen": 42926080
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007975606820461384,
+ "loss": 3.1409,
+ "theoretical_loss": 5.430847849459438,
+ "tokens_seen": 42991616
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007975446339017052,
+ "loss": 3.0722,
+ "theoretical_loss": 5.429571387075203,
+ "tokens_seen": 43057152
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007975285857572718,
+ "loss": 3.055,
+ "theoretical_loss": 5.428297409138947,
+ "tokens_seen": 43122688
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007975125376128386,
+ "loss": 2.936,
+ "theoretical_loss": 5.427025907050307,
+ "tokens_seen": 43188224
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007974964894684052,
+ "loss": 3.0608,
+ "theoretical_loss": 5.425756872251682,
+ "tokens_seen": 43253760
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797480441323972,
+ "loss": 2.969,
+ "theoretical_loss": 5.42449029622796,
+ "tokens_seen": 43319296
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007974643931795386,
+ "loss": 2.982,
+ "theoretical_loss": 5.423226170506243,
+ "tokens_seen": 43384832
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007974483450351054,
+ "loss": 2.959,
+ "theoretical_loss": 5.421964486655567,
+ "tokens_seen": 43450368
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797432296890672,
+ "loss": 3.1053,
+ "theoretical_loss": 5.420705236286646,
+ "tokens_seen": 43515904
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007974162487462388,
+ "loss": 3.1943,
+ "theoretical_loss": 5.41944841105159,
+ "tokens_seen": 43581440
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007974002006018054,
+ "loss": 2.9255,
+ "theoretical_loss": 5.418194002643643,
+ "tokens_seen": 43646976
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007973841524573722,
+ "loss": 3.0381,
+ "theoretical_loss": 5.416942002796926,
+ "tokens_seen": 43712512
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007973681043129388,
+ "loss": 2.9989,
+ "theoretical_loss": 5.415692403286162,
+ "tokens_seen": 43778048
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007973520561685056,
+ "loss": 3.0323,
+ "theoretical_loss": 5.414445195926426,
+ "tokens_seen": 43843584
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007973360080240722,
+ "loss": 2.9815,
+ "theoretical_loss": 5.413200372572884,
+ "tokens_seen": 43909120
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797319959879639,
+ "loss": 3.0141,
+ "theoretical_loss": 5.411957925120538,
+ "tokens_seen": 43974656
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007973039117352056,
+ "loss": 3.0708,
+ "theoretical_loss": 5.410717845503969,
+ "tokens_seen": 44040192
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007972878635907724,
+ "loss": 2.8975,
+ "theoretical_loss": 5.40948012569709,
+ "tokens_seen": 44105728
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797271815446339,
+ "loss": 3.0037,
+ "theoretical_loss": 5.408244757712893,
+ "tokens_seen": 44171264
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007972557673019058,
+ "loss": 2.9777,
+ "theoretical_loss": 5.407011733603199,
+ "tokens_seen": 44236800
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007972397191574724,
+ "loss": 2.885,
+ "theoretical_loss": 5.4057810454584185,
+ "tokens_seen": 44302336
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007972236710130392,
+ "loss": 2.9414,
+ "theoretical_loss": 5.404552685407301,
+ "tokens_seen": 44367872
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007972076228686058,
+ "loss": 3.097,
+ "theoretical_loss": 5.403326645616698,
+ "tokens_seen": 44433408
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007971915747241726,
+ "loss": 2.9061,
+ "theoretical_loss": 5.402102918291316,
+ "tokens_seen": 44498944
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007971755265797392,
+ "loss": 3.0093,
+ "theoretical_loss": 5.400881495673488,
+ "tokens_seen": 44564480
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797159478435306,
+ "loss": 3.103,
+ "theoretical_loss": 5.3996623700429245,
+ "tokens_seen": 44630016
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007971434302908726,
+ "loss": 2.9274,
+ "theoretical_loss": 5.398445533716492,
+ "tokens_seen": 44695552
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007971273821464394,
+ "loss": 3.0244,
+ "theoretical_loss": 5.3972309790479684,
+ "tokens_seen": 44761088
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000797111334002006,
+ "loss": 2.9874,
+ "theoretical_loss": 5.39601869842782,
+ "tokens_seen": 44826624
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970952858575728,
+ "loss": 2.893,
+ "theoretical_loss": 5.394808684282968,
+ "tokens_seen": 44892160
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970792377131394,
+ "loss": 2.9119,
+ "theoretical_loss": 5.393600929076564,
+ "tokens_seen": 44957696
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970631895687062,
+ "loss": 2.845,
+ "theoretical_loss": 5.392395425307758,
+ "tokens_seen": 45023232
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970471414242728,
+ "loss": 2.8557,
+ "theoretical_loss": 5.391192165511484,
+ "tokens_seen": 45088768
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970310932798396,
+ "loss": 2.9916,
+ "theoretical_loss": 5.389991142258227,
+ "tokens_seen": 45154304
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007970150451354062,
+ "loss": 3.0289,
+ "theoretical_loss": 5.38879234815381,
+ "tokens_seen": 45219840
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000796998996990973,
+ "loss": 2.857,
+ "theoretical_loss": 5.387595775839175,
+ "tokens_seen": 45285376
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007969829488465396,
+ "loss": 2.9597,
+ "theoretical_loss": 5.386401417990159,
+ "tokens_seen": 45350912
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007969669007021064,
+ "loss": 2.9621,
+ "theoretical_loss": 5.385209267317286,
+ "tokens_seen": 45416448
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000796950852557673,
+ "loss": 2.9259,
+ "theoretical_loss": 5.3840193165655466,
+ "tokens_seen": 45481984
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007969348044132398,
+ "loss": 2.8076,
+ "theoretical_loss": 5.382831558514191,
+ "tokens_seen": 45547520
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007969187562688065,
+ "loss": 2.9423,
+ "theoretical_loss": 5.3816459859765144,
+ "tokens_seen": 45613056
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007969027081243732,
+ "loss": 2.7761,
+ "theoretical_loss": 5.380462591799651,
+ "tokens_seen": 45678592
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968866599799399,
+ "loss": 2.8484,
+ "theoretical_loss": 5.379281368864362,
+ "tokens_seen": 45744128
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968706118355066,
+ "loss": 2.8477,
+ "theoretical_loss": 5.378102310084834,
+ "tokens_seen": 45809664
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 22096,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.943756580352783,
+ "objective/train/theoretical_loss": 5.376925408408475,
+ "objective/train/tokens_used": 66335200,
+ "theoretical_loss": 5.376925408408475,
+ "tokens_seen": 45875200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968545636910733,
+ "loss": 2.9656,
+ "theoretical_loss": 5.376925408408475,
+ "tokens_seen": 45875200
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.00079683851554664,
+ "loss": 2.8387,
+ "theoretical_loss": 5.375750656815705,
+ "tokens_seen": 45940736
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968224674022067,
+ "loss": 2.9168,
+ "theoretical_loss": 5.37457804831976,
+ "tokens_seen": 46006272
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007968064192577734,
+ "loss": 2.8562,
+ "theoretical_loss": 5.373407575966493,
+ "tokens_seen": 46071808
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967903711133401,
+ "loss": 2.8091,
+ "theoretical_loss": 5.372239232834173,
+ "tokens_seen": 46137344
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967743229689068,
+ "loss": 2.9054,
+ "theoretical_loss": 5.3710730120332855,
+ "tokens_seen": 46202880
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967582748244735,
+ "loss": 2.9022,
+ "theoretical_loss": 5.369908906706346,
+ "tokens_seen": 46268416
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967422266800402,
+ "loss": 2.9656,
+ "theoretical_loss": 5.368746910027696,
+ "tokens_seen": 46333952
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967261785356069,
+ "loss": 2.7845,
+ "theoretical_loss": 5.3675870152033145,
+ "tokens_seen": 46399488
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007967101303911736,
+ "loss": 2.8521,
+ "theoretical_loss": 5.36642921547063,
+ "tokens_seen": 46465024
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007966940822467403,
+ "loss": 2.8249,
+ "theoretical_loss": 5.365273504098326,
+ "tokens_seen": 46530560
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.000796678034102307,
+ "loss": 2.7694,
+ "theoretical_loss": 5.364119874386157,
+ "tokens_seen": 46596096
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007966619859578737,
+ "loss": 2.9288,
+ "theoretical_loss": 5.362968319664756,
+ "tokens_seen": 46661632
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007966459378134404,
+ "loss": 2.9025,
+ "theoretical_loss": 5.361818833295455,
+ "tokens_seen": 46727168
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007966298896690071,
+ "loss": 2.9219,
+ "theoretical_loss": 5.3606714086701,
+ "tokens_seen": 46792704
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007966138415245738,
+ "loss": 2.9099,
+ "theoretical_loss": 5.359526039210861,
+ "tokens_seen": 46858240
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965977933801405,
+ "loss": 2.88,
+ "theoretical_loss": 5.358382718370063,
+ "tokens_seen": 46923776
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965817452357072,
+ "loss": 2.8903,
+ "theoretical_loss": 5.357241439629992,
+ "tokens_seen": 46989312
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965656970912739,
+ "loss": 2.7691,
+ "theoretical_loss": 5.3561021965027305,
+ "tokens_seen": 47054848
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965496489468405,
+ "loss": 2.8294,
+ "theoretical_loss": 5.354964982529967,
+ "tokens_seen": 47120384
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965336008024073,
+ "loss": 2.8102,
+ "theoretical_loss": 5.353829791282831,
+ "tokens_seen": 47185920
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965175526579739,
+ "loss": 2.7822,
+ "theoretical_loss": 5.35269661636171,
+ "tokens_seen": 47251456
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007965015045135407,
+ "loss": 2.8876,
+ "theoretical_loss": 5.35156545139608,
+ "tokens_seen": 47316992
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964854563691073,
+ "loss": 2.7635,
+ "theoretical_loss": 5.350436290044332,
+ "tokens_seen": 47382528
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964694082246741,
+ "loss": 2.8377,
+ "theoretical_loss": 5.3493091259936,
+ "tokens_seen": 47448064
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964533600802407,
+ "loss": 2.8324,
+ "theoretical_loss": 5.348183952959596,
+ "tokens_seen": 47513600
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964373119358075,
+ "loss": 2.7908,
+ "theoretical_loss": 5.347060764686436,
+ "tokens_seen": 47579136
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964212637913741,
+ "loss": 2.8496,
+ "theoretical_loss": 5.345939554946472,
+ "tokens_seen": 47644672
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007964052156469408,
+ "loss": 2.7488,
+ "theoretical_loss": 5.344820317540132,
+ "tokens_seen": 47710208
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963891675025075,
+ "loss": 2.8114,
+ "theoretical_loss": 5.343703046295751,
+ "tokens_seen": 47775744
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963731193580743,
+ "loss": 2.8671,
+ "theoretical_loss": 5.342587735069408,
+ "tokens_seen": 47841280
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963570712136409,
+ "loss": 2.9205,
+ "theoretical_loss": 5.3414743777447615,
+ "tokens_seen": 47906816
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963410230692077,
+ "loss": 2.7384,
+ "theoretical_loss": 5.340362968232892,
+ "tokens_seen": 47972352
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963249749247743,
+ "loss": 2.9322,
+ "theoretical_loss": 5.339253500472138,
+ "tokens_seen": 48037888
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007963089267803411,
+ "loss": 2.6983,
+ "theoretical_loss": 5.338145968427941,
+ "tokens_seen": 48103424
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962928786359077,
+ "loss": 2.8289,
+ "theoretical_loss": 5.337040366092687,
+ "tokens_seen": 48168960
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962768304914745,
+ "loss": 2.9222,
+ "theoretical_loss": 5.335936687485542,
+ "tokens_seen": 48234496
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962607823470411,
+ "loss": 2.7881,
+ "theoretical_loss": 5.334834926652308,
+ "tokens_seen": 48300032
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962447342026079,
+ "loss": 2.7958,
+ "theoretical_loss": 5.33373507766526,
+ "tokens_seen": 48365568
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962286860581745,
+ "loss": 2.8646,
+ "theoretical_loss": 5.332637134622999,
+ "tokens_seen": 48431104
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007962126379137413,
+ "loss": 2.7199,
+ "theoretical_loss": 5.331541091650294,
+ "tokens_seen": 48496640
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961965897693079,
+ "loss": 2.8311,
+ "theoretical_loss": 5.330446942897931,
+ "tokens_seen": 48562176
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961805416248747,
+ "loss": 2.8684,
+ "theoretical_loss": 5.32935468254257,
+ "tokens_seen": 48627712
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961644934804413,
+ "loss": 2.822,
+ "theoretical_loss": 5.328264304786586,
+ "tokens_seen": 48693248
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961484453360081,
+ "loss": 2.7969,
+ "theoretical_loss": 5.327175803857929,
+ "tokens_seen": 48758784
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961323971915747,
+ "loss": 2.7726,
+ "theoretical_loss": 5.326089174009972,
+ "tokens_seen": 48824320
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961163490471415,
+ "loss": 2.7235,
+ "theoretical_loss": 5.325004409521368,
+ "tokens_seen": 48889856
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007961003009027081,
+ "loss": 2.7985,
+ "theoretical_loss": 5.323921504695903,
+ "tokens_seen": 48955392
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960842527582749,
+ "loss": 2.9327,
+ "theoretical_loss": 5.322840453862356,
+ "tokens_seen": 49020928
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960682046138415,
+ "loss": 2.7259,
+ "theoretical_loss": 5.3217612513743475,
+ "tokens_seen": 49086464
+ },
+ {
+ "epoch": 0.01,
+ "objective/train/docs_used": 22959,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.8861923217773438,
+ "objective/train/theoretical_loss": 5.320683891610208,
+ "objective/train/tokens_used": 69612000,
+ "theoretical_loss": 5.320683891610208,
+ "tokens_seen": 49152000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960521564694083,
+ "loss": 2.8363,
+ "theoretical_loss": 5.320683891610208,
+ "tokens_seen": 49152000
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960361083249749,
+ "loss": 2.8106,
+ "theoretical_loss": 5.3196083689728315,
+ "tokens_seen": 49217536
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960200601805417,
+ "loss": 2.8124,
+ "theoretical_loss": 5.318534677889536,
+ "tokens_seen": 49283072
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007960040120361083,
+ "loss": 2.6993,
+ "theoretical_loss": 5.317462812811925,
+ "tokens_seen": 49348608
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007959879638916751,
+ "loss": 2.7378,
+ "theoretical_loss": 5.316392768215751,
+ "tokens_seen": 49414144
+ },
+ {
+ "epoch": 0.01,
+ "learning_rate": 0.0007959719157472417,
+ "loss": 2.7691,
+ "theoretical_loss": 5.315324538600778,
+ "tokens_seen": 49479680
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007959558676028085,
+ "loss": 2.9151,
+ "theoretical_loss": 5.31425811849064,
+ "tokens_seen": 49545216
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007959398194583751,
+ "loss": 2.616,
+ "theoretical_loss": 5.313193502432718,
+ "tokens_seen": 49610752
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007959237713139419,
+ "loss": 2.7306,
+ "theoretical_loss": 5.312130684997992,
+ "tokens_seen": 49676288
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007959077231695085,
+ "loss": 2.801,
+ "theoretical_loss": 5.31106966078092,
+ "tokens_seen": 49741824
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958916750250753,
+ "loss": 2.736,
+ "theoretical_loss": 5.310010424399295,
+ "tokens_seen": 49807360
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958756268806419,
+ "loss": 2.7149,
+ "theoretical_loss": 5.308952970494124,
+ "tokens_seen": 49872896
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958595787362087,
+ "loss": 2.7643,
+ "theoretical_loss": 5.307897293729489,
+ "tokens_seen": 49938432
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958435305917753,
+ "loss": 2.7688,
+ "theoretical_loss": 5.306843388792423,
+ "tokens_seen": 50003968
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958274824473421,
+ "loss": 2.6344,
+ "theoretical_loss": 5.305791250392778,
+ "tokens_seen": 50069504
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007958114343029087,
+ "loss": 2.7253,
+ "theoretical_loss": 5.304740873263098,
+ "tokens_seen": 50135040
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957953861584755,
+ "loss": 2.7176,
+ "theoretical_loss": 5.303692252158496,
+ "tokens_seen": 50200576
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957793380140421,
+ "loss": 2.7815,
+ "theoretical_loss": 5.302645381856522,
+ "tokens_seen": 50266112
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957632898696089,
+ "loss": 2.7198,
+ "theoretical_loss": 5.30160025715704,
+ "tokens_seen": 50331648
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957472417251755,
+ "loss": 2.5469,
+ "theoretical_loss": 5.300556872882108,
+ "tokens_seen": 50397184
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957311935807423,
+ "loss": 2.6499,
+ "theoretical_loss": 5.299515223875848,
+ "tokens_seen": 50462720
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007957151454363089,
+ "loss": 2.7165,
+ "theoretical_loss": 5.2984753050043265,
+ "tokens_seen": 50528256
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956990972918757,
+ "loss": 2.6517,
+ "theoretical_loss": 5.2974371111554355,
+ "tokens_seen": 50593792
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956830491474423,
+ "loss": 2.6449,
+ "theoretical_loss": 5.296400637238764,
+ "tokens_seen": 50659328
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956670010030091,
+ "loss": 2.5609,
+ "theoretical_loss": 5.295365878185488,
+ "tokens_seen": 50724864
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956509528585757,
+ "loss": 2.4886,
+ "theoretical_loss": 5.294332828948243,
+ "tokens_seen": 50790400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956349047141425,
+ "loss": 2.6212,
+ "theoretical_loss": 5.2933014845010105,
+ "tokens_seen": 50855936
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956188565697091,
+ "loss": 2.6007,
+ "theoretical_loss": 5.292271839838996,
+ "tokens_seen": 50921472
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007956028084252759,
+ "loss": 2.7466,
+ "theoretical_loss": 5.291243889978516,
+ "tokens_seen": 50987008
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007955867602808426,
+ "loss": 2.7354,
+ "theoretical_loss": 5.290217629956882,
+ "tokens_seen": 51052544
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007955707121364093,
+ "loss": 2.8293,
+ "theoretical_loss": 5.289193054832282,
+ "tokens_seen": 51118080
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000795554663991976,
+ "loss": 2.7158,
+ "theoretical_loss": 5.288170159683668,
+ "tokens_seen": 51183616
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007955386158475427,
+ "loss": 2.6579,
+ "theoretical_loss": 5.287148939610645,
+ "tokens_seen": 51249152
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007955225677031094,
+ "loss": 2.5181,
+ "theoretical_loss": 5.28612938973335,
+ "tokens_seen": 51314688
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007955065195586761,
+ "loss": 2.7695,
+ "theoretical_loss": 5.285111505192349,
+ "tokens_seen": 51380224
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954904714142428,
+ "loss": 2.6868,
+ "theoretical_loss": 5.284095281148522,
+ "tokens_seen": 51445760
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954744232698095,
+ "loss": 2.747,
+ "theoretical_loss": 5.283080712782951,
+ "tokens_seen": 51511296
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954583751253762,
+ "loss": 2.6446,
+ "theoretical_loss": 5.282067795296811,
+ "tokens_seen": 51576832
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954423269809429,
+ "loss": 2.6414,
+ "theoretical_loss": 5.28105652391126,
+ "tokens_seen": 51642368
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954262788365096,
+ "loss": 2.6543,
+ "theoretical_loss": 5.280046893867333,
+ "tokens_seen": 51707904
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007954102306920763,
+ "loss": 2.5674,
+ "theoretical_loss": 5.279038900425832,
+ "tokens_seen": 51773440
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000795394182547643,
+ "loss": 2.6305,
+ "theoretical_loss": 5.278032538867221,
+ "tokens_seen": 51838976
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007953781344032097,
+ "loss": 2.5573,
+ "theoretical_loss": 5.277027804491517,
+ "tokens_seen": 51904512
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007953620862587764,
+ "loss": 2.6,
+ "theoretical_loss": 5.2760246926181855,
+ "tokens_seen": 51970048
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007953460381143431,
+ "loss": 2.6884,
+ "theoretical_loss": 5.275023198586037,
+ "tokens_seen": 52035584
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007953299899699098,
+ "loss": 2.6203,
+ "theoretical_loss": 5.274023317753125,
+ "tokens_seen": 52101120
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007953139418254765,
+ "loss": 2.6784,
+ "theoretical_loss": 5.273025045496631,
+ "tokens_seen": 52166656
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952978936810432,
+ "loss": 2.5281,
+ "theoretical_loss": 5.272028377212777,
+ "tokens_seen": 52232192
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952818455366099,
+ "loss": 2.5807,
+ "theoretical_loss": 5.271033308316715,
+ "tokens_seen": 52297728
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952657973921766,
+ "loss": 2.6348,
+ "theoretical_loss": 5.270039834242425,
+ "tokens_seen": 52363264
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 24243,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.5289902687072754,
+ "objective/train/theoretical_loss": 5.269047950442619,
+ "objective/train/tokens_used": 72888800,
+ "theoretical_loss": 5.269047950442619,
+ "tokens_seen": 52428800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952497492477433,
+ "loss": 2.485,
+ "theoretical_loss": 5.269047950442619,
+ "tokens_seen": 52428800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.00079523370110331,
+ "loss": 2.6376,
+ "theoretical_loss": 5.2680576523886335,
+ "tokens_seen": 52494336
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952176529588767,
+ "loss": 2.7194,
+ "theoretical_loss": 5.2670689355703395,
+ "tokens_seen": 52559872
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007952016048144434,
+ "loss": 2.6341,
+ "theoretical_loss": 5.266081795496035,
+ "tokens_seen": 52625408
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951855566700101,
+ "loss": 2.5662,
+ "theoretical_loss": 5.265096227692354,
+ "tokens_seen": 52690944
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951695085255768,
+ "loss": 2.6624,
+ "theoretical_loss": 5.264112227704162,
+ "tokens_seen": 52756480
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951534603811434,
+ "loss": 2.7557,
+ "theoretical_loss": 5.263129791094466,
+ "tokens_seen": 52822016
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951374122367102,
+ "loss": 2.6062,
+ "theoretical_loss": 5.26214891344431,
+ "tokens_seen": 52887552
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951213640922768,
+ "loss": 2.5699,
+ "theoretical_loss": 5.26116959035269,
+ "tokens_seen": 52953088
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007951053159478436,
+ "loss": 2.506,
+ "theoretical_loss": 5.260191817436452,
+ "tokens_seen": 53018624
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007950892678034102,
+ "loss": 2.6012,
+ "theoretical_loss": 5.259215590330195,
+ "tokens_seen": 53084160
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000795073219658977,
+ "loss": 2.5899,
+ "theoretical_loss": 5.258240904686186,
+ "tokens_seen": 53149696
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007950571715145436,
+ "loss": 2.5064,
+ "theoretical_loss": 5.25726775617426,
+ "tokens_seen": 53215232
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007950411233701104,
+ "loss": 2.4435,
+ "theoretical_loss": 5.256296140481728,
+ "tokens_seen": 53280768
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000795025075225677,
+ "loss": 2.5262,
+ "theoretical_loss": 5.255326053313292,
+ "tokens_seen": 53346304
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007950090270812438,
+ "loss": 2.5646,
+ "theoretical_loss": 5.254357490390941,
+ "tokens_seen": 53411840
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007949929789368104,
+ "loss": 2.4133,
+ "theoretical_loss": 5.253390447453873,
+ "tokens_seen": 53477376
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007949769307923772,
+ "loss": 2.6187,
+ "theoretical_loss": 5.252424920258396,
+ "tokens_seen": 53542912
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007949608826479438,
+ "loss": 2.5912,
+ "theoretical_loss": 5.251460904577845,
+ "tokens_seen": 53608448
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007949448345035106,
+ "loss": 2.5508,
+ "theoretical_loss": 5.250498396202486,
+ "tokens_seen": 53673984
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007949287863590772,
+ "loss": 2.5112,
+ "theoretical_loss": 5.249537390939436,
+ "tokens_seen": 53739520
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794912738214644,
+ "loss": 2.4452,
+ "theoretical_loss": 5.248577884612564,
+ "tokens_seen": 53805056
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948966900702106,
+ "loss": 2.4712,
+ "theoretical_loss": 5.247619873062416,
+ "tokens_seen": 53870592
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948806419257774,
+ "loss": 2.5985,
+ "theoretical_loss": 5.246663352146122,
+ "tokens_seen": 53936128
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794864593781344,
+ "loss": 2.6442,
+ "theoretical_loss": 5.245708317737305,
+ "tokens_seen": 54001664
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948485456369108,
+ "loss": 2.4876,
+ "theoretical_loss": 5.244754765726007,
+ "tokens_seen": 54067200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948324974924774,
+ "loss": 2.5376,
+ "theoretical_loss": 5.243802692018596,
+ "tokens_seen": 54132736
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948164493480442,
+ "loss": 2.4553,
+ "theoretical_loss": 5.242852092537677,
+ "tokens_seen": 54198272
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007948004012036108,
+ "loss": 2.6929,
+ "theoretical_loss": 5.241902963222023,
+ "tokens_seen": 54263808
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007947843530591776,
+ "loss": 2.5035,
+ "theoretical_loss": 5.240955300026476,
+ "tokens_seen": 54329344
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007947683049147442,
+ "loss": 2.5704,
+ "theoretical_loss": 5.240009098921874,
+ "tokens_seen": 54394880
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794752256770311,
+ "loss": 2.4209,
+ "theoretical_loss": 5.239064355894965,
+ "tokens_seen": 54460416
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007947362086258776,
+ "loss": 2.5877,
+ "theoretical_loss": 5.238121066948324,
+ "tokens_seen": 54525952
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007947201604814444,
+ "loss": 2.4356,
+ "theoretical_loss": 5.237179228100272,
+ "tokens_seen": 54591488
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794704112337011,
+ "loss": 2.498,
+ "theoretical_loss": 5.2362388353848,
+ "tokens_seen": 54657024
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946880641925778,
+ "loss": 2.426,
+ "theoretical_loss": 5.23529988485148,
+ "tokens_seen": 54722560
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946720160481444,
+ "loss": 2.527,
+ "theoretical_loss": 5.2343623725653945,
+ "tokens_seen": 54788096
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946559679037112,
+ "loss": 2.5915,
+ "theoretical_loss": 5.233426294607049,
+ "tokens_seen": 54853632
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946399197592778,
+ "loss": 2.5706,
+ "theoretical_loss": 5.232491647072299,
+ "tokens_seen": 54919168
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946238716148446,
+ "loss": 2.5656,
+ "theoretical_loss": 5.231558426072269,
+ "tokens_seen": 54984704
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007946078234704112,
+ "loss": 2.4501,
+ "theoretical_loss": 5.230626627733274,
+ "tokens_seen": 55050240
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794591775325978,
+ "loss": 2.5487,
+ "theoretical_loss": 5.2296962481967455,
+ "tokens_seen": 55115776
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007945757271815446,
+ "loss": 2.3679,
+ "theoretical_loss": 5.228767283619151,
+ "tokens_seen": 55181312
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007945596790371114,
+ "loss": 2.6326,
+ "theoretical_loss": 5.227839730171918,
+ "tokens_seen": 55246848
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794543630892678,
+ "loss": 2.4476,
+ "theoretical_loss": 5.2269135840413625,
+ "tokens_seen": 55312384
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007945275827482448,
+ "loss": 2.309,
+ "theoretical_loss": 5.225988841428606,
+ "tokens_seen": 55377920
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007945115346038114,
+ "loss": 2.2936,
+ "theoretical_loss": 5.225065498549508,
+ "tokens_seen": 55443456
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007944954864593782,
+ "loss": 2.4656,
+ "theoretical_loss": 5.224143551634588,
+ "tokens_seen": 55508992
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007944794383149448,
+ "loss": 2.428,
+ "theoretical_loss": 5.22322299692895,
+ "tokens_seen": 55574528
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007944633901705116,
+ "loss": 2.4442,
+ "theoretical_loss": 5.222303830692211,
+ "tokens_seen": 55640064
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 25141,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.6088738441467285,
+ "objective/train/theoretical_loss": 5.22138604919843,
+ "objective/train/tokens_used": 76165600,
+ "theoretical_loss": 5.22138604919843,
+ "tokens_seen": 55705600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007944473420260782,
+ "loss": 2.5252,
+ "theoretical_loss": 5.22138604919843,
+ "tokens_seen": 55705600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794431293881645,
+ "loss": 2.6352,
+ "theoretical_loss": 5.220469648736031,
+ "tokens_seen": 55771136
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007944152457372116,
+ "loss": 2.4382,
+ "theoretical_loss": 5.219554625607731,
+ "tokens_seen": 55836672
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943991975927784,
+ "loss": 2.5073,
+ "theoretical_loss": 5.218640976130474,
+ "tokens_seen": 55902208
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794383149448345,
+ "loss": 2.502,
+ "theoretical_loss": 5.217728696635353,
+ "tokens_seen": 55967744
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943671013039118,
+ "loss": 2.3945,
+ "theoretical_loss": 5.216817783467543,
+ "tokens_seen": 56033280
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943510531594784,
+ "loss": 2.6394,
+ "theoretical_loss": 5.215908232986227,
+ "tokens_seen": 56098816
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943350050150452,
+ "loss": 2.4114,
+ "theoretical_loss": 5.215000041564533,
+ "tokens_seen": 56164352
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943189568706118,
+ "loss": 2.47,
+ "theoretical_loss": 5.214093205589455,
+ "tokens_seen": 56229888
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007943029087261786,
+ "loss": 2.4864,
+ "theoretical_loss": 5.213187721461791,
+ "tokens_seen": 56295424
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007942868605817452,
+ "loss": 2.5074,
+ "theoretical_loss": 5.212283585596071,
+ "tokens_seen": 56360960
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794270812437312,
+ "loss": 2.3196,
+ "theoretical_loss": 5.211380794420492,
+ "tokens_seen": 56426496
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007942547642928786,
+ "loss": 2.3877,
+ "theoretical_loss": 5.210479344376845,
+ "tokens_seen": 56492032
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007942387161484454,
+ "loss": 2.5349,
+ "theoretical_loss": 5.209579231920451,
+ "tokens_seen": 56557568
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794222668004012,
+ "loss": 2.4561,
+ "theoretical_loss": 5.208680453520092,
+ "tokens_seen": 56623104
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007942066198595788,
+ "loss": 2.4743,
+ "theoretical_loss": 5.2077830056579515,
+ "tokens_seen": 56688640
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007941905717151455,
+ "loss": 2.5124,
+ "theoretical_loss": 5.206886884829535,
+ "tokens_seen": 56754176
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007941745235707122,
+ "loss": 2.3724,
+ "theoretical_loss": 5.205992087543614,
+ "tokens_seen": 56819712
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007941584754262789,
+ "loss": 2.5328,
+ "theoretical_loss": 5.205098610322162,
+ "tokens_seen": 56885248
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007941424272818456,
+ "loss": 2.6109,
+ "theoretical_loss": 5.204206449700279,
+ "tokens_seen": 56950784
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007941263791374123,
+ "loss": 2.4425,
+ "theoretical_loss": 5.203315602226139,
+ "tokens_seen": 57016320
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000794110330992979,
+ "loss": 2.353,
+ "theoretical_loss": 5.2024260644609175,
+ "tokens_seen": 57081856
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940942828485457,
+ "loss": 2.4324,
+ "theoretical_loss": 5.201537832978731,
+ "tokens_seen": 57147392
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940782347041124,
+ "loss": 2.3614,
+ "theoretical_loss": 5.200650904366574,
+ "tokens_seen": 57212928
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940621865596791,
+ "loss": 2.4856,
+ "theoretical_loss": 5.199765275224252,
+ "tokens_seen": 57278464
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940461384152458,
+ "loss": 2.4181,
+ "theoretical_loss": 5.198880942164323,
+ "tokens_seen": 57344000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940300902708125,
+ "loss": 2.351,
+ "theoretical_loss": 5.197997901812036,
+ "tokens_seen": 57409536
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007940140421263792,
+ "loss": 2.5241,
+ "theoretical_loss": 5.197116150805264,
+ "tokens_seen": 57475072
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939979939819459,
+ "loss": 2.3707,
+ "theoretical_loss": 5.196235685794446,
+ "tokens_seen": 57540608
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939819458375126,
+ "loss": 2.4242,
+ "theoretical_loss": 5.195356503442525,
+ "tokens_seen": 57606144
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939658976930794,
+ "loss": 2.4264,
+ "theoretical_loss": 5.194478600424889,
+ "tokens_seen": 57671680
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000793949849548646,
+ "loss": 2.4905,
+ "theoretical_loss": 5.193601973429306,
+ "tokens_seen": 57737216
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939338014042128,
+ "loss": 2.3264,
+ "theoretical_loss": 5.192726619155867,
+ "tokens_seen": 57802752
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939177532597794,
+ "loss": 2.2947,
+ "theoretical_loss": 5.191852534316928,
+ "tokens_seen": 57868288
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007939017051153462,
+ "loss": 2.5367,
+ "theoretical_loss": 5.190979715637049,
+ "tokens_seen": 57933824
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938856569709128,
+ "loss": 2.4345,
+ "theoretical_loss": 5.190108159852929,
+ "tokens_seen": 57999360
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938696088264795,
+ "loss": 2.48,
+ "theoretical_loss": 5.189237863713357,
+ "tokens_seen": 58064896
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938535606820462,
+ "loss": 2.5036,
+ "theoretical_loss": 5.188368823979152,
+ "tokens_seen": 58130432
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938375125376129,
+ "loss": 2.2739,
+ "theoretical_loss": 5.187501037423096,
+ "tokens_seen": 58195968
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938214643931796,
+ "loss": 2.3866,
+ "theoretical_loss": 5.186634500829886,
+ "tokens_seen": 58261504
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007938054162487463,
+ "loss": 2.4664,
+ "theoretical_loss": 5.185769210996076,
+ "tokens_seen": 58327040
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000793789368104313,
+ "loss": 2.2739,
+ "theoretical_loss": 5.184905164730012,
+ "tokens_seen": 58392576
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007937733199598797,
+ "loss": 2.4244,
+ "theoretical_loss": 5.184042358851785,
+ "tokens_seen": 58458112
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007937572718154463,
+ "loss": 2.3718,
+ "theoretical_loss": 5.1831807901931715,
+ "tokens_seen": 58523648
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007937412236710131,
+ "loss": 2.2701,
+ "theoretical_loss": 5.18232045559757,
+ "tokens_seen": 58589184
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007937251755265797,
+ "loss": 2.3716,
+ "theoretical_loss": 5.181461351919959,
+ "tokens_seen": 58654720
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007937091273821465,
+ "loss": 2.5091,
+ "theoretical_loss": 5.180603476026832,
+ "tokens_seen": 58720256
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936930792377131,
+ "loss": 2.3163,
+ "theoretical_loss": 5.179746824796146,
+ "tokens_seen": 58785792
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936770310932799,
+ "loss": 2.2918,
+ "theoretical_loss": 5.178891395117262,
+ "tokens_seen": 58851328
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936609829488465,
+ "loss": 2.4097,
+ "theoretical_loss": 5.178037183890902,
+ "tokens_seen": 58916864
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 26367,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.4226865768432617,
+ "objective/train/theoretical_loss": 5.177184188029082,
+ "objective/train/tokens_used": 79442400,
+ "theoretical_loss": 5.177184188029082,
+ "tokens_seen": 58982400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936449348044133,
+ "loss": 2.3572,
+ "theoretical_loss": 5.177184188029082,
+ "tokens_seen": 58982400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936288866599799,
+ "loss": 2.452,
+ "theoretical_loss": 5.1763324044550645,
+ "tokens_seen": 59047936
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007936128385155467,
+ "loss": 2.3822,
+ "theoretical_loss": 5.175481830103307,
+ "tokens_seen": 59113472
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935967903711133,
+ "loss": 2.376,
+ "theoretical_loss": 5.174632461919406,
+ "tokens_seen": 59179008
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935807422266801,
+ "loss": 2.3096,
+ "theoretical_loss": 5.173784296860043,
+ "tokens_seen": 59244544
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935646940822467,
+ "loss": 2.4012,
+ "theoretical_loss": 5.172937331892934,
+ "tokens_seen": 59310080
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935486459378135,
+ "loss": 2.2779,
+ "theoretical_loss": 5.17209156399678,
+ "tokens_seen": 59375616
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935325977933801,
+ "loss": 2.4276,
+ "theoretical_loss": 5.171246990161208,
+ "tokens_seen": 59441152
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935165496489469,
+ "loss": 2.4266,
+ "theoretical_loss": 5.17040360738673,
+ "tokens_seen": 59506688
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007935005015045135,
+ "loss": 2.2993,
+ "theoretical_loss": 5.169561412684677,
+ "tokens_seen": 59572224
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934844533600803,
+ "loss": 2.4694,
+ "theoretical_loss": 5.168720403077165,
+ "tokens_seen": 59637760
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934684052156469,
+ "loss": 2.2188,
+ "theoretical_loss": 5.167880575597031,
+ "tokens_seen": 59703296
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934523570712137,
+ "loss": 2.2279,
+ "theoretical_loss": 5.167041927287787,
+ "tokens_seen": 59768832
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934363089267803,
+ "loss": 2.4385,
+ "theoretical_loss": 5.166204455203575,
+ "tokens_seen": 59834368
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934202607823471,
+ "loss": 2.2816,
+ "theoretical_loss": 5.16536815640911,
+ "tokens_seen": 59899904
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007934042126379137,
+ "loss": 2.4746,
+ "theoretical_loss": 5.1645330279796315,
+ "tokens_seen": 59965440
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933881644934805,
+ "loss": 2.3473,
+ "theoretical_loss": 5.1636990670008585,
+ "tokens_seen": 60030976
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933721163490471,
+ "loss": 2.4463,
+ "theoretical_loss": 5.162866270568938,
+ "tokens_seen": 60096512
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933560682046139,
+ "loss": 2.3849,
+ "theoretical_loss": 5.162034635790397,
+ "tokens_seen": 60162048
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933400200601805,
+ "loss": 2.384,
+ "theoretical_loss": 5.1612041597820895,
+ "tokens_seen": 60227584
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933239719157473,
+ "loss": 2.4535,
+ "theoretical_loss": 5.160374839671159,
+ "tokens_seen": 60293120
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007933079237713139,
+ "loss": 2.3911,
+ "theoretical_loss": 5.1595466725949795,
+ "tokens_seen": 60358656
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932918756268807,
+ "loss": 2.3502,
+ "theoretical_loss": 5.158719655701115,
+ "tokens_seen": 60424192
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932758274824473,
+ "loss": 2.4282,
+ "theoretical_loss": 5.157893786147269,
+ "tokens_seen": 60489728
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932597793380141,
+ "loss": 2.3445,
+ "theoretical_loss": 5.157069061101238,
+ "tokens_seen": 60555264
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932437311935807,
+ "loss": 2.3545,
+ "theoretical_loss": 5.156245477740866,
+ "tokens_seen": 60620800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932276830491475,
+ "loss": 2.2794,
+ "theoretical_loss": 5.155423033253997,
+ "tokens_seen": 60686336
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007932116349047141,
+ "loss": 2.2316,
+ "theoretical_loss": 5.154601724838431,
+ "tokens_seen": 60751872
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931955867602809,
+ "loss": 2.3536,
+ "theoretical_loss": 5.153781549701872,
+ "tokens_seen": 60817408
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931795386158475,
+ "loss": 2.4524,
+ "theoretical_loss": 5.15296250506189,
+ "tokens_seen": 60882944
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931634904714143,
+ "loss": 2.3219,
+ "theoretical_loss": 5.152144588145871,
+ "tokens_seen": 60948480
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931474423269809,
+ "loss": 2.2977,
+ "theoretical_loss": 5.151327796190975,
+ "tokens_seen": 61014016
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931313941825477,
+ "loss": 2.2728,
+ "theoretical_loss": 5.1505121264440845,
+ "tokens_seen": 61079552
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007931153460381143,
+ "loss": 2.4291,
+ "theoretical_loss": 5.149697576161769,
+ "tokens_seen": 61145088
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930992978936811,
+ "loss": 2.258,
+ "theoretical_loss": 5.148884142610232,
+ "tokens_seen": 61210624
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930832497492477,
+ "loss": 2.3553,
+ "theoretical_loss": 5.148071823065274,
+ "tokens_seen": 61276160
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930672016048145,
+ "loss": 2.3682,
+ "theoretical_loss": 5.147260614812246,
+ "tokens_seen": 61341696
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930511534603811,
+ "loss": 2.3699,
+ "theoretical_loss": 5.146450515146002,
+ "tokens_seen": 61407232
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930351053159479,
+ "loss": 2.2222,
+ "theoretical_loss": 5.145641521370861,
+ "tokens_seen": 61472768
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930190571715145,
+ "loss": 2.3013,
+ "theoretical_loss": 5.144833630800562,
+ "tokens_seen": 61538304
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007930030090270813,
+ "loss": 2.3248,
+ "theoretical_loss": 5.1440268407582215,
+ "tokens_seen": 61603840
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929869608826479,
+ "loss": 2.3713,
+ "theoretical_loss": 5.143221148576288,
+ "tokens_seen": 61669376
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929709127382147,
+ "loss": 2.1181,
+ "theoretical_loss": 5.142416551596505,
+ "tokens_seen": 61734912
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929548645937813,
+ "loss": 2.1712,
+ "theoretical_loss": 5.141613047169862,
+ "tokens_seen": 61800448
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929388164493481,
+ "loss": 2.3425,
+ "theoretical_loss": 5.14081063265656,
+ "tokens_seen": 61865984
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929227683049147,
+ "loss": 2.2708,
+ "theoretical_loss": 5.140009305425963,
+ "tokens_seen": 61931520
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007929067201604815,
+ "loss": 2.205,
+ "theoretical_loss": 5.13920906285656,
+ "tokens_seen": 61997056
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007928906720160481,
+ "loss": 2.3632,
+ "theoretical_loss": 5.138409902335925,
+ "tokens_seen": 62062592
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007928746238716149,
+ "loss": 2.2693,
+ "theoretical_loss": 5.137611821260673,
+ "tokens_seen": 62128128
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007928585757271815,
+ "loss": 2.2251,
+ "theoretical_loss": 5.13681481703642,
+ "tokens_seen": 62193664
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 27626,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.3975930213928223,
+ "objective/train/theoretical_loss": 5.136018887077743,
+ "objective/train/tokens_used": 82719200,
+ "theoretical_loss": 5.136018887077743,
+ "tokens_seen": 62259200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007928425275827483,
+ "loss": 2.3101,
+ "theoretical_loss": 5.136018887077743,
+ "tokens_seen": 62259200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000792826479438315,
+ "loss": 2.446,
+ "theoretical_loss": 5.13522402880814,
+ "tokens_seen": 62324736
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007928104312938817,
+ "loss": 2.4389,
+ "theoretical_loss": 5.13443023965999,
+ "tokens_seen": 62390272
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927943831494484,
+ "loss": 2.3297,
+ "theoretical_loss": 5.13363751707451,
+ "tokens_seen": 62455808
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927783350050151,
+ "loss": 2.2738,
+ "theoretical_loss": 5.132845858501721,
+ "tokens_seen": 62521344
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927622868605819,
+ "loss": 2.2318,
+ "theoretical_loss": 5.132055261400403,
+ "tokens_seen": 62586880
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927462387161485,
+ "loss": 2.193,
+ "theoretical_loss": 5.131265723238059,
+ "tokens_seen": 62652416
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927301905717153,
+ "loss": 2.2944,
+ "theoretical_loss": 5.130477241490876,
+ "tokens_seen": 62717952
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007927141424272819,
+ "loss": 2.2733,
+ "theoretical_loss": 5.129689813643683,
+ "tokens_seen": 62783488
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926980942828487,
+ "loss": 2.2248,
+ "theoretical_loss": 5.128903437189917,
+ "tokens_seen": 62849024
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926820461384153,
+ "loss": 2.2887,
+ "theoretical_loss": 5.128118109631581,
+ "tokens_seen": 62914560
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926659979939821,
+ "loss": 2.31,
+ "theoretical_loss": 5.127333828479208,
+ "tokens_seen": 62980096
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926499498495487,
+ "loss": 2.2063,
+ "theoretical_loss": 5.126550591251821,
+ "tokens_seen": 63045632
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926339017051155,
+ "loss": 2.3662,
+ "theoretical_loss": 5.125768395476896,
+ "tokens_seen": 63111168
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926178535606821,
+ "loss": 2.2167,
+ "theoretical_loss": 5.124987238690329,
+ "tokens_seen": 63176704
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007926018054162489,
+ "loss": 2.3044,
+ "theoretical_loss": 5.124207118436388,
+ "tokens_seen": 63242240
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925857572718155,
+ "loss": 2.3506,
+ "theoretical_loss": 5.123428032267688,
+ "tokens_seen": 63307776
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925697091273823,
+ "loss": 2.2806,
+ "theoretical_loss": 5.122649977745146,
+ "tokens_seen": 63373312
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925536609829489,
+ "loss": 2.3118,
+ "theoretical_loss": 5.121872952437947,
+ "tokens_seen": 63438848
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925376128385157,
+ "loss": 2.3615,
+ "theoretical_loss": 5.121096953923507,
+ "tokens_seen": 63504384
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925215646940823,
+ "loss": 2.3037,
+ "theoretical_loss": 5.1203219797874375,
+ "tokens_seen": 63569920
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007925055165496491,
+ "loss": 2.3262,
+ "theoretical_loss": 5.119548027623509,
+ "tokens_seen": 63635456
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924894684052157,
+ "loss": 2.3319,
+ "theoretical_loss": 5.118775095033612,
+ "tokens_seen": 63700992
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924734202607824,
+ "loss": 2.2909,
+ "theoretical_loss": 5.118003179627729,
+ "tokens_seen": 63766528
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924573721163491,
+ "loss": 2.24,
+ "theoretical_loss": 5.11723227902389,
+ "tokens_seen": 63832064
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924413239719158,
+ "loss": 2.3249,
+ "theoretical_loss": 5.1164623908481435,
+ "tokens_seen": 63897600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924252758274825,
+ "loss": 2.2101,
+ "theoretical_loss": 5.115693512734515,
+ "tokens_seen": 63963136
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007924092276830492,
+ "loss": 2.1202,
+ "theoretical_loss": 5.114925642324982,
+ "tokens_seen": 64028672
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007923931795386159,
+ "loss": 2.267,
+ "theoretical_loss": 5.114158777269429,
+ "tokens_seen": 64094208
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007923771313941826,
+ "loss": 2.2321,
+ "theoretical_loss": 5.113392915225617,
+ "tokens_seen": 64159744
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007923610832497493,
+ "loss": 2.35,
+ "theoretical_loss": 5.112628053859153,
+ "tokens_seen": 64225280
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000792345035105316,
+ "loss": 2.0774,
+ "theoretical_loss": 5.1118641908434475,
+ "tokens_seen": 64290816
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007923289869608826,
+ "loss": 2.2412,
+ "theoretical_loss": 5.111101323859687,
+ "tokens_seen": 64356352
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007923129388164494,
+ "loss": 2.3277,
+ "theoretical_loss": 5.1103394505968005,
+ "tokens_seen": 64421888
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000792296890672016,
+ "loss": 2.315,
+ "theoretical_loss": 5.109578568751417,
+ "tokens_seen": 64487424
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922808425275828,
+ "loss": 2.413,
+ "theoretical_loss": 5.108818676027845,
+ "tokens_seen": 64552960
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922647943831494,
+ "loss": 2.1771,
+ "theoretical_loss": 5.108059770138027,
+ "tokens_seen": 64618496
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922487462387162,
+ "loss": 2.3026,
+ "theoretical_loss": 5.107301848801519,
+ "tokens_seen": 64684032
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922326980942828,
+ "loss": 2.253,
+ "theoretical_loss": 5.106544909745443,
+ "tokens_seen": 64749568
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922166499498496,
+ "loss": 2.1783,
+ "theoretical_loss": 5.105788950704467,
+ "tokens_seen": 64815104
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007922006018054162,
+ "loss": 2.2324,
+ "theoretical_loss": 5.105033969420764,
+ "tokens_seen": 64880640
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000792184553660983,
+ "loss": 2.2459,
+ "theoretical_loss": 5.104279963643982,
+ "tokens_seen": 64946176
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007921685055165496,
+ "loss": 2.1426,
+ "theoretical_loss": 5.103526931131217,
+ "tokens_seen": 65011712
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007921524573721164,
+ "loss": 2.2505,
+ "theoretical_loss": 5.102774869646972,
+ "tokens_seen": 65077248
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000792136409227683,
+ "loss": 2.0972,
+ "theoretical_loss": 5.102023776963131,
+ "tokens_seen": 65142784
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007921203610832498,
+ "loss": 2.1683,
+ "theoretical_loss": 5.101273650858926,
+ "tokens_seen": 65208320
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007921043129388164,
+ "loss": 2.2065,
+ "theoretical_loss": 5.100524489120902,
+ "tokens_seen": 65273856
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007920882647943832,
+ "loss": 2.2527,
+ "theoretical_loss": 5.099776289542893,
+ "tokens_seen": 65339392
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007920722166499498,
+ "loss": 2.1725,
+ "theoretical_loss": 5.099029049925985,
+ "tokens_seen": 65404928
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007920561685055166,
+ "loss": 2.2228,
+ "theoretical_loss": 5.098282768078483,
+ "tokens_seen": 65470464
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 28581,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.2630820274353027,
+ "objective/train/theoretical_loss": 5.097537441815887,
+ "objective/train/tokens_used": 85996000,
+ "theoretical_loss": 5.097537441815887,
+ "tokens_seen": 65536000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007920401203610832,
+ "loss": 2.244,
+ "theoretical_loss": 5.097537441815887,
+ "tokens_seen": 65536000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.00079202407221665,
+ "loss": 2.2649,
+ "theoretical_loss": 5.096793068960857,
+ "tokens_seen": 65601536
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007920080240722166,
+ "loss": 2.2591,
+ "theoretical_loss": 5.096049647343182,
+ "tokens_seen": 65667072
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007919919759277834,
+ "loss": 2.256,
+ "theoretical_loss": 5.095307174799752,
+ "tokens_seen": 65732608
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.00079197592778335,
+ "loss": 2.2502,
+ "theoretical_loss": 5.094565649174524,
+ "tokens_seen": 65798144
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007919598796389168,
+ "loss": 2.1983,
+ "theoretical_loss": 5.0938250683184965,
+ "tokens_seen": 65863680
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007919438314944834,
+ "loss": 2.261,
+ "theoretical_loss": 5.093085430089674,
+ "tokens_seen": 65929216
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007919277833500502,
+ "loss": 2.3129,
+ "theoretical_loss": 5.092346732353046,
+ "tokens_seen": 65994752
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007919117352056168,
+ "loss": 2.2523,
+ "theoretical_loss": 5.091608972980545,
+ "tokens_seen": 66060288
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007918956870611836,
+ "loss": 2.2646,
+ "theoretical_loss": 5.090872149851029,
+ "tokens_seen": 66125824
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007918796389167502,
+ "loss": 2.2916,
+ "theoretical_loss": 5.090136260850245,
+ "tokens_seen": 66191360
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791863590772317,
+ "loss": 2.1602,
+ "theoretical_loss": 5.089401303870801,
+ "tokens_seen": 66256896
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007918475426278836,
+ "loss": 2.2429,
+ "theoretical_loss": 5.088667276812138,
+ "tokens_seen": 66322432
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007918314944834504,
+ "loss": 2.1725,
+ "theoretical_loss": 5.087934177580502,
+ "tokens_seen": 66387968
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791815446339017,
+ "loss": 2.2774,
+ "theoretical_loss": 5.087202004088914,
+ "tokens_seen": 66453504
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917993981945838,
+ "loss": 2.1667,
+ "theoretical_loss": 5.086470754257139,
+ "tokens_seen": 66519040
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917833500501504,
+ "loss": 2.2032,
+ "theoretical_loss": 5.0857404260116645,
+ "tokens_seen": 66584576
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917673019057172,
+ "loss": 2.148,
+ "theoretical_loss": 5.085011017285662,
+ "tokens_seen": 66650112
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917512537612838,
+ "loss": 2.2979,
+ "theoretical_loss": 5.084282526018972,
+ "tokens_seen": 66715648
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917352056168506,
+ "loss": 2.2236,
+ "theoretical_loss": 5.083554950158062,
+ "tokens_seen": 66781184
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007917191574724172,
+ "loss": 2.1649,
+ "theoretical_loss": 5.0828282876560085,
+ "tokens_seen": 66846720
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791703109327984,
+ "loss": 2.2443,
+ "theoretical_loss": 5.0821025364724655,
+ "tokens_seen": 66912256
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007916870611835506,
+ "loss": 2.3334,
+ "theoretical_loss": 5.081377694573638,
+ "tokens_seen": 66977792
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007916710130391174,
+ "loss": 2.2181,
+ "theoretical_loss": 5.080653759932255,
+ "tokens_seen": 67043328
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791654964894684,
+ "loss": 2.1852,
+ "theoretical_loss": 5.079930730527538,
+ "tokens_seen": 67108864
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007916389167502508,
+ "loss": 2.1556,
+ "theoretical_loss": 5.079208604345181,
+ "tokens_seen": 67174400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007916228686058174,
+ "loss": 2.2345,
+ "theoretical_loss": 5.078487379377318,
+ "tokens_seen": 67239936
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007916068204613842,
+ "loss": 2.1317,
+ "theoretical_loss": 5.0777670536225,
+ "tokens_seen": 67305472
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791590772316951,
+ "loss": 2.1145,
+ "theoretical_loss": 5.07704762508566,
+ "tokens_seen": 67371008
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007915747241725176,
+ "loss": 2.1057,
+ "theoretical_loss": 5.076329091778101,
+ "tokens_seen": 67436544
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007915586760280844,
+ "loss": 2.1766,
+ "theoretical_loss": 5.075611451717457,
+ "tokens_seen": 67502080
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791542627883651,
+ "loss": 2.1238,
+ "theoretical_loss": 5.0748947029276685,
+ "tokens_seen": 67567616
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007915265797392178,
+ "loss": 2.2196,
+ "theoretical_loss": 5.074178843438963,
+ "tokens_seen": 67633152
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007915105315947844,
+ "loss": 2.2516,
+ "theoretical_loss": 5.073463871287823,
+ "tokens_seen": 67698688
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007914944834503512,
+ "loss": 2.2305,
+ "theoretical_loss": 5.07274978451696,
+ "tokens_seen": 67764224
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007914784353059178,
+ "loss": 2.2993,
+ "theoretical_loss": 5.072036581175293,
+ "tokens_seen": 67829760
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007914623871614846,
+ "loss": 2.2447,
+ "theoretical_loss": 5.071324259317921,
+ "tokens_seen": 67895296
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007914463390170512,
+ "loss": 2.1236,
+ "theoretical_loss": 5.070612817006094,
+ "tokens_seen": 67960832
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791430290872618,
+ "loss": 2.3058,
+ "theoretical_loss": 5.06990225230719,
+ "tokens_seen": 68026368
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007914142427281846,
+ "loss": 2.1259,
+ "theoretical_loss": 5.069192563294694,
+ "tokens_seen": 68091904
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913981945837514,
+ "loss": 2.3627,
+ "theoretical_loss": 5.068483748048166,
+ "tokens_seen": 68157440
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791382146439318,
+ "loss": 2.3496,
+ "theoretical_loss": 5.06777580465322,
+ "tokens_seen": 68222976
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913660982948848,
+ "loss": 2.315,
+ "theoretical_loss": 5.067068731201498,
+ "tokens_seen": 68288512
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913500501504514,
+ "loss": 2.1493,
+ "theoretical_loss": 5.066362525790646,
+ "tokens_seen": 68354048
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913340020060182,
+ "loss": 2.113,
+ "theoretical_loss": 5.065657186524287,
+ "tokens_seen": 68419584
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913179538615848,
+ "loss": 2.2397,
+ "theoretical_loss": 5.064952711512001,
+ "tokens_seen": 68485120
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007913019057171516,
+ "loss": 2.2762,
+ "theoretical_loss": 5.064249098869295,
+ "tokens_seen": 68550656
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007912858575727182,
+ "loss": 2.2793,
+ "theoretical_loss": 5.063546346717585,
+ "tokens_seen": 68616192
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791269809428285,
+ "loss": 2.2232,
+ "theoretical_loss": 5.062844453184166,
+ "tokens_seen": 68681728
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007912537612838516,
+ "loss": 2.2233,
+ "theoretical_loss": 5.062143416402192,
+ "tokens_seen": 68747264
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 29892,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.302198648452759,
+ "objective/train/theoretical_loss": 5.061443234510648,
+ "objective/train/tokens_used": 89272800,
+ "theoretical_loss": 5.061443234510648,
+ "tokens_seen": 68812800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007912377131394184,
+ "loss": 2.2377,
+ "theoretical_loss": 5.061443234510648,
+ "tokens_seen": 68812800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791221664994985,
+ "loss": 2.2414,
+ "theoretical_loss": 5.060743905654332,
+ "tokens_seen": 68878336
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007912056168505518,
+ "loss": 2.2892,
+ "theoretical_loss": 5.0600454279838285,
+ "tokens_seen": 68943872
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007911895687061184,
+ "loss": 2.1666,
+ "theoretical_loss": 5.059347799655483,
+ "tokens_seen": 69009408
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007911735205616852,
+ "loss": 2.1416,
+ "theoretical_loss": 5.05865101883138,
+ "tokens_seen": 69074944
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007911574724172518,
+ "loss": 2.1452,
+ "theoretical_loss": 5.057955083679321,
+ "tokens_seen": 69140480
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007911414242728186,
+ "loss": 2.2087,
+ "theoretical_loss": 5.0572599923728,
+ "tokens_seen": 69206016
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007911253761283852,
+ "loss": 2.2024,
+ "theoretical_loss": 5.056565743090982,
+ "tokens_seen": 69271552
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791109327983952,
+ "loss": 2.0612,
+ "theoretical_loss": 5.055872334018677,
+ "tokens_seen": 69337088
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007910932798395186,
+ "loss": 2.2113,
+ "theoretical_loss": 5.0551797633463185,
+ "tokens_seen": 69402624
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007910772316950853,
+ "loss": 2.1887,
+ "theoretical_loss": 5.054488029269946,
+ "tokens_seen": 69468160
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000791061183550652,
+ "loss": 2.2918,
+ "theoretical_loss": 5.053797129991171,
+ "tokens_seen": 69533696
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007910451354062187,
+ "loss": 2.2494,
+ "theoretical_loss": 5.053107063717167,
+ "tokens_seen": 69599232
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007910290872617854,
+ "loss": 2.2314,
+ "theoretical_loss": 5.052417828660639,
+ "tokens_seen": 69664768
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007910130391173521,
+ "loss": 2.2445,
+ "theoretical_loss": 5.051729423039802,
+ "tokens_seen": 69730304
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909969909729188,
+ "loss": 2.2076,
+ "theoretical_loss": 5.0510418450783625,
+ "tokens_seen": 69795840
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909809428284855,
+ "loss": 2.2421,
+ "theoretical_loss": 5.050355093005495,
+ "tokens_seen": 69861376
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909648946840522,
+ "loss": 2.0975,
+ "theoretical_loss": 5.049669165055816,
+ "tokens_seen": 69926912
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909488465396189,
+ "loss": 2.1393,
+ "theoretical_loss": 5.048984059469369,
+ "tokens_seen": 69992448
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909327983951855,
+ "loss": 2.1042,
+ "theoretical_loss": 5.048299774491598,
+ "tokens_seen": 70057984
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909167502507523,
+ "loss": 2.1535,
+ "theoretical_loss": 5.047616308373328,
+ "tokens_seen": 70123520
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007909007021063189,
+ "loss": 2.2246,
+ "theoretical_loss": 5.046933659370742,
+ "tokens_seen": 70189056
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908846539618857,
+ "loss": 2.1902,
+ "theoretical_loss": 5.046251825745361,
+ "tokens_seen": 70254592
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908686058174523,
+ "loss": 2.247,
+ "theoretical_loss": 5.0455708057640205,
+ "tokens_seen": 70320128
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908525576730191,
+ "loss": 2.0948,
+ "theoretical_loss": 5.044890597698853,
+ "tokens_seen": 70385664
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908365095285857,
+ "loss": 2.2894,
+ "theoretical_loss": 5.044211199827263,
+ "tokens_seen": 70451200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908204613841525,
+ "loss": 2.1606,
+ "theoretical_loss": 5.04353261043191,
+ "tokens_seen": 70516736
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007908044132397191,
+ "loss": 2.2046,
+ "theoretical_loss": 5.0428548278006815,
+ "tokens_seen": 70582272
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907883650952859,
+ "loss": 2.0641,
+ "theoretical_loss": 5.042177850226682,
+ "tokens_seen": 70647808
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907723169508525,
+ "loss": 2.1669,
+ "theoretical_loss": 5.0415016760082,
+ "tokens_seen": 70713344
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907562688064193,
+ "loss": 2.1651,
+ "theoretical_loss": 5.040826303448698,
+ "tokens_seen": 70778880
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907402206619859,
+ "loss": 2.1432,
+ "theoretical_loss": 5.040151730856788,
+ "tokens_seen": 70844416
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907241725175527,
+ "loss": 2.1211,
+ "theoretical_loss": 5.039477956546207,
+ "tokens_seen": 70909952
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007907081243731193,
+ "loss": 2.2094,
+ "theoretical_loss": 5.038804978835808,
+ "tokens_seen": 70975488
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906920762286861,
+ "loss": 2.2419,
+ "theoretical_loss": 5.038132796049523,
+ "tokens_seen": 71041024
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906760280842527,
+ "loss": 2.1796,
+ "theoretical_loss": 5.037461406516362,
+ "tokens_seen": 71106560
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906599799398195,
+ "loss": 2.1457,
+ "theoretical_loss": 5.036790808570377,
+ "tokens_seen": 71172096
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906439317953861,
+ "loss": 2.1222,
+ "theoretical_loss": 5.036121000550651,
+ "tokens_seen": 71237632
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906278836509529,
+ "loss": 2.2514,
+ "theoretical_loss": 5.035451980801276,
+ "tokens_seen": 71303168
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007906118355065195,
+ "loss": 2.1864,
+ "theoretical_loss": 5.034783747671332,
+ "tokens_seen": 71368704
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905957873620863,
+ "loss": 2.148,
+ "theoretical_loss": 5.034116299514871,
+ "tokens_seen": 71434240
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905797392176529,
+ "loss": 2.1176,
+ "theoretical_loss": 5.033449634690893,
+ "tokens_seen": 71499776
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905636910732197,
+ "loss": 2.0967,
+ "theoretical_loss": 5.032783751563327,
+ "tokens_seen": 71565312
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905476429287863,
+ "loss": 2.1888,
+ "theoretical_loss": 5.032118648501017,
+ "tokens_seen": 71630848
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905315947843531,
+ "loss": 2.3203,
+ "theoretical_loss": 5.031454323877698,
+ "tokens_seen": 71696384
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007905155466399197,
+ "loss": 2.1714,
+ "theoretical_loss": 5.030790776071976,
+ "tokens_seen": 71761920
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904994984954865,
+ "loss": 2.0794,
+ "theoretical_loss": 5.030128003467312,
+ "tokens_seen": 71827456
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904834503510531,
+ "loss": 2.1015,
+ "theoretical_loss": 5.029466004452004,
+ "tokens_seen": 71892992
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904674022066199,
+ "loss": 2.0396,
+ "theoretical_loss": 5.028804777419165,
+ "tokens_seen": 71958528
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904513540621865,
+ "loss": 2.1914,
+ "theoretical_loss": 5.028144320766701,
+ "tokens_seen": 72024064
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 30813,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.0049593448638916,
+ "objective/train/theoretical_loss": 5.0274846328973055,
+ "objective/train/tokens_used": 92549600,
+ "theoretical_loss": 5.0274846328973055,
+ "tokens_seen": 72089600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904353059177533,
+ "loss": 2.1007,
+ "theoretical_loss": 5.0274846328973055,
+ "tokens_seen": 72089600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904192577733199,
+ "loss": 2.2275,
+ "theoretical_loss": 5.0268257122184234,
+ "tokens_seen": 72155136
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007904032096288867,
+ "loss": 2.1329,
+ "theoretical_loss": 5.026167557142247,
+ "tokens_seen": 72220672
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903871614844534,
+ "loss": 2.098,
+ "theoretical_loss": 5.02551016608569,
+ "tokens_seen": 72286208
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903711133400201,
+ "loss": 2.2127,
+ "theoretical_loss": 5.024853537470369,
+ "tokens_seen": 72351744
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903550651955868,
+ "loss": 2.0479,
+ "theoretical_loss": 5.02419766972259,
+ "tokens_seen": 72417280
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903390170511535,
+ "loss": 2.2583,
+ "theoretical_loss": 5.023542561273327,
+ "tokens_seen": 72482816
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903229689067202,
+ "loss": 2.2263,
+ "theoretical_loss": 5.022888210558206,
+ "tokens_seen": 72548352
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007903069207622869,
+ "loss": 2.2046,
+ "theoretical_loss": 5.02223461601748,
+ "tokens_seen": 72613888
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007902908726178537,
+ "loss": 2.3318,
+ "theoretical_loss": 5.021581776096022,
+ "tokens_seen": 72679424
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007902748244734203,
+ "loss": 2.131,
+ "theoretical_loss": 5.0209296892433,
+ "tokens_seen": 72744960
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000790258776328987,
+ "loss": 2.2132,
+ "theoretical_loss": 5.020278353913362,
+ "tokens_seen": 72810496
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007902427281845537,
+ "loss": 2.0379,
+ "theoretical_loss": 5.019627768564816,
+ "tokens_seen": 72876032
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007902266800401205,
+ "loss": 2.1585,
+ "theoretical_loss": 5.018977931660814,
+ "tokens_seen": 72941568
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007902106318956871,
+ "loss": 2.1721,
+ "theoretical_loss": 5.018328841669037,
+ "tokens_seen": 73007104
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901945837512539,
+ "loss": 2.3544,
+ "theoretical_loss": 5.017680497061671,
+ "tokens_seen": 73072640
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901785356068205,
+ "loss": 2.175,
+ "theoretical_loss": 5.017032896315397,
+ "tokens_seen": 73138176
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901624874623873,
+ "loss": 2.0713,
+ "theoretical_loss": 5.016386037911371,
+ "tokens_seen": 73203712
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901464393179539,
+ "loss": 2.2291,
+ "theoretical_loss": 5.015739920335203,
+ "tokens_seen": 73269248
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901303911735207,
+ "loss": 2.185,
+ "theoretical_loss": 5.015094542076945,
+ "tokens_seen": 73334784
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007901143430290873,
+ "loss": 2.09,
+ "theoretical_loss": 5.014449901631075,
+ "tokens_seen": 73400320
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900982948846541,
+ "loss": 2.1841,
+ "theoretical_loss": 5.013805997496473,
+ "tokens_seen": 73465856
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900822467402207,
+ "loss": 2.1307,
+ "theoretical_loss": 5.013162828176412,
+ "tokens_seen": 73531392
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900661985957875,
+ "loss": 2.2505,
+ "theoretical_loss": 5.012520392178539,
+ "tokens_seen": 73596928
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900501504513541,
+ "loss": 2.1488,
+ "theoretical_loss": 5.011878688014852,
+ "tokens_seen": 73662464
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900341023069209,
+ "loss": 2.1883,
+ "theoretical_loss": 5.011237714201694,
+ "tokens_seen": 73728000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900180541624875,
+ "loss": 2.0638,
+ "theoretical_loss": 5.010597469259729,
+ "tokens_seen": 73793536
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007900020060180543,
+ "loss": 2.1461,
+ "theoretical_loss": 5.009957951713927,
+ "tokens_seen": 73859072
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899859578736209,
+ "loss": 2.1418,
+ "theoretical_loss": 5.009319160093551,
+ "tokens_seen": 73924608
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899699097291877,
+ "loss": 2.1104,
+ "theoretical_loss": 5.008681092932137,
+ "tokens_seen": 73990144
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899538615847543,
+ "loss": 2.2886,
+ "theoretical_loss": 5.008043748767477,
+ "tokens_seen": 74055680
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899378134403211,
+ "loss": 2.1436,
+ "theoretical_loss": 5.007407126141608,
+ "tokens_seen": 74121216
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899217652958877,
+ "loss": 2.171,
+ "theoretical_loss": 5.00677122360079,
+ "tokens_seen": 74186752
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007899057171514545,
+ "loss": 2.189,
+ "theoretical_loss": 5.006136039695496,
+ "tokens_seen": 74252288
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898896690070211,
+ "loss": 2.1575,
+ "theoretical_loss": 5.005501572980391,
+ "tokens_seen": 74317824
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898736208625879,
+ "loss": 2.0729,
+ "theoretical_loss": 5.004867822014317,
+ "tokens_seen": 74383360
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898575727181545,
+ "loss": 2.144,
+ "theoretical_loss": 5.004234785360282,
+ "tokens_seen": 74448896
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898415245737213,
+ "loss": 2.1052,
+ "theoretical_loss": 5.0036024615854355,
+ "tokens_seen": 74514432
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898254764292879,
+ "loss": 2.113,
+ "theoretical_loss": 5.002970849261065,
+ "tokens_seen": 74579968
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007898094282848547,
+ "loss": 2.1806,
+ "theoretical_loss": 5.002339946962568,
+ "tokens_seen": 74645504
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897933801404213,
+ "loss": 2.1274,
+ "theoretical_loss": 5.001709753269443,
+ "tokens_seen": 74711040
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897773319959881,
+ "loss": 2.1426,
+ "theoretical_loss": 5.001080266765275,
+ "tokens_seen": 74776576
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897612838515547,
+ "loss": 2.2028,
+ "theoretical_loss": 5.0004514860377185,
+ "tokens_seen": 74842112
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897452357071215,
+ "loss": 2.1624,
+ "theoretical_loss": 4.999823409678481,
+ "tokens_seen": 74907648
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897291875626881,
+ "loss": 2.1102,
+ "theoretical_loss": 4.999196036283308,
+ "tokens_seen": 74973184
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007897131394182549,
+ "loss": 2.196,
+ "theoretical_loss": 4.9985693644519715,
+ "tokens_seen": 75038720
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896970912738215,
+ "loss": 2.1068,
+ "theoretical_loss": 4.997943392788251,
+ "tokens_seen": 75104256
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896810431293882,
+ "loss": 2.0444,
+ "theoretical_loss": 4.997318119899922,
+ "tokens_seen": 75169792
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896649949849549,
+ "loss": 1.96,
+ "theoretical_loss": 4.996693544398734,
+ "tokens_seen": 75235328
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896489468405216,
+ "loss": 2.133,
+ "theoretical_loss": 4.996069664900406,
+ "tokens_seen": 75300864
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 32032,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.0214736461639404,
+ "objective/train/theoretical_loss": 4.9954464800246035,
+ "objective/train/tokens_used": 95826400,
+ "theoretical_loss": 4.9954464800246035,
+ "tokens_seen": 75366400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896328986960883,
+ "loss": 2.0347,
+ "theoretical_loss": 4.9954464800246035,
+ "tokens_seen": 75366400
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789616850551655,
+ "loss": 2.1474,
+ "theoretical_loss": 4.994823988394928,
+ "tokens_seen": 75431936
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007896008024072217,
+ "loss": 2.1111,
+ "theoretical_loss": 4.994202188638901,
+ "tokens_seen": 75497472
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895847542627884,
+ "loss": 2.1961,
+ "theoretical_loss": 4.9935810793879485,
+ "tokens_seen": 75563008
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895687061183551,
+ "loss": 2.1599,
+ "theoretical_loss": 4.992960659277389,
+ "tokens_seen": 75628544
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895526579739218,
+ "loss": 2.1578,
+ "theoretical_loss": 4.992340926946417,
+ "tokens_seen": 75694080
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895366098294884,
+ "loss": 2.041,
+ "theoretical_loss": 4.991721881038091,
+ "tokens_seen": 75759616
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895205616850552,
+ "loss": 2.1556,
+ "theoretical_loss": 4.991103520199314,
+ "tokens_seen": 75825152
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007895045135406218,
+ "loss": 2.0935,
+ "theoretical_loss": 4.990485843080824,
+ "tokens_seen": 75890688
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007894884653961886,
+ "loss": 2.153,
+ "theoretical_loss": 4.989868848337181,
+ "tokens_seen": 75956224
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007894724172517552,
+ "loss": 2.1098,
+ "theoretical_loss": 4.989252534626749,
+ "tokens_seen": 76021760
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789456369107322,
+ "loss": 2.2057,
+ "theoretical_loss": 4.988636900611683,
+ "tokens_seen": 76087296
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007894403209628886,
+ "loss": 2.1796,
+ "theoretical_loss": 4.988021944957915,
+ "tokens_seen": 76152832
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007894242728184554,
+ "loss": 2.1348,
+ "theoretical_loss": 4.987407666335143,
+ "tokens_seen": 76218368
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789408224674022,
+ "loss": 2.003,
+ "theoretical_loss": 4.986794063416813,
+ "tokens_seen": 76283904
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893921765295888,
+ "loss": 2.2363,
+ "theoretical_loss": 4.986181134880107,
+ "tokens_seen": 76349440
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893761283851554,
+ "loss": 2.1553,
+ "theoretical_loss": 4.98556887940593,
+ "tokens_seen": 76414976
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893600802407222,
+ "loss": 2.0457,
+ "theoretical_loss": 4.984957295678894,
+ "tokens_seen": 76480512
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893440320962888,
+ "loss": 2.0834,
+ "theoretical_loss": 4.98434638238731,
+ "tokens_seen": 76546048
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893279839518556,
+ "loss": 2.1583,
+ "theoretical_loss": 4.983736138223165,
+ "tokens_seen": 76611584
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007893119358074222,
+ "loss": 2.1141,
+ "theoretical_loss": 4.983126561882118,
+ "tokens_seen": 76677120
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789295887662989,
+ "loss": 2.1146,
+ "theoretical_loss": 4.982517652063482,
+ "tokens_seen": 76742656
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007892798395185556,
+ "loss": 2.0532,
+ "theoretical_loss": 4.981909407470209,
+ "tokens_seen": 76808192
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007892637913741224,
+ "loss": 2.1857,
+ "theoretical_loss": 4.981301826808882,
+ "tokens_seen": 76873728
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789247743229689,
+ "loss": 2.1357,
+ "theoretical_loss": 4.9806949087896975,
+ "tokens_seen": 76939264
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007892316950852558,
+ "loss": 1.9885,
+ "theoretical_loss": 4.980088652126453,
+ "tokens_seen": 77004800
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007892156469408224,
+ "loss": 2.1321,
+ "theoretical_loss": 4.9794830555365355,
+ "tokens_seen": 77070336
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891995987963892,
+ "loss": 2.0872,
+ "theoretical_loss": 4.978878117740907,
+ "tokens_seen": 77135872
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891835506519559,
+ "loss": 2.1593,
+ "theoretical_loss": 4.978273837464091,
+ "tokens_seen": 77201408
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891675025075226,
+ "loss": 2.0579,
+ "theoretical_loss": 4.977670213434161,
+ "tokens_seen": 77266944
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891514543630893,
+ "loss": 2.2943,
+ "theoretical_loss": 4.977067244382729,
+ "tokens_seen": 77332480
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789135406218656,
+ "loss": 1.9335,
+ "theoretical_loss": 4.976464929044928,
+ "tokens_seen": 77398016
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891193580742227,
+ "loss": 2.0717,
+ "theoretical_loss": 4.975863266159403,
+ "tokens_seen": 77463552
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007891033099297894,
+ "loss": 2.0471,
+ "theoretical_loss": 4.975262254468299,
+ "tokens_seen": 77529088
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007890872617853561,
+ "loss": 2.07,
+ "theoretical_loss": 4.9746618927172435,
+ "tokens_seen": 77594624
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007890712136409228,
+ "loss": 2.1022,
+ "theoretical_loss": 4.9740621796553395,
+ "tokens_seen": 77660160
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007890551654964895,
+ "loss": 2.191,
+ "theoretical_loss": 4.973463114035149,
+ "tokens_seen": 77725696
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007890391173520562,
+ "loss": 2.0829,
+ "theoretical_loss": 4.972864694612684,
+ "tokens_seen": 77791232
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000789023069207623,
+ "loss": 2.0328,
+ "theoretical_loss": 4.972266920147391,
+ "tokens_seen": 77856768
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007890070210631896,
+ "loss": 2.1021,
+ "theoretical_loss": 4.971669789402139,
+ "tokens_seen": 77922304
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007889909729187563,
+ "loss": 2.045,
+ "theoretical_loss": 4.971073301143207,
+ "tokens_seen": 77987840
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788974924774323,
+ "loss": 2.1521,
+ "theoretical_loss": 4.970477454140278,
+ "tokens_seen": 78053376
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007889588766298897,
+ "loss": 2.1709,
+ "theoretical_loss": 4.9698822471664155,
+ "tokens_seen": 78118912
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007889428284854564,
+ "loss": 2.0816,
+ "theoretical_loss": 4.96928767899806,
+ "tokens_seen": 78184448
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007889267803410232,
+ "loss": 2.1294,
+ "theoretical_loss": 4.9686937484150135,
+ "tokens_seen": 78249984
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007889107321965898,
+ "loss": 2.0653,
+ "theoretical_loss": 4.96810045420043,
+ "tokens_seen": 78315520
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007888946840521566,
+ "loss": 2.0877,
+ "theoretical_loss": 4.967507795140797,
+ "tokens_seen": 78381056
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007888786359077232,
+ "loss": 2.17,
+ "theoretical_loss": 4.966915770025935,
+ "tokens_seen": 78446592
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.00078886258776329,
+ "loss": 2.1473,
+ "theoretical_loss": 4.966324377648973,
+ "tokens_seen": 78512128
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007888465396188566,
+ "loss": 2.1472,
+ "theoretical_loss": 4.965733616806345,
+ "tokens_seen": 78577664
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 32979,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.1013200283050537,
+ "objective/train/theoretical_loss": 4.965143486297777,
+ "objective/train/tokens_used": 99103200,
+ "theoretical_loss": 4.965143486297777,
+ "tokens_seen": 78643200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007888304914744234,
+ "loss": 2.0807,
+ "theoretical_loss": 4.965143486297777,
+ "tokens_seen": 78643200
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.00078881444332999,
+ "loss": 2.1694,
+ "theoretical_loss": 4.964553984926271,
+ "tokens_seen": 78708736
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887983951855568,
+ "loss": 2.0942,
+ "theoretical_loss": 4.963965111498098,
+ "tokens_seen": 78774272
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887823470411234,
+ "loss": 2.1027,
+ "theoretical_loss": 4.963376864822784,
+ "tokens_seen": 78839808
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887662988966902,
+ "loss": 2.1018,
+ "theoretical_loss": 4.962789243713102,
+ "tokens_seen": 78905344
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887502507522568,
+ "loss": 2.1184,
+ "theoretical_loss": 4.962202246985054,
+ "tokens_seen": 78970880
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887342026078236,
+ "loss": 2.0848,
+ "theoretical_loss": 4.9616158734578635,
+ "tokens_seen": 79036416
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007887181544633902,
+ "loss": 2.0948,
+ "theoretical_loss": 4.961030121953965,
+ "tokens_seen": 79101952
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788702106318957,
+ "loss": 2.1181,
+ "theoretical_loss": 4.960444991298992,
+ "tokens_seen": 79167488
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007886860581745236,
+ "loss": 2.1234,
+ "theoretical_loss": 4.959860480321762,
+ "tokens_seen": 79233024
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007886700100300904,
+ "loss": 2.112,
+ "theoretical_loss": 4.959276587854272,
+ "tokens_seen": 79298560
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788653961885657,
+ "loss": 2.1383,
+ "theoretical_loss": 4.958693312731681,
+ "tokens_seen": 79364096
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007886379137412238,
+ "loss": 2.1764,
+ "theoretical_loss": 4.9581106537923,
+ "tokens_seen": 79429632
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007886218655967904,
+ "loss": 2.0779,
+ "theoretical_loss": 4.957528609877587,
+ "tokens_seen": 79495168
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007886058174523572,
+ "loss": 2.0655,
+ "theoretical_loss": 4.9569471798321265,
+ "tokens_seen": 79560704
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007885897693079238,
+ "loss": 2.013,
+ "theoretical_loss": 4.9563663625036245,
+ "tokens_seen": 79626240
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007885737211634906,
+ "loss": 2.0405,
+ "theoretical_loss": 4.955786156742898,
+ "tokens_seen": 79691776
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007885576730190572,
+ "loss": 2.0635,
+ "theoretical_loss": 4.955206561403859,
+ "tokens_seen": 79757312
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788541624874624,
+ "loss": 2.1037,
+ "theoretical_loss": 4.954627575343509,
+ "tokens_seen": 79822848
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007885255767301906,
+ "loss": 1.9574,
+ "theoretical_loss": 4.954049197421925,
+ "tokens_seen": 79888384
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007885095285857574,
+ "loss": 2.1531,
+ "theoretical_loss": 4.953471426502249,
+ "tokens_seen": 79953920
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788493480441324,
+ "loss": 2.157,
+ "theoretical_loss": 4.952894261450679,
+ "tokens_seen": 80019456
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007884774322968908,
+ "loss": 2.044,
+ "theoretical_loss": 4.952317701136457,
+ "tokens_seen": 80084992
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007884613841524574,
+ "loss": 2.1789,
+ "theoretical_loss": 4.9517417444318585,
+ "tokens_seen": 80150528
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007884453360080242,
+ "loss": 2.1454,
+ "theoretical_loss": 4.951166390212181,
+ "tokens_seen": 80216064
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007884292878635908,
+ "loss": 2.0481,
+ "theoretical_loss": 4.950591637355737,
+ "tokens_seen": 80281600
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007884132397191576,
+ "loss": 2.0555,
+ "theoretical_loss": 4.950017484743835,
+ "tokens_seen": 80347136
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007883971915747242,
+ "loss": 2.0603,
+ "theoretical_loss": 4.949443931260783,
+ "tokens_seen": 80412672
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788381143430291,
+ "loss": 2.0551,
+ "theoretical_loss": 4.948870975793863,
+ "tokens_seen": 80478208
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007883650952858576,
+ "loss": 2.1272,
+ "theoretical_loss": 4.9482986172333305,
+ "tokens_seen": 80543744
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007883490471414244,
+ "loss": 2.035,
+ "theoretical_loss": 4.9477268544724,
+ "tokens_seen": 80609280
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788332998996991,
+ "loss": 2.0542,
+ "theoretical_loss": 4.947155686407239,
+ "tokens_seen": 80674816
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007883169508525578,
+ "loss": 2.1393,
+ "theoretical_loss": 4.946585111936951,
+ "tokens_seen": 80740352
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007883009027081244,
+ "loss": 2.0737,
+ "theoretical_loss": 4.94601512996357,
+ "tokens_seen": 80805888
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882848545636911,
+ "loss": 1.9408,
+ "theoretical_loss": 4.945445739392049,
+ "tokens_seen": 80871424
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882688064192578,
+ "loss": 1.8858,
+ "theoretical_loss": 4.944876939130252,
+ "tokens_seen": 80936960
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882527582748245,
+ "loss": 2.0729,
+ "theoretical_loss": 4.94430872808894,
+ "tokens_seen": 81002496
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882367101303912,
+ "loss": 2.04,
+ "theoretical_loss": 4.943741105181765,
+ "tokens_seen": 81068032
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882206619859579,
+ "loss": 2.0131,
+ "theoretical_loss": 4.943174069325255,
+ "tokens_seen": 81133568
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007882046138415246,
+ "loss": 1.9603,
+ "theoretical_loss": 4.94260761943881,
+ "tokens_seen": 81199104
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007881885656970913,
+ "loss": 1.985,
+ "theoretical_loss": 4.942041754444688,
+ "tokens_seen": 81264640
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.000788172517552658,
+ "loss": 2.039,
+ "theoretical_loss": 4.941476473267996,
+ "tokens_seen": 81330176
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007881564694082247,
+ "loss": 2.1917,
+ "theoretical_loss": 4.940911774836682,
+ "tokens_seen": 81395712
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007881404212637913,
+ "loss": 2.155,
+ "theoretical_loss": 4.940347658081521,
+ "tokens_seen": 81461248
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007881243731193581,
+ "loss": 2.0186,
+ "theoretical_loss": 4.939784121936112,
+ "tokens_seen": 81526784
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007881083249749247,
+ "loss": 2.0964,
+ "theoretical_loss": 4.9392211653368605,
+ "tokens_seen": 81592320
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880922768304915,
+ "loss": 1.9986,
+ "theoretical_loss": 4.938658787222975,
+ "tokens_seen": 81657856
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880762286860581,
+ "loss": 2.078,
+ "theoretical_loss": 4.938096986536452,
+ "tokens_seen": 81723392
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880601805416249,
+ "loss": 2.0657,
+ "theoretical_loss": 4.937535762222075,
+ "tokens_seen": 81788928
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880441323971915,
+ "loss": 2.111,
+ "theoretical_loss": 4.936975113227394,
+ "tokens_seen": 81854464
+ },
+ {
+ "epoch": 0.02,
+ "objective/train/docs_used": 34316,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.1313977241516113,
+ "objective/train/theoretical_loss": 4.936415038502723,
+ "objective/train/tokens_used": 102380000,
+ "theoretical_loss": 4.936415038502723,
+ "tokens_seen": 81920000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880280842527583,
+ "loss": 2.1647,
+ "theoretical_loss": 4.936415038502723,
+ "tokens_seen": 81920000
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007880120361083249,
+ "loss": 2.0975,
+ "theoretical_loss": 4.93585553700113,
+ "tokens_seen": 81985536
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879959879638917,
+ "loss": 2.0226,
+ "theoretical_loss": 4.935296607678426,
+ "tokens_seen": 82051072
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879799398194584,
+ "loss": 2.0912,
+ "theoretical_loss": 4.934738249493156,
+ "tokens_seen": 82116608
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879638916750251,
+ "loss": 2.0794,
+ "theoretical_loss": 4.934180461406591,
+ "tokens_seen": 82182144
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879478435305918,
+ "loss": 2.0681,
+ "theoretical_loss": 4.933623242382714,
+ "tokens_seen": 82247680
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879317953861585,
+ "loss": 2.0694,
+ "theoretical_loss": 4.9330665913882195,
+ "tokens_seen": 82313216
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007879157472417252,
+ "loss": 2.0263,
+ "theoretical_loss": 4.932510507392495,
+ "tokens_seen": 82378752
+ },
+ {
+ "epoch": 0.02,
+ "learning_rate": 0.0007878996990972919,
+ "loss": 2.0941,
+ "theoretical_loss": 4.931954989367618,
+ "tokens_seen": 82444288
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007878836509528586,
+ "loss": 2.1159,
+ "theoretical_loss": 4.931400036288343,
+ "tokens_seen": 82509824
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007878676028084253,
+ "loss": 1.965,
+ "theoretical_loss": 4.930845647132097,
+ "tokens_seen": 82575360
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000787851554663992,
+ "loss": 2.0173,
+ "theoretical_loss": 4.9302918208789634,
+ "tokens_seen": 82640896
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007878355065195587,
+ "loss": 2.0566,
+ "theoretical_loss": 4.929738556511681,
+ "tokens_seen": 82706432
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007878194583751254,
+ "loss": 2.0425,
+ "theoretical_loss": 4.929185853015629,
+ "tokens_seen": 82771968
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007878034102306921,
+ "loss": 2.0891,
+ "theoretical_loss": 4.928633709378822,
+ "tokens_seen": 82837504
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877873620862588,
+ "loss": 2.1192,
+ "theoretical_loss": 4.9280821245918975,
+ "tokens_seen": 82903040
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877713139418255,
+ "loss": 1.8945,
+ "theoretical_loss": 4.92753109764811,
+ "tokens_seen": 82968576
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877552657973922,
+ "loss": 2.1443,
+ "theoretical_loss": 4.92698062754332,
+ "tokens_seen": 83034112
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877392176529589,
+ "loss": 2.263,
+ "theoretical_loss": 4.9264307132759875,
+ "tokens_seen": 83099648
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877231695085256,
+ "loss": 2.0305,
+ "theoretical_loss": 4.92588135384716,
+ "tokens_seen": 83165184
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007877071213640923,
+ "loss": 2.0397,
+ "theoretical_loss": 4.925332548260469,
+ "tokens_seen": 83230720
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000787691073219659,
+ "loss": 1.9951,
+ "theoretical_loss": 4.924784295522113,
+ "tokens_seen": 83296256
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007876750250752257,
+ "loss": 2.014,
+ "theoretical_loss": 4.92423659464086,
+ "tokens_seen": 83361792
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007876589769307924,
+ "loss": 2.0954,
+ "theoretical_loss": 4.923689444628027,
+ "tokens_seen": 83427328
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007876429287863591,
+ "loss": 1.9732,
+ "theoretical_loss": 4.923142844497482,
+ "tokens_seen": 83492864
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007876268806419258,
+ "loss": 2.0625,
+ "theoretical_loss": 4.922596793265625,
+ "tokens_seen": 83558400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007876108324974925,
+ "loss": 2.0099,
+ "theoretical_loss": 4.922051289951391,
+ "tokens_seen": 83623936
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007875947843530592,
+ "loss": 1.9741,
+ "theoretical_loss": 4.921506333576232,
+ "tokens_seen": 83689472
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007875787362086259,
+ "loss": 2.0757,
+ "theoretical_loss": 4.920961923164114,
+ "tokens_seen": 83755008
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007875626880641927,
+ "loss": 2.0747,
+ "theoretical_loss": 4.920418057741504,
+ "tokens_seen": 83820544
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007875466399197593,
+ "loss": 2.1795,
+ "theoretical_loss": 4.919874736337368,
+ "tokens_seen": 83886080
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000787530591775326,
+ "loss": 2.0879,
+ "theoretical_loss": 4.9193319579831565,
+ "tokens_seen": 83951616
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007875145436308927,
+ "loss": 2.0928,
+ "theoretical_loss": 4.918789721712799,
+ "tokens_seen": 84017152
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874984954864595,
+ "loss": 2.1112,
+ "theoretical_loss": 4.918248026562697,
+ "tokens_seen": 84082688
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874824473420261,
+ "loss": 2.0547,
+ "theoretical_loss": 4.917706871571712,
+ "tokens_seen": 84148224
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874663991975929,
+ "loss": 2.1856,
+ "theoretical_loss": 4.9171662557811615,
+ "tokens_seen": 84213760
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874503510531595,
+ "loss": 2.1379,
+ "theoretical_loss": 4.916626178234807,
+ "tokens_seen": 84279296
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874343029087263,
+ "loss": 2.0925,
+ "theoretical_loss": 4.916086637978851,
+ "tokens_seen": 84344832
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874182547642929,
+ "loss": 2.1199,
+ "theoretical_loss": 4.915547634061921,
+ "tokens_seen": 84410368
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007874022066198597,
+ "loss": 2.206,
+ "theoretical_loss": 4.91500916553507,
+ "tokens_seen": 84475904
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873861584754263,
+ "loss": 2.1473,
+ "theoretical_loss": 4.914471231451762,
+ "tokens_seen": 84541440
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873701103309931,
+ "loss": 2.1729,
+ "theoretical_loss": 4.913933830867868,
+ "tokens_seen": 84606976
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873540621865597,
+ "loss": 2.0473,
+ "theoretical_loss": 4.913396962841655,
+ "tokens_seen": 84672512
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873380140421265,
+ "loss": 2.1081,
+ "theoretical_loss": 4.9128606264337815,
+ "tokens_seen": 84738048
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873219658976931,
+ "loss": 2.2149,
+ "theoretical_loss": 4.912324820707285,
+ "tokens_seen": 84803584
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007873059177532599,
+ "loss": 2.1657,
+ "theoretical_loss": 4.911789544727581,
+ "tokens_seen": 84869120
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872898696088265,
+ "loss": 2.0486,
+ "theoretical_loss": 4.911254797562448,
+ "tokens_seen": 84934656
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872738214643933,
+ "loss": 2.1763,
+ "theoretical_loss": 4.910720578282021,
+ "tokens_seen": 85000192
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872577733199599,
+ "loss": 2.025,
+ "theoretical_loss": 4.91018688595879,
+ "tokens_seen": 85065728
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872417251755267,
+ "loss": 2.0928,
+ "theoretical_loss": 4.909653719667584,
+ "tokens_seen": 85131264
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 35098,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9277470111846924,
+ "objective/train/theoretical_loss": 4.909121078485567,
+ "objective/train/tokens_used": 105656800,
+ "theoretical_loss": 4.909121078485567,
+ "tokens_seen": 85196800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872256770310933,
+ "loss": 2.1105,
+ "theoretical_loss": 4.909121078485567,
+ "tokens_seen": 85196800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007872096288866601,
+ "loss": 2.0636,
+ "theoretical_loss": 4.908588961492235,
+ "tokens_seen": 85262336
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871935807422267,
+ "loss": 2.1548,
+ "theoretical_loss": 4.908057367769396,
+ "tokens_seen": 85327872
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871775325977935,
+ "loss": 2.146,
+ "theoretical_loss": 4.907526296401175,
+ "tokens_seen": 85393408
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871614844533601,
+ "loss": 2.0305,
+ "theoretical_loss": 4.906995746474001,
+ "tokens_seen": 85458944
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871454363089269,
+ "loss": 2.1158,
+ "theoretical_loss": 4.9064657170766,
+ "tokens_seen": 85524480
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871293881644935,
+ "loss": 2.1002,
+ "theoretical_loss": 4.905936207299984,
+ "tokens_seen": 85590016
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007871133400200603,
+ "loss": 2.0993,
+ "theoretical_loss": 4.90540721623745,
+ "tokens_seen": 85655552
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870972918756269,
+ "loss": 1.978,
+ "theoretical_loss": 4.904878742984569,
+ "tokens_seen": 85721088
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870812437311937,
+ "loss": 2.0845,
+ "theoretical_loss": 4.904350786639178,
+ "tokens_seen": 85786624
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870651955867603,
+ "loss": 2.1174,
+ "theoretical_loss": 4.903823346301374,
+ "tokens_seen": 85852160
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870491474423271,
+ "loss": 1.9148,
+ "theoretical_loss": 4.903296421073506,
+ "tokens_seen": 85917696
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870330992978937,
+ "loss": 2.203,
+ "theoretical_loss": 4.902770010060166,
+ "tokens_seen": 85983232
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870170511534605,
+ "loss": 2.1191,
+ "theoretical_loss": 4.902244112368188,
+ "tokens_seen": 86048768
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007870010030090271,
+ "loss": 2.1491,
+ "theoretical_loss": 4.901718727106631,
+ "tokens_seen": 86114304
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869849548645939,
+ "loss": 2.1635,
+ "theoretical_loss": 4.90119385338678,
+ "tokens_seen": 86179840
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869689067201605,
+ "loss": 1.9647,
+ "theoretical_loss": 4.900669490322134,
+ "tokens_seen": 86245376
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869528585757273,
+ "loss": 2.0817,
+ "theoretical_loss": 4.900145637028402,
+ "tokens_seen": 86310912
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869368104312939,
+ "loss": 2.1156,
+ "theoretical_loss": 4.899622292623493,
+ "tokens_seen": 86376448
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869207622868607,
+ "loss": 1.9936,
+ "theoretical_loss": 4.8990994562275105,
+ "tokens_seen": 86441984
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007869047141424273,
+ "loss": 2.1465,
+ "theoretical_loss": 4.898577126962746,
+ "tokens_seen": 86507520
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868886659979941,
+ "loss": 2.1501,
+ "theoretical_loss": 4.898055303953669,
+ "tokens_seen": 86573056
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868726178535607,
+ "loss": 2.0958,
+ "theoretical_loss": 4.897533986326922,
+ "tokens_seen": 86638592
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868565697091274,
+ "loss": 2.0493,
+ "theoretical_loss": 4.897013173211316,
+ "tokens_seen": 86704128
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868405215646941,
+ "loss": 2.061,
+ "theoretical_loss": 4.896492863737818,
+ "tokens_seen": 86769664
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868244734202608,
+ "loss": 2.1131,
+ "theoretical_loss": 4.895973057039548,
+ "tokens_seen": 86835200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007868084252758275,
+ "loss": 2.0576,
+ "theoretical_loss": 4.89545375225177,
+ "tokens_seen": 86900736
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007867923771313942,
+ "loss": 2.0952,
+ "theoretical_loss": 4.894934948511889,
+ "tokens_seen": 86966272
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007867763289869609,
+ "loss": 2.0713,
+ "theoretical_loss": 4.894416644959437,
+ "tokens_seen": 87031808
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007867602808425276,
+ "loss": 2.1069,
+ "theoretical_loss": 4.893898840736071,
+ "tokens_seen": 87097344
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007867442326980943,
+ "loss": 2.0183,
+ "theoretical_loss": 4.8933815349855685,
+ "tokens_seen": 87162880
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786728184553661,
+ "loss": 2.1052,
+ "theoretical_loss": 4.892864726853814,
+ "tokens_seen": 87228416
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007867121364092277,
+ "loss": 2.0933,
+ "theoretical_loss": 4.892348415488799,
+ "tokens_seen": 87293952
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866960882647944,
+ "loss": 2.0157,
+ "theoretical_loss": 4.891832600040609,
+ "tokens_seen": 87359488
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866800401203611,
+ "loss": 2.1374,
+ "theoretical_loss": 4.891317279661422,
+ "tokens_seen": 87425024
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866639919759278,
+ "loss": 2.0133,
+ "theoretical_loss": 4.890802453505498,
+ "tokens_seen": 87490560
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866479438314945,
+ "loss": 1.901,
+ "theoretical_loss": 4.890288120729174,
+ "tokens_seen": 87556096
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866318956870612,
+ "loss": 2.1402,
+ "theoretical_loss": 4.88977428049086,
+ "tokens_seen": 87621632
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007866158475426279,
+ "loss": 1.9311,
+ "theoretical_loss": 4.889260931951026,
+ "tokens_seen": 87687168
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865997993981946,
+ "loss": 2.1391,
+ "theoretical_loss": 4.888748074272201,
+ "tokens_seen": 87752704
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865837512537613,
+ "loss": 2.1003,
+ "theoretical_loss": 4.888235706618964,
+ "tokens_seen": 87818240
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786567703109328,
+ "loss": 2.0694,
+ "theoretical_loss": 4.887723828157939,
+ "tokens_seen": 87883776
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865516549648947,
+ "loss": 2.0203,
+ "theoretical_loss": 4.8872124380577855,
+ "tokens_seen": 87949312
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865356068204614,
+ "loss": 2.0587,
+ "theoretical_loss": 4.886701535489195,
+ "tokens_seen": 88014848
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865195586760281,
+ "loss": 2.1354,
+ "theoretical_loss": 4.886191119624883,
+ "tokens_seen": 88080384
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007865035105315948,
+ "loss": 2.034,
+ "theoretical_loss": 4.885681189639584,
+ "tokens_seen": 88145920
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007864874623871615,
+ "loss": 2.1003,
+ "theoretical_loss": 4.885171744710044,
+ "tokens_seen": 88211456
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007864714142427282,
+ "loss": 2.1345,
+ "theoretical_loss": 4.884662784015012,
+ "tokens_seen": 88276992
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007864553660982949,
+ "loss": 2.0893,
+ "theoretical_loss": 4.884154306735239,
+ "tokens_seen": 88342528
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007864393179538616,
+ "loss": 2.1243,
+ "theoretical_loss": 4.8836463120534646,
+ "tokens_seen": 88408064
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 36417,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8406845331192017,
+ "objective/train/theoretical_loss": 4.88313879915442,
+ "objective/train/tokens_used": 108933600,
+ "theoretical_loss": 4.88313879915442,
+ "tokens_seen": 88473600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007864232698094283,
+ "loss": 1.8913,
+ "theoretical_loss": 4.88313879915442,
+ "tokens_seen": 88473600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786407221664995,
+ "loss": 2.1626,
+ "theoretical_loss": 4.882631767224812,
+ "tokens_seen": 88539136
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863911735205617,
+ "loss": 2.0407,
+ "theoretical_loss": 4.882125215453321,
+ "tokens_seen": 88604672
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863751253761284,
+ "loss": 2.0725,
+ "theoretical_loss": 4.881619143030598,
+ "tokens_seen": 88670208
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863590772316951,
+ "loss": 2.007,
+ "theoretical_loss": 4.8811135491492506,
+ "tokens_seen": 88735744
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863430290872618,
+ "loss": 2.0586,
+ "theoretical_loss": 4.880608433003846,
+ "tokens_seen": 88801280
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863269809428285,
+ "loss": 2.0653,
+ "theoretical_loss": 4.880103793790896,
+ "tokens_seen": 88866816
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007863109327983952,
+ "loss": 2.0433,
+ "theoretical_loss": 4.87959963070886,
+ "tokens_seen": 88932352
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786294884653962,
+ "loss": 2.0326,
+ "theoretical_loss": 4.879095942958127,
+ "tokens_seen": 88997888
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007862788365095286,
+ "loss": 2.0379,
+ "theoretical_loss": 4.878592729741024,
+ "tokens_seen": 89063424
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007862627883650953,
+ "loss": 2.1523,
+ "theoretical_loss": 4.878089990261797,
+ "tokens_seen": 89128960
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786246740220662,
+ "loss": 1.9848,
+ "theoretical_loss": 4.87758772372661,
+ "tokens_seen": 89194496
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007862306920762287,
+ "loss": 1.9817,
+ "theoretical_loss": 4.877085929343544,
+ "tokens_seen": 89260032
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007862146439317954,
+ "loss": 2.1303,
+ "theoretical_loss": 4.876584606322583,
+ "tokens_seen": 89325568
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861985957873622,
+ "loss": 2.0567,
+ "theoretical_loss": 4.8760837538756086,
+ "tokens_seen": 89391104
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861825476429288,
+ "loss": 1.9851,
+ "theoretical_loss": 4.875583371216404,
+ "tokens_seen": 89456640
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861664994984956,
+ "loss": 2.0458,
+ "theoretical_loss": 4.875083457560633,
+ "tokens_seen": 89522176
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861504513540622,
+ "loss": 1.9532,
+ "theoretical_loss": 4.874584012125847,
+ "tokens_seen": 89587712
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786134403209629,
+ "loss": 2.0171,
+ "theoretical_loss": 4.874085034131472,
+ "tokens_seen": 89653248
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861183550651956,
+ "loss": 2.016,
+ "theoretical_loss": 4.873586522798803,
+ "tokens_seen": 89718784
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007861023069207624,
+ "loss": 2.0034,
+ "theoretical_loss": 4.873088477351005,
+ "tokens_seen": 89784320
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000786086258776329,
+ "loss": 2.0347,
+ "theoretical_loss": 4.8725908970130964,
+ "tokens_seen": 89849856
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007860702106318958,
+ "loss": 2.0416,
+ "theoretical_loss": 4.872093781011952,
+ "tokens_seen": 89915392
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007860541624874624,
+ "loss": 1.9762,
+ "theoretical_loss": 4.871597128576292,
+ "tokens_seen": 89980928
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007860381143430292,
+ "loss": 1.9752,
+ "theoretical_loss": 4.8711009389366815,
+ "tokens_seen": 90046464
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007860220661985958,
+ "loss": 2.0416,
+ "theoretical_loss": 4.870605211325517,
+ "tokens_seen": 90112000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007860060180541626,
+ "loss": 1.9874,
+ "theoretical_loss": 4.870109944977029,
+ "tokens_seen": 90177536
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007859899699097292,
+ "loss": 2.0183,
+ "theoretical_loss": 4.869615139127273,
+ "tokens_seen": 90243072
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785973921765296,
+ "loss": 2.079,
+ "theoretical_loss": 4.869120793014117,
+ "tokens_seen": 90308608
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007859578736208626,
+ "loss": 2.039,
+ "theoretical_loss": 4.868626905877251,
+ "tokens_seen": 90374144
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007859418254764294,
+ "loss": 2.0082,
+ "theoretical_loss": 4.8681334769581674,
+ "tokens_seen": 90439680
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785925777331996,
+ "loss": 1.88,
+ "theoretical_loss": 4.867640505500161,
+ "tokens_seen": 90505216
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007859097291875628,
+ "loss": 2.0596,
+ "theoretical_loss": 4.8671479907483235,
+ "tokens_seen": 90570752
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007858936810431294,
+ "loss": 2.0777,
+ "theoretical_loss": 4.866655931949537,
+ "tokens_seen": 90636288
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007858776328986962,
+ "loss": 2.0637,
+ "theoretical_loss": 4.86616432835247,
+ "tokens_seen": 90701824
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007858615847542628,
+ "loss": 1.9995,
+ "theoretical_loss": 4.865673179207572,
+ "tokens_seen": 90767360
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007858455366098296,
+ "loss": 2.1029,
+ "theoretical_loss": 4.865182483767063,
+ "tokens_seen": 90832896
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007858294884653962,
+ "loss": 2.0009,
+ "theoretical_loss": 4.864692241284933,
+ "tokens_seen": 90898432
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785813440320963,
+ "loss": 2.0887,
+ "theoretical_loss": 4.864202451016939,
+ "tokens_seen": 90963968
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857973921765296,
+ "loss": 2.0239,
+ "theoretical_loss": 4.863713112220592,
+ "tokens_seen": 91029504
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857813440320964,
+ "loss": 2.0429,
+ "theoretical_loss": 4.8632242241551555,
+ "tokens_seen": 91095040
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785765295887663,
+ "loss": 2.0759,
+ "theoretical_loss": 4.862735786081642,
+ "tokens_seen": 91160576
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857492477432298,
+ "loss": 2.014,
+ "theoretical_loss": 4.862247797262806,
+ "tokens_seen": 91226112
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857331995987964,
+ "loss": 2.0947,
+ "theoretical_loss": 4.861760256963136,
+ "tokens_seen": 91291648
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857171514543632,
+ "loss": 2.0521,
+ "theoretical_loss": 4.861273164448854,
+ "tokens_seen": 91357184
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007857011033099298,
+ "loss": 1.9563,
+ "theoretical_loss": 4.860786518987906,
+ "tokens_seen": 91422720
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007856850551654966,
+ "loss": 2.0038,
+ "theoretical_loss": 4.86030031984996,
+ "tokens_seen": 91488256
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007856690070210632,
+ "loss": 1.9585,
+ "theoretical_loss": 4.859814566306397,
+ "tokens_seen": 91553792
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.00078565295887663,
+ "loss": 1.9885,
+ "theoretical_loss": 4.859329257630311,
+ "tokens_seen": 91619328
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007856369107321966,
+ "loss": 2.0648,
+ "theoretical_loss": 4.858844393096497,
+ "tokens_seen": 91684864
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 37720,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.0330617427825928,
+ "objective/train/theoretical_loss": 4.858359971981454,
+ "objective/train/tokens_used": 112210400,
+ "theoretical_loss": 4.858359971981454,
+ "tokens_seen": 91750400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007856208625877634,
+ "loss": 2.0313,
+ "theoretical_loss": 4.858359971981454,
+ "tokens_seen": 91750400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.00078560481444333,
+ "loss": 2.1479,
+ "theoretical_loss": 4.85787599356337,
+ "tokens_seen": 91815936
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855887662988968,
+ "loss": 2.0328,
+ "theoretical_loss": 4.857392457122128,
+ "tokens_seen": 91881472
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855727181544634,
+ "loss": 2.1289,
+ "theoretical_loss": 4.856909361939288,
+ "tokens_seen": 91947008
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855566700100302,
+ "loss": 1.9571,
+ "theoretical_loss": 4.856426707298096,
+ "tokens_seen": 92012544
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855406218655968,
+ "loss": 1.9227,
+ "theoretical_loss": 4.8559444924834665,
+ "tokens_seen": 92078080
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855245737211636,
+ "loss": 1.9564,
+ "theoretical_loss": 4.855462716781985,
+ "tokens_seen": 92143616
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007855085255767302,
+ "loss": 2.0934,
+ "theoretical_loss": 4.854981379481901,
+ "tokens_seen": 92209152
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785492477432297,
+ "loss": 2.0322,
+ "theoretical_loss": 4.854500479873119,
+ "tokens_seen": 92274688
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007854764292878636,
+ "loss": 1.9538,
+ "theoretical_loss": 4.854020017247203,
+ "tokens_seen": 92340224
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007854603811434303,
+ "loss": 1.9776,
+ "theoretical_loss": 4.853539990897358,
+ "tokens_seen": 92405760
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785444332998997,
+ "loss": 2.0646,
+ "theoretical_loss": 4.85306040011844,
+ "tokens_seen": 92471296
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007854282848545637,
+ "loss": 1.9781,
+ "theoretical_loss": 4.852581244206938,
+ "tokens_seen": 92536832
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007854122367101304,
+ "loss": 2.0651,
+ "theoretical_loss": 4.852102522460975,
+ "tokens_seen": 92602368
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853961885656971,
+ "loss": 1.9221,
+ "theoretical_loss": 4.851624234180306,
+ "tokens_seen": 92667904
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853801404212638,
+ "loss": 2.1273,
+ "theoretical_loss": 4.851146378666305,
+ "tokens_seen": 92733440
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853640922768305,
+ "loss": 1.9866,
+ "theoretical_loss": 4.85066895522197,
+ "tokens_seen": 92798976
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853480441323972,
+ "loss": 2.0728,
+ "theoretical_loss": 4.850191963151908,
+ "tokens_seen": 92864512
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853319959879639,
+ "loss": 2.1431,
+ "theoretical_loss": 4.849715401762337,
+ "tokens_seen": 92930048
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007853159478435306,
+ "loss": 2.0067,
+ "theoretical_loss": 4.849239270361078,
+ "tokens_seen": 92995584
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852998996990973,
+ "loss": 1.9941,
+ "theoretical_loss": 4.848763568257554,
+ "tokens_seen": 93061120
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785283851554664,
+ "loss": 2.0881,
+ "theoretical_loss": 4.848288294762779,
+ "tokens_seen": 93126656
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852678034102307,
+ "loss": 2.0293,
+ "theoretical_loss": 4.847813449189358,
+ "tokens_seen": 93192192
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852517552657974,
+ "loss": 1.958,
+ "theoretical_loss": 4.847339030851482,
+ "tokens_seen": 93257728
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852357071213641,
+ "loss": 1.9812,
+ "theoretical_loss": 4.84686503906492,
+ "tokens_seen": 93323264
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852196589769308,
+ "loss": 2.1079,
+ "theoretical_loss": 4.846391473147019,
+ "tokens_seen": 93388800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007852036108324975,
+ "loss": 1.9527,
+ "theoretical_loss": 4.8459183324166935,
+ "tokens_seen": 93454336
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007851875626880642,
+ "loss": 1.9455,
+ "theoretical_loss": 4.845445616194426,
+ "tokens_seen": 93519872
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007851715145436309,
+ "loss": 2.1299,
+ "theoretical_loss": 4.844973323802259,
+ "tokens_seen": 93585408
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007851554663991976,
+ "loss": 2.0971,
+ "theoretical_loss": 4.844501454563792,
+ "tokens_seen": 93650944
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007851394182547643,
+ "loss": 1.9184,
+ "theoretical_loss": 4.844030007804177,
+ "tokens_seen": 93716480
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000785123370110331,
+ "loss": 1.9424,
+ "theoretical_loss": 4.843558982850113,
+ "tokens_seen": 93782016
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007851073219658977,
+ "loss": 2.0786,
+ "theoretical_loss": 4.84308837902984,
+ "tokens_seen": 93847552
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850912738214644,
+ "loss": 1.9767,
+ "theoretical_loss": 4.842618195673138,
+ "tokens_seen": 93913088
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850752256770311,
+ "loss": 2.0386,
+ "theoretical_loss": 4.84214843211132,
+ "tokens_seen": 93978624
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850591775325978,
+ "loss": 1.9345,
+ "theoretical_loss": 4.841679087677226,
+ "tokens_seen": 94044160
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850431293881645,
+ "loss": 2.1009,
+ "theoretical_loss": 4.841210161705223,
+ "tokens_seen": 94109696
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850270812437312,
+ "loss": 2.0208,
+ "theoretical_loss": 4.840741653531195,
+ "tokens_seen": 94175232
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007850110330992979,
+ "loss": 1.8831,
+ "theoretical_loss": 4.840273562492545,
+ "tokens_seen": 94240768
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007849949849548646,
+ "loss": 2.0515,
+ "theoretical_loss": 4.839805887928181,
+ "tokens_seen": 94306304
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007849789368104313,
+ "loss": 1.9214,
+ "theoretical_loss": 4.839338629178522,
+ "tokens_seen": 94371840
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000784962888665998,
+ "loss": 1.9477,
+ "theoretical_loss": 4.8388717855854875,
+ "tokens_seen": 94437376
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007849468405215647,
+ "loss": 1.927,
+ "theoretical_loss": 4.838405356492494,
+ "tokens_seen": 94502912
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007849307923771314,
+ "loss": 1.9606,
+ "theoretical_loss": 4.83793934124445,
+ "tokens_seen": 94568448
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007849147442326981,
+ "loss": 1.9475,
+ "theoretical_loss": 4.837473739187754,
+ "tokens_seen": 94633984
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848986960882648,
+ "loss": 2.0574,
+ "theoretical_loss": 4.837008549670285,
+ "tokens_seen": 94699520
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848826479438315,
+ "loss": 2.0748,
+ "theoretical_loss": 4.8365437720414075,
+ "tokens_seen": 94765056
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848665997993982,
+ "loss": 2.0284,
+ "theoretical_loss": 4.836079405651956,
+ "tokens_seen": 94830592
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848505516549649,
+ "loss": 2.039,
+ "theoretical_loss": 4.835615449854238,
+ "tokens_seen": 94896128
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848345035105317,
+ "loss": 1.9871,
+ "theoretical_loss": 4.8351519040020285,
+ "tokens_seen": 94961664
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 38548,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.0551602840423584,
+ "objective/train/theoretical_loss": 4.834688767450562,
+ "objective/train/tokens_used": 115487200,
+ "theoretical_loss": 4.834688767450562,
+ "tokens_seen": 95027200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007848184553660983,
+ "loss": 2.0169,
+ "theoretical_loss": 4.834688767450562,
+ "tokens_seen": 95027200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000784802407221665,
+ "loss": 1.9129,
+ "theoretical_loss": 4.8342260395565315,
+ "tokens_seen": 95092736
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847863590772317,
+ "loss": 1.9872,
+ "theoretical_loss": 4.833763719678085,
+ "tokens_seen": 95158272
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847703109327985,
+ "loss": 1.954,
+ "theoretical_loss": 4.833301807174822,
+ "tokens_seen": 95223808
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847542627883651,
+ "loss": 2.0831,
+ "theoretical_loss": 4.83284030140778,
+ "tokens_seen": 95289344
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847382146439319,
+ "loss": 1.894,
+ "theoretical_loss": 4.832379201739442,
+ "tokens_seen": 95354880
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847221664994985,
+ "loss": 1.9736,
+ "theoretical_loss": 4.831918507533728,
+ "tokens_seen": 95420416
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007847061183550653,
+ "loss": 2.0363,
+ "theoretical_loss": 4.831458218155989,
+ "tokens_seen": 95485952
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846900702106319,
+ "loss": 2.0462,
+ "theoretical_loss": 4.830998332973005,
+ "tokens_seen": 95551488
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846740220661987,
+ "loss": 1.8864,
+ "theoretical_loss": 4.830538851352976,
+ "tokens_seen": 95617024
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846579739217653,
+ "loss": 1.9409,
+ "theoretical_loss": 4.830079772665529,
+ "tokens_seen": 95682560
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846419257773321,
+ "loss": 1.9883,
+ "theoretical_loss": 4.829621096281702,
+ "tokens_seen": 95748096
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846258776328987,
+ "loss": 1.8937,
+ "theoretical_loss": 4.8291628215739415,
+ "tokens_seen": 95813632
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007846098294884655,
+ "loss": 1.9911,
+ "theoretical_loss": 4.828704947916108,
+ "tokens_seen": 95879168
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845937813440321,
+ "loss": 1.9991,
+ "theoretical_loss": 4.82824747468346,
+ "tokens_seen": 95944704
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845777331995989,
+ "loss": 1.9115,
+ "theoretical_loss": 4.827790401252658,
+ "tokens_seen": 96010240
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845616850551655,
+ "loss": 1.9385,
+ "theoretical_loss": 4.827333727001756,
+ "tokens_seen": 96075776
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845456369107323,
+ "loss": 1.9522,
+ "theoretical_loss": 4.8268774513101995,
+ "tokens_seen": 96141312
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845295887662989,
+ "loss": 1.8942,
+ "theoretical_loss": 4.82642157355882,
+ "tokens_seen": 96206848
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007845135406218657,
+ "loss": 1.9496,
+ "theoretical_loss": 4.825966093129834,
+ "tokens_seen": 96272384
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844974924774323,
+ "loss": 1.9805,
+ "theoretical_loss": 4.825511009406833,
+ "tokens_seen": 96337920
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844814443329991,
+ "loss": 1.9984,
+ "theoretical_loss": 4.825056321774788,
+ "tokens_seen": 96403456
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844653961885657,
+ "loss": 1.9623,
+ "theoretical_loss": 4.8246020296200385,
+ "tokens_seen": 96468992
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844493480441325,
+ "loss": 1.9433,
+ "theoretical_loss": 4.82414813233029,
+ "tokens_seen": 96534528
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844332998996991,
+ "loss": 2.0465,
+ "theoretical_loss": 4.823694629294609,
+ "tokens_seen": 96600064
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844172517552659,
+ "loss": 2.0518,
+ "theoretical_loss": 4.823241519903428,
+ "tokens_seen": 96665600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007844012036108325,
+ "loss": 1.9394,
+ "theoretical_loss": 4.8227888035485265,
+ "tokens_seen": 96731136
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007843851554663993,
+ "loss": 1.8898,
+ "theoretical_loss": 4.82233647962304,
+ "tokens_seen": 96796672
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000784369107321966,
+ "loss": 1.9072,
+ "theoretical_loss": 4.821884547521449,
+ "tokens_seen": 96862208
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007843530591775327,
+ "loss": 2.0767,
+ "theoretical_loss": 4.821433006639578,
+ "tokens_seen": 96927744
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007843370110330994,
+ "loss": 1.9428,
+ "theoretical_loss": 4.820981856374589,
+ "tokens_seen": 96993280
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007843209628886661,
+ "loss": 1.9907,
+ "theoretical_loss": 4.8205310961249825,
+ "tokens_seen": 97058816
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007843049147442328,
+ "loss": 1.9943,
+ "theoretical_loss": 4.820080725290589,
+ "tokens_seen": 97124352
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842888665997995,
+ "loss": 1.9588,
+ "theoretical_loss": 4.819630743272566,
+ "tokens_seen": 97189888
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842728184553662,
+ "loss": 2.0534,
+ "theoretical_loss": 4.819181149473396,
+ "tokens_seen": 97255424
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842567703109329,
+ "loss": 2.0432,
+ "theoretical_loss": 4.81873194329688,
+ "tokens_seen": 97320960
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842407221664996,
+ "loss": 2.0024,
+ "theoretical_loss": 4.81828312414814,
+ "tokens_seen": 97386496
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842246740220663,
+ "loss": 2.0566,
+ "theoretical_loss": 4.817834691433602,
+ "tokens_seen": 97452032
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007842086258776329,
+ "loss": 1.9204,
+ "theoretical_loss": 4.81738664456101,
+ "tokens_seen": 97517568
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841925777331997,
+ "loss": 1.8443,
+ "theoretical_loss": 4.816938982939407,
+ "tokens_seen": 97583104
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841765295887663,
+ "loss": 2.0024,
+ "theoretical_loss": 4.816491705979138,
+ "tokens_seen": 97648640
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841604814443331,
+ "loss": 2.0781,
+ "theoretical_loss": 4.816044813091848,
+ "tokens_seen": 97714176
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841444332998997,
+ "loss": 2.0516,
+ "theoretical_loss": 4.815598303690473,
+ "tokens_seen": 97779712
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841283851554665,
+ "loss": 1.8457,
+ "theoretical_loss": 4.8151521771892405,
+ "tokens_seen": 97845248
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007841123370110331,
+ "loss": 1.9794,
+ "theoretical_loss": 4.814706433003665,
+ "tokens_seen": 97910784
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840962888665999,
+ "loss": 1.988,
+ "theoretical_loss": 4.8142610705505415,
+ "tokens_seen": 97976320
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840802407221665,
+ "loss": 1.9506,
+ "theoretical_loss": 4.813816089247945,
+ "tokens_seen": 98041856
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840641925777332,
+ "loss": 2.0677,
+ "theoretical_loss": 4.813371488515227,
+ "tokens_seen": 98107392
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840481444332999,
+ "loss": 1.8749,
+ "theoretical_loss": 4.812927267773008,
+ "tokens_seen": 98172928
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840320962888666,
+ "loss": 2.0423,
+ "theoretical_loss": 4.812483426443181,
+ "tokens_seen": 98238464
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 39774,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.065314292907715,
+ "objective/train/theoretical_loss": 4.812039963948898,
+ "objective/train/tokens_used": 118764000,
+ "theoretical_loss": 4.812039963948898,
+ "tokens_seen": 98304000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007840160481444333,
+ "loss": 1.9389,
+ "theoretical_loss": 4.812039963948898,
+ "tokens_seen": 98304000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000784,
+ "loss": 2.0512,
+ "theoretical_loss": 4.811596879714575,
+ "tokens_seen": 98369536
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839839518555667,
+ "loss": 2.0692,
+ "theoretical_loss": 4.811154173165886,
+ "tokens_seen": 98435072
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839679037111334,
+ "loss": 2.0087,
+ "theoretical_loss": 4.810711843729758,
+ "tokens_seen": 98500608
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839518555667001,
+ "loss": 1.9157,
+ "theoretical_loss": 4.810269890834364,
+ "tokens_seen": 98566144
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839358074222668,
+ "loss": 1.8269,
+ "theoretical_loss": 4.809828313909129,
+ "tokens_seen": 98631680
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839197592778335,
+ "loss": 1.9956,
+ "theoretical_loss": 4.809387112384721,
+ "tokens_seen": 98697216
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007839037111334002,
+ "loss": 1.9715,
+ "theoretical_loss": 4.808946285693043,
+ "tokens_seen": 98762752
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007838876629889669,
+ "loss": 1.8838,
+ "theoretical_loss": 4.808505833267237,
+ "tokens_seen": 98828288
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007838716148445336,
+ "loss": 1.8904,
+ "theoretical_loss": 4.808065754541676,
+ "tokens_seen": 98893824
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007838555667001003,
+ "loss": 1.89,
+ "theoretical_loss": 4.807626048951965,
+ "tokens_seen": 98959360
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783839518555667,
+ "loss": 1.9809,
+ "theoretical_loss": 4.807186715934931,
+ "tokens_seen": 99024896
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007838234704112337,
+ "loss": 1.8864,
+ "theoretical_loss": 4.806747754928622,
+ "tokens_seen": 99090432
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007838074222668004,
+ "loss": 1.8412,
+ "theoretical_loss": 4.806309165372311,
+ "tokens_seen": 99155968
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837913741223671,
+ "loss": 1.8538,
+ "theoretical_loss": 4.805870946706479,
+ "tokens_seen": 99221504
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837753259779338,
+ "loss": 2.0107,
+ "theoretical_loss": 4.805433098372822,
+ "tokens_seen": 99287040
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837592778335005,
+ "loss": 2.0084,
+ "theoretical_loss": 4.8049956198142425,
+ "tokens_seen": 99352576
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837432296890672,
+ "loss": 1.9323,
+ "theoretical_loss": 4.804558510474852,
+ "tokens_seen": 99418112
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837271815446339,
+ "loss": 1.9652,
+ "theoretical_loss": 4.804121769799959,
+ "tokens_seen": 99483648
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007837111334002006,
+ "loss": 1.8399,
+ "theoretical_loss": 4.8036853972360705,
+ "tokens_seen": 99549184
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007836950852557673,
+ "loss": 1.9356,
+ "theoretical_loss": 4.80324939223089,
+ "tokens_seen": 99614720
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783679037111334,
+ "loss": 2.0503,
+ "theoretical_loss": 4.802813754233311,
+ "tokens_seen": 99680256
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007836629889669007,
+ "loss": 1.9646,
+ "theoretical_loss": 4.802378482693417,
+ "tokens_seen": 99745792
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007836469408224674,
+ "loss": 1.9134,
+ "theoretical_loss": 4.801943577062472,
+ "tokens_seen": 99811328
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007836308926780341,
+ "loss": 1.9831,
+ "theoretical_loss": 4.801509036792925,
+ "tokens_seen": 99876864
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007836148445336008,
+ "loss": 2.1101,
+ "theoretical_loss": 4.801074861338401,
+ "tokens_seen": 99942400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007835987963891675,
+ "loss": 1.9731,
+ "theoretical_loss": 4.800641050153703,
+ "tokens_seen": 100007936
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007835827482447342,
+ "loss": 2.0708,
+ "theoretical_loss": 4.800207602694801,
+ "tokens_seen": 100073472
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783566700100301,
+ "loss": 1.9291,
+ "theoretical_loss": 4.7997745184188325,
+ "tokens_seen": 100139008
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007835506519558676,
+ "loss": 2.0162,
+ "theoretical_loss": 4.799341796784106,
+ "tokens_seen": 100204544
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007835346038114343,
+ "loss": 2.0161,
+ "theoretical_loss": 4.798909437250086,
+ "tokens_seen": 100270080
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783518555667001,
+ "loss": 1.872,
+ "theoretical_loss": 4.798477439277397,
+ "tokens_seen": 100335616
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007835025075225677,
+ "loss": 1.9359,
+ "theoretical_loss": 4.7980458023278185,
+ "tokens_seen": 100401152
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007834864593781344,
+ "loss": 2.046,
+ "theoretical_loss": 4.797614525864282,
+ "tokens_seen": 100466688
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007834704112337011,
+ "loss": 2.0471,
+ "theoretical_loss": 4.797183609350869,
+ "tokens_seen": 100532224
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007834543630892678,
+ "loss": 1.9752,
+ "theoretical_loss": 4.796753052252802,
+ "tokens_seen": 100597760
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007834383149448346,
+ "loss": 1.8555,
+ "theoretical_loss": 4.79632285403645,
+ "tokens_seen": 100663296
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007834222668004012,
+ "loss": 1.9888,
+ "theoretical_loss": 4.795893014169323,
+ "tokens_seen": 100728832
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783406218655968,
+ "loss": 1.9027,
+ "theoretical_loss": 4.795463532120058,
+ "tokens_seen": 100794368
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007833901705115346,
+ "loss": 1.9115,
+ "theoretical_loss": 4.795034407358435,
+ "tokens_seen": 100859904
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007833741223671014,
+ "loss": 1.9345,
+ "theoretical_loss": 4.794605639355357,
+ "tokens_seen": 100925440
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783358074222668,
+ "loss": 1.9462,
+ "theoretical_loss": 4.794177227582855,
+ "tokens_seen": 100990976
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007833420260782348,
+ "loss": 1.9011,
+ "theoretical_loss": 4.793749171514085,
+ "tokens_seen": 101056512
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007833259779338014,
+ "loss": 1.8762,
+ "theoretical_loss": 4.793321470623321,
+ "tokens_seen": 101122048
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007833099297893682,
+ "loss": 1.9244,
+ "theoretical_loss": 4.792894124385955,
+ "tokens_seen": 101187584
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007832938816449348,
+ "loss": 1.9741,
+ "theoretical_loss": 4.792467132278494,
+ "tokens_seen": 101253120
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007832778335005016,
+ "loss": 1.9348,
+ "theoretical_loss": 4.792040493778553,
+ "tokens_seen": 101318656
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007832617853560682,
+ "loss": 2.0157,
+ "theoretical_loss": 4.791614208364859,
+ "tokens_seen": 101384192
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783245737211635,
+ "loss": 1.9452,
+ "theoretical_loss": 4.791188275517239,
+ "tokens_seen": 101449728
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007832296890672016,
+ "loss": 1.9476,
+ "theoretical_loss": 4.7907626947166255,
+ "tokens_seen": 101515264
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 40787,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9380860328674316,
+ "objective/train/theoretical_loss": 4.790337465445049,
+ "objective/train/tokens_used": 122040800,
+ "theoretical_loss": 4.790337465445049,
+ "tokens_seen": 101580800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007832136409227684,
+ "loss": 1.9874,
+ "theoretical_loss": 4.790337465445049,
+ "tokens_seen": 101580800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783197592778335,
+ "loss": 2.06,
+ "theoretical_loss": 4.7899125871856345,
+ "tokens_seen": 101646336
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831815446339018,
+ "loss": 1.8105,
+ "theoretical_loss": 4.789488059422599,
+ "tokens_seen": 101711872
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831654964894685,
+ "loss": 2.0042,
+ "theoretical_loss": 4.789063881641251,
+ "tokens_seen": 101777408
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831494483450352,
+ "loss": 2.0481,
+ "theoretical_loss": 4.7886400533279865,
+ "tokens_seen": 101842944
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831334002006019,
+ "loss": 1.9965,
+ "theoretical_loss": 4.788216573970281,
+ "tokens_seen": 101908480
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831173520561686,
+ "loss": 1.9324,
+ "theoretical_loss": 4.787793443056694,
+ "tokens_seen": 101974016
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007831013039117353,
+ "loss": 1.8745,
+ "theoretical_loss": 4.787370660076862,
+ "tokens_seen": 102039552
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000783085255767302,
+ "loss": 2.0481,
+ "theoretical_loss": 4.786948224521495,
+ "tokens_seen": 102105088
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007830692076228687,
+ "loss": 1.9234,
+ "theoretical_loss": 4.786526135882378,
+ "tokens_seen": 102170624
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007830531594784354,
+ "loss": 2.0611,
+ "theoretical_loss": 4.786104393652359,
+ "tokens_seen": 102236160
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007830371113340021,
+ "loss": 1.9584,
+ "theoretical_loss": 4.785682997325358,
+ "tokens_seen": 102301696
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007830210631895688,
+ "loss": 1.9192,
+ "theoretical_loss": 4.7852619463963535,
+ "tokens_seen": 102367232
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007830050150451355,
+ "loss": 1.8914,
+ "theoretical_loss": 4.784841240361389,
+ "tokens_seen": 102432768
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007829889669007022,
+ "loss": 1.9203,
+ "theoretical_loss": 4.784420878717558,
+ "tokens_seen": 102498304
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007829729187562689,
+ "loss": 1.9652,
+ "theoretical_loss": 4.7840008609630145,
+ "tokens_seen": 102563840
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007829568706118356,
+ "loss": 1.9889,
+ "theoretical_loss": 4.783581186596962,
+ "tokens_seen": 102629376
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007829408224674023,
+ "loss": 1.8846,
+ "theoretical_loss": 4.78316185511965,
+ "tokens_seen": 102694912
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782924774322969,
+ "loss": 2.0007,
+ "theoretical_loss": 4.782742866032379,
+ "tokens_seen": 102760448
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007829087261785357,
+ "loss": 1.8957,
+ "theoretical_loss": 4.782324218837486,
+ "tokens_seen": 102825984
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828926780341024,
+ "loss": 1.9462,
+ "theoretical_loss": 4.781905913038351,
+ "tokens_seen": 102891520
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828766298896691,
+ "loss": 1.9478,
+ "theoretical_loss": 4.781487948139393,
+ "tokens_seen": 102957056
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828605817452358,
+ "loss": 1.9648,
+ "theoretical_loss": 4.7810703236460625,
+ "tokens_seen": 103022592
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828445336008025,
+ "loss": 1.9738,
+ "theoretical_loss": 4.780653039064843,
+ "tokens_seen": 103088128
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828284854563692,
+ "loss": 1.9539,
+ "theoretical_loss": 4.780236093903245,
+ "tokens_seen": 103153664
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007828124373119358,
+ "loss": 1.8898,
+ "theoretical_loss": 4.779819487669805,
+ "tokens_seen": 103219200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007827963891675026,
+ "loss": 1.8978,
+ "theoretical_loss": 4.779403219874085,
+ "tokens_seen": 103284736
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007827803410230692,
+ "loss": 1.97,
+ "theoretical_loss": 4.778987290026665,
+ "tokens_seen": 103350272
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782764292878636,
+ "loss": 1.9551,
+ "theoretical_loss": 4.778571697639142,
+ "tokens_seen": 103415808
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007827482447342026,
+ "loss": 1.8563,
+ "theoretical_loss": 4.778156442224132,
+ "tokens_seen": 103481344
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007827321965897694,
+ "loss": 2.0819,
+ "theoretical_loss": 4.7777415232952585,
+ "tokens_seen": 103546880
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782716148445336,
+ "loss": 1.9497,
+ "theoretical_loss": 4.777326940367155,
+ "tokens_seen": 103612416
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007827001003009028,
+ "loss": 2.0545,
+ "theoretical_loss": 4.776912692955463,
+ "tokens_seen": 103677952
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826840521564694,
+ "loss": 1.9213,
+ "theoretical_loss": 4.776498780576826,
+ "tokens_seen": 103743488
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826680040120361,
+ "loss": 1.9726,
+ "theoretical_loss": 4.776085202748893,
+ "tokens_seen": 103809024
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826519558676028,
+ "loss": 1.8898,
+ "theoretical_loss": 4.775671958990305,
+ "tokens_seen": 103874560
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826359077231695,
+ "loss": 2.004,
+ "theoretical_loss": 4.775259048820702,
+ "tokens_seen": 103940096
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826198595787362,
+ "loss": 1.9216,
+ "theoretical_loss": 4.7748464717607195,
+ "tokens_seen": 104005632
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007826038114343029,
+ "loss": 1.8736,
+ "theoretical_loss": 4.774434227331979,
+ "tokens_seen": 104071168
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007825877632898696,
+ "loss": 1.9748,
+ "theoretical_loss": 4.774022315057092,
+ "tokens_seen": 104136704
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007825717151454363,
+ "loss": 1.9511,
+ "theoretical_loss": 4.773610734459654,
+ "tokens_seen": 104202240
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782555667001003,
+ "loss": 1.8595,
+ "theoretical_loss": 4.773199485064243,
+ "tokens_seen": 104267776
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007825396188565697,
+ "loss": 1.8144,
+ "theoretical_loss": 4.772788566396418,
+ "tokens_seen": 104333312
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007825235707121364,
+ "loss": 1.9711,
+ "theoretical_loss": 4.772377977982714,
+ "tokens_seen": 104398848
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007825075225677031,
+ "loss": 1.8884,
+ "theoretical_loss": 4.771967719350641,
+ "tokens_seen": 104464384
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824914744232698,
+ "loss": 1.9437,
+ "theoretical_loss": 4.77155779002868,
+ "tokens_seen": 104529920
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824754262788365,
+ "loss": 1.8921,
+ "theoretical_loss": 4.771148189546282,
+ "tokens_seen": 104595456
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824593781344032,
+ "loss": 2.0112,
+ "theoretical_loss": 4.770738917433864,
+ "tokens_seen": 104660992
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824433299899699,
+ "loss": 1.9376,
+ "theoretical_loss": 4.770329973222809,
+ "tokens_seen": 104726528
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824272818455366,
+ "loss": 1.966,
+ "theoretical_loss": 4.769921356445458,
+ "tokens_seen": 104792064
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 42078,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.965983510017395,
+ "objective/train/theoretical_loss": 4.769513066635114,
+ "objective/train/tokens_used": 125317600,
+ "theoretical_loss": 4.769513066635114,
+ "tokens_seen": 104857600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007824112337011033,
+ "loss": 1.9,
+ "theoretical_loss": 4.769513066635114,
+ "tokens_seen": 104857600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.00078239518555667,
+ "loss": 1.9222,
+ "theoretical_loss": 4.769105103326036,
+ "tokens_seen": 104923136
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007823791374122367,
+ "loss": 2.0369,
+ "theoretical_loss": 4.768697466053435,
+ "tokens_seen": 104988672
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007823630892678034,
+ "loss": 1.9214,
+ "theoretical_loss": 4.768290154353474,
+ "tokens_seen": 105054208
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007823470411233701,
+ "loss": 1.9252,
+ "theoretical_loss": 4.767883167763267,
+ "tokens_seen": 105119744
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007823309929789368,
+ "loss": 1.9184,
+ "theoretical_loss": 4.76747650582087,
+ "tokens_seen": 105185280
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007823149448345035,
+ "loss": 1.8375,
+ "theoretical_loss": 4.767070168065285,
+ "tokens_seen": 105250816
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822988966900702,
+ "loss": 1.8323,
+ "theoretical_loss": 4.766664154036456,
+ "tokens_seen": 105316352
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822828485456369,
+ "loss": 1.7604,
+ "theoretical_loss": 4.766258463275265,
+ "tokens_seen": 105381888
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822668004012036,
+ "loss": 1.9705,
+ "theoretical_loss": 4.7658530953235285,
+ "tokens_seen": 105447424
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822507522567703,
+ "loss": 1.9077,
+ "theoretical_loss": 4.765448049723998,
+ "tokens_seen": 105512960
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782234704112337,
+ "loss": 1.7903,
+ "theoretical_loss": 4.765043326020358,
+ "tokens_seen": 105578496
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822186559679037,
+ "loss": 1.9913,
+ "theoretical_loss": 4.764638923757217,
+ "tokens_seen": 105644032
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007822026078234704,
+ "loss": 1.8003,
+ "theoretical_loss": 4.764234842480114,
+ "tokens_seen": 105709568
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821865596790371,
+ "loss": 1.8746,
+ "theoretical_loss": 4.7638310817355105,
+ "tokens_seen": 105775104
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821705115346038,
+ "loss": 1.9363,
+ "theoretical_loss": 4.763427641070789,
+ "tokens_seen": 105840640
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821544633901705,
+ "loss": 1.8989,
+ "theoretical_loss": 4.76302452003425,
+ "tokens_seen": 105906176
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821384152457372,
+ "loss": 1.9954,
+ "theoretical_loss": 4.7626217181751125,
+ "tokens_seen": 105971712
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821223671013039,
+ "loss": 1.9583,
+ "theoretical_loss": 4.762219235043508,
+ "tokens_seen": 106037248
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007821063189568706,
+ "loss": 1.9582,
+ "theoretical_loss": 4.761817070190481,
+ "tokens_seen": 106102784
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007820902708124373,
+ "loss": 2.018,
+ "theoretical_loss": 4.761415223167982,
+ "tokens_seen": 106168320
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000782074222668004,
+ "loss": 2.0643,
+ "theoretical_loss": 4.761013693528874,
+ "tokens_seen": 106233856
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007820581745235707,
+ "loss": 1.9534,
+ "theoretical_loss": 4.760612480826917,
+ "tokens_seen": 106299392
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007820421263791375,
+ "loss": 1.9656,
+ "theoretical_loss": 4.76021158461678,
+ "tokens_seen": 106364928
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007820260782347041,
+ "loss": 1.9398,
+ "theoretical_loss": 4.759811004454027,
+ "tokens_seen": 106430464
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007820100300902709,
+ "loss": 1.9891,
+ "theoretical_loss": 4.759410739895122,
+ "tokens_seen": 106496000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007819939819458375,
+ "loss": 1.9332,
+ "theoretical_loss": 4.759010790497422,
+ "tokens_seen": 106561536
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007819779338014043,
+ "loss": 1.9756,
+ "theoretical_loss": 4.758611155819178,
+ "tokens_seen": 106627072
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781961885656971,
+ "loss": 1.9532,
+ "theoretical_loss": 4.758211835419531,
+ "tokens_seen": 106692608
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007819458375125377,
+ "loss": 1.8169,
+ "theoretical_loss": 4.757812828858508,
+ "tokens_seen": 106758144
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007819297893681044,
+ "loss": 1.8261,
+ "theoretical_loss": 4.757414135697024,
+ "tokens_seen": 106823680
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007819137412236711,
+ "loss": 1.9218,
+ "theoretical_loss": 4.757015755496877,
+ "tokens_seen": 106889216
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818976930792378,
+ "loss": 1.7847,
+ "theoretical_loss": 4.756617687820743,
+ "tokens_seen": 106954752
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818816449348045,
+ "loss": 1.9165,
+ "theoretical_loss": 4.756219932232181,
+ "tokens_seen": 107020288
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818655967903712,
+ "loss": 1.9031,
+ "theoretical_loss": 4.755822488295621,
+ "tokens_seen": 107085824
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818495486459379,
+ "loss": 2.0325,
+ "theoretical_loss": 4.755425355576373,
+ "tokens_seen": 107151360
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818335005015046,
+ "loss": 1.9744,
+ "theoretical_loss": 4.755028533640614,
+ "tokens_seen": 107216896
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007818174523570713,
+ "loss": 1.9273,
+ "theoretical_loss": 4.754632022055392,
+ "tokens_seen": 107282432
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781801404212638,
+ "loss": 1.9457,
+ "theoretical_loss": 4.754235820388624,
+ "tokens_seen": 107347968
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817853560682047,
+ "loss": 1.7709,
+ "theoretical_loss": 4.7538399282090875,
+ "tokens_seen": 107413504
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817693079237714,
+ "loss": 1.9709,
+ "theoretical_loss": 4.753444345086428,
+ "tokens_seen": 107479040
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817532597793381,
+ "loss": 1.8569,
+ "theoretical_loss": 4.753049070591146,
+ "tokens_seen": 107544576
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817372116349048,
+ "loss": 1.8751,
+ "theoretical_loss": 4.752654104294605,
+ "tokens_seen": 107610112
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817211634904715,
+ "loss": 1.9358,
+ "theoretical_loss": 4.752259445769022,
+ "tokens_seen": 107675648
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007817051153460382,
+ "loss": 1.8664,
+ "theoretical_loss": 4.751865094587465,
+ "tokens_seen": 107741184
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007816890672016049,
+ "loss": 1.8228,
+ "theoretical_loss": 4.75147105032386,
+ "tokens_seen": 107806720
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007816730190571716,
+ "loss": 1.8293,
+ "theoretical_loss": 4.751077312552976,
+ "tokens_seen": 107872256
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007816569709127383,
+ "loss": 1.8754,
+ "theoretical_loss": 4.750683880850433,
+ "tokens_seen": 107937792
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781640922768305,
+ "loss": 1.9043,
+ "theoretical_loss": 4.750290754792694,
+ "tokens_seen": 108003328
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007816248746238717,
+ "loss": 1.8329,
+ "theoretical_loss": 4.749897933957065,
+ "tokens_seen": 108068864
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 42949,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9812370538711548,
+ "objective/train/theoretical_loss": 4.749505417921692,
+ "objective/train/tokens_used": 128594400,
+ "theoretical_loss": 4.749505417921692,
+ "tokens_seen": 108134400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007816088264794384,
+ "loss": 1.989,
+ "theoretical_loss": 4.749505417921692,
+ "tokens_seen": 108134400
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815927783350051,
+ "loss": 1.9188,
+ "theoretical_loss": 4.7491132062655605,
+ "tokens_seen": 108199936
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815767301905718,
+ "loss": 1.8283,
+ "theoretical_loss": 4.748721298568491,
+ "tokens_seen": 108265472
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815606820461385,
+ "loss": 2.0035,
+ "theoretical_loss": 4.748329694411137,
+ "tokens_seen": 108331008
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815446339017052,
+ "loss": 1.9027,
+ "theoretical_loss": 4.747938393374987,
+ "tokens_seen": 108396544
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815285857572719,
+ "loss": 1.8085,
+ "theoretical_loss": 4.747547395042355,
+ "tokens_seen": 108462080
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007815125376128386,
+ "loss": 1.8566,
+ "theoretical_loss": 4.747156698996388,
+ "tokens_seen": 108527616
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814964894684053,
+ "loss": 1.8354,
+ "theoretical_loss": 4.746766304821053,
+ "tokens_seen": 108593152
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781480441323972,
+ "loss": 1.98,
+ "theoretical_loss": 4.746376212101142,
+ "tokens_seen": 108658688
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814643931795387,
+ "loss": 1.98,
+ "theoretical_loss": 4.745986420422267,
+ "tokens_seen": 108724224
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814483450351054,
+ "loss": 1.8918,
+ "theoretical_loss": 4.745596929370863,
+ "tokens_seen": 108789760
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814322968906721,
+ "loss": 1.9303,
+ "theoretical_loss": 4.745207738534177,
+ "tokens_seen": 108855296
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814162487462388,
+ "loss": 1.8991,
+ "theoretical_loss": 4.744818847500274,
+ "tokens_seen": 108920832
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007814002006018055,
+ "loss": 1.9216,
+ "theoretical_loss": 4.744430255858029,
+ "tokens_seen": 108986368
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813841524573721,
+ "loss": 1.8368,
+ "theoretical_loss": 4.744041963197132,
+ "tokens_seen": 109051904
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813681043129389,
+ "loss": 1.9503,
+ "theoretical_loss": 4.743653969108076,
+ "tokens_seen": 109117440
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813520561685055,
+ "loss": 1.8052,
+ "theoretical_loss": 4.743266273182163,
+ "tokens_seen": 109182976
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813360080240723,
+ "loss": 1.8667,
+ "theoretical_loss": 4.742878875011501,
+ "tokens_seen": 109248512
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813199598796389,
+ "loss": 1.8948,
+ "theoretical_loss": 4.742491774188997,
+ "tokens_seen": 109314048
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007813039117352057,
+ "loss": 2.0147,
+ "theoretical_loss": 4.742104970308359,
+ "tokens_seen": 109379584
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007812878635907723,
+ "loss": 1.8349,
+ "theoretical_loss": 4.741718462964095,
+ "tokens_seen": 109445120
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781271815446339,
+ "loss": 2.0293,
+ "theoretical_loss": 4.741332251751509,
+ "tokens_seen": 109510656
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007812557673019057,
+ "loss": 1.9101,
+ "theoretical_loss": 4.740946336266697,
+ "tokens_seen": 109576192
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007812397191574724,
+ "loss": 1.9097,
+ "theoretical_loss": 4.740560716106549,
+ "tokens_seen": 109641728
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007812236710130391,
+ "loss": 1.9329,
+ "theoretical_loss": 4.740175390868742,
+ "tokens_seen": 109707264
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007812076228686058,
+ "loss": 1.8663,
+ "theoretical_loss": 4.739790360151744,
+ "tokens_seen": 109772800
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007811915747241725,
+ "loss": 1.9846,
+ "theoretical_loss": 4.73940562355481,
+ "tokens_seen": 109838336
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007811755265797392,
+ "loss": 1.8642,
+ "theoretical_loss": 4.739021180677976,
+ "tokens_seen": 109903872
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007811594784353059,
+ "loss": 1.8875,
+ "theoretical_loss": 4.73863703112206,
+ "tokens_seen": 109969408
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007811434302908726,
+ "loss": 1.8295,
+ "theoretical_loss": 4.738253174488662,
+ "tokens_seen": 110034944
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007811273821464393,
+ "loss": 1.8563,
+ "theoretical_loss": 4.73786961038016,
+ "tokens_seen": 110100480
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000781111334002006,
+ "loss": 1.9474,
+ "theoretical_loss": 4.737486338399704,
+ "tokens_seen": 110166016
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810952858575727,
+ "loss": 2.0176,
+ "theoretical_loss": 4.737103358151225,
+ "tokens_seen": 110231552
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810792377131394,
+ "loss": 1.8906,
+ "theoretical_loss": 4.7367206692394195,
+ "tokens_seen": 110297088
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810631895687061,
+ "loss": 1.8809,
+ "theoretical_loss": 4.736338271269759,
+ "tokens_seen": 110362624
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810471414242728,
+ "loss": 1.931,
+ "theoretical_loss": 4.735956163848478,
+ "tokens_seen": 110428160
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810310932798395,
+ "loss": 1.968,
+ "theoretical_loss": 4.735574346582582,
+ "tokens_seen": 110493696
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007810150451354062,
+ "loss": 1.9077,
+ "theoretical_loss": 4.735192819079838,
+ "tokens_seen": 110559232
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809989969909729,
+ "loss": 1.8817,
+ "theoretical_loss": 4.734811580948779,
+ "tokens_seen": 110624768
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809829488465396,
+ "loss": 1.8448,
+ "theoretical_loss": 4.734430631798692,
+ "tokens_seen": 110690304
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809669007021063,
+ "loss": 1.9445,
+ "theoretical_loss": 4.734049971239628,
+ "tokens_seen": 110755840
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780950852557673,
+ "loss": 1.9162,
+ "theoretical_loss": 4.7336695988823925,
+ "tokens_seen": 110821376
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809348044132397,
+ "loss": 1.9709,
+ "theoretical_loss": 4.733289514338546,
+ "tokens_seen": 110886912
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809187562688064,
+ "loss": 1.8657,
+ "theoretical_loss": 4.7329097172204,
+ "tokens_seen": 110952448
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007809027081243731,
+ "loss": 1.7885,
+ "theoretical_loss": 4.73253020714102,
+ "tokens_seen": 111017984
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808866599799398,
+ "loss": 1.9659,
+ "theoretical_loss": 4.732150983714217,
+ "tokens_seen": 111083520
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808706118355065,
+ "loss": 1.8298,
+ "theoretical_loss": 4.731772046554552,
+ "tokens_seen": 111149056
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808545636910732,
+ "loss": 1.9417,
+ "theoretical_loss": 4.731393395277331,
+ "tokens_seen": 111214592
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808385155466399,
+ "loss": 1.8618,
+ "theoretical_loss": 4.731015029498598,
+ "tokens_seen": 111280128
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808224674022066,
+ "loss": 1.9002,
+ "theoretical_loss": 4.730636948835148,
+ "tokens_seen": 111345664
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 44130,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9857747554779053,
+ "objective/train/theoretical_loss": 4.730259152904507,
+ "objective/train/tokens_used": 131871200,
+ "theoretical_loss": 4.730259152904507,
+ "tokens_seen": 111411200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007808064192577733,
+ "loss": 1.9456,
+ "theoretical_loss": 4.730259152904507,
+ "tokens_seen": 111411200
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.00078079037111334,
+ "loss": 1.7301,
+ "theoretical_loss": 4.7298816413249405,
+ "tokens_seen": 111476736
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007807743229689067,
+ "loss": 1.7904,
+ "theoretical_loss": 4.729504413715454,
+ "tokens_seen": 111542272
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007807582748244735,
+ "loss": 1.8232,
+ "theoretical_loss": 4.729127469695783,
+ "tokens_seen": 111607808
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007807422266800401,
+ "loss": 1.8824,
+ "theoretical_loss": 4.728750808886394,
+ "tokens_seen": 111673344
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007807261785356069,
+ "loss": 1.9541,
+ "theoretical_loss": 4.72837443090849,
+ "tokens_seen": 111738880
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007807101303911736,
+ "loss": 1.9042,
+ "theoretical_loss": 4.727998335383996,
+ "tokens_seen": 111804416
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007806940822467403,
+ "loss": 1.8814,
+ "theoretical_loss": 4.727622521935563,
+ "tokens_seen": 111869952
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780678034102307,
+ "loss": 1.8453,
+ "theoretical_loss": 4.7272469901865755,
+ "tokens_seen": 111935488
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007806619859578737,
+ "loss": 1.8953,
+ "theoretical_loss": 4.726871739761132,
+ "tokens_seen": 112001024
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007806459378134404,
+ "loss": 1.8651,
+ "theoretical_loss": 4.726496770284056,
+ "tokens_seen": 112066560
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007806298896690071,
+ "loss": 1.9383,
+ "theoretical_loss": 4.726122081380891,
+ "tokens_seen": 112132096
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007806138415245738,
+ "loss": 1.8434,
+ "theoretical_loss": 4.725747672677894,
+ "tokens_seen": 112197632
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805977933801405,
+ "loss": 1.9197,
+ "theoretical_loss": 4.725373543802043,
+ "tokens_seen": 112263168
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805817452357072,
+ "loss": 1.9461,
+ "theoretical_loss": 4.724999694381027,
+ "tokens_seen": 112328704
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805656970912739,
+ "loss": 1.858,
+ "theoretical_loss": 4.724626124043247,
+ "tokens_seen": 112394240
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805496489468406,
+ "loss": 1.9208,
+ "theoretical_loss": 4.724252832417817,
+ "tokens_seen": 112459776
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805336008024073,
+ "loss": 1.7769,
+ "theoretical_loss": 4.723879819134558,
+ "tokens_seen": 112525312
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780517552657974,
+ "loss": 1.807,
+ "theoretical_loss": 4.7235070838239945,
+ "tokens_seen": 112590848
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007805015045135407,
+ "loss": 1.9201,
+ "theoretical_loss": 4.723134626117363,
+ "tokens_seen": 112656384
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804854563691074,
+ "loss": 1.8909,
+ "theoretical_loss": 4.722762445646598,
+ "tokens_seen": 112721920
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804694082246741,
+ "loss": 1.9641,
+ "theoretical_loss": 4.722390542044339,
+ "tokens_seen": 112787456
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804533600802408,
+ "loss": 1.8542,
+ "theoretical_loss": 4.722018914943924,
+ "tokens_seen": 112852992
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804373119358075,
+ "loss": 1.7947,
+ "theoretical_loss": 4.721647563979385,
+ "tokens_seen": 112918528
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804212637913742,
+ "loss": 1.7628,
+ "theoretical_loss": 4.72127648878546,
+ "tokens_seen": 112984064
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007804052156469409,
+ "loss": 2.0437,
+ "theoretical_loss": 4.720905688997572,
+ "tokens_seen": 113049600
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007803891675025076,
+ "loss": 1.8696,
+ "theoretical_loss": 4.720535164251842,
+ "tokens_seen": 113115136
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007803731193580743,
+ "loss": 2.0169,
+ "theoretical_loss": 4.720164914185084,
+ "tokens_seen": 113180672
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780357071213641,
+ "loss": 1.9562,
+ "theoretical_loss": 4.719794938434794,
+ "tokens_seen": 113246208
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007803410230692077,
+ "loss": 1.8761,
+ "theoretical_loss": 4.7194252366391645,
+ "tokens_seen": 113311744
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007803249749247744,
+ "loss": 1.8482,
+ "theoretical_loss": 4.719055808437068,
+ "tokens_seen": 113377280
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007803089267803411,
+ "loss": 1.8214,
+ "theoretical_loss": 4.718686653468065,
+ "tokens_seen": 113442816
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802928786359078,
+ "loss": 1.89,
+ "theoretical_loss": 4.718317771372398,
+ "tokens_seen": 113508352
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802768304914745,
+ "loss": 2.0138,
+ "theoretical_loss": 4.7179491617909886,
+ "tokens_seen": 113573888
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802607823470412,
+ "loss": 1.8962,
+ "theoretical_loss": 4.717580824365439,
+ "tokens_seen": 113639424
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802447342026079,
+ "loss": 1.9767,
+ "theoretical_loss": 4.71721275873803,
+ "tokens_seen": 113704960
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802286860581746,
+ "loss": 1.8079,
+ "theoretical_loss": 4.716844964551717,
+ "tokens_seen": 113770496
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007802126379137413,
+ "loss": 1.9601,
+ "theoretical_loss": 4.716477441450131,
+ "tokens_seen": 113836032
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780196589769308,
+ "loss": 1.8804,
+ "theoretical_loss": 4.716110189077575,
+ "tokens_seen": 113901568
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801805416248747,
+ "loss": 1.9204,
+ "theoretical_loss": 4.715743207079022,
+ "tokens_seen": 113967104
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801644934804414,
+ "loss": 1.8374,
+ "theoretical_loss": 4.715376495100118,
+ "tokens_seen": 114032640
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801484453360081,
+ "loss": 1.9237,
+ "theoretical_loss": 4.7150100527871714,
+ "tokens_seen": 114098176
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801323971915748,
+ "loss": 1.8361,
+ "theoretical_loss": 4.714643879787161,
+ "tokens_seen": 114163712
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801163490471415,
+ "loss": 1.9508,
+ "theoretical_loss": 4.714277975747729,
+ "tokens_seen": 114229248
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007801003009027082,
+ "loss": 1.8377,
+ "theoretical_loss": 4.7139123403171785,
+ "tokens_seen": 114294784
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007800842527582749,
+ "loss": 1.771,
+ "theoretical_loss": 4.713546973144476,
+ "tokens_seen": 114360320
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007800682046138416,
+ "loss": 1.7296,
+ "theoretical_loss": 4.713181873879248,
+ "tokens_seen": 114425856
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007800521564694083,
+ "loss": 1.9301,
+ "theoretical_loss": 4.712817042171776,
+ "tokens_seen": 114491392
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000780036108324975,
+ "loss": 1.804,
+ "theoretical_loss": 4.712452477673001,
+ "tokens_seen": 114556928
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007800200601805417,
+ "loss": 1.7899,
+ "theoretical_loss": 4.712088180034517,
+ "tokens_seen": 114622464
+ },
+ {
+ "epoch": 0.03,
+ "objective/train/docs_used": 45100,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7656515836715698,
+ "objective/train/theoretical_loss": 4.711724148908571,
+ "objective/train/tokens_used": 135148000,
+ "theoretical_loss": 4.711724148908571,
+ "tokens_seen": 114688000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007800040120361084,
+ "loss": 1.7773,
+ "theoretical_loss": 4.711724148908571,
+ "tokens_seen": 114688000
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000779987963891675,
+ "loss": 1.8698,
+ "theoretical_loss": 4.711360383948064,
+ "tokens_seen": 114753536
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007799719157472418,
+ "loss": 1.81,
+ "theoretical_loss": 4.710996884806542,
+ "tokens_seen": 114819072
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007799558676028084,
+ "loss": 1.8963,
+ "theoretical_loss": 4.710633651138206,
+ "tokens_seen": 114884608
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007799398194583752,
+ "loss": 2.0256,
+ "theoretical_loss": 4.710270682597898,
+ "tokens_seen": 114950144
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007799237713139418,
+ "loss": 1.9627,
+ "theoretical_loss": 4.709907978841109,
+ "tokens_seen": 115015680
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007799077231695086,
+ "loss": 1.9976,
+ "theoretical_loss": 4.709545539523971,
+ "tokens_seen": 115081216
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007798916750250752,
+ "loss": 1.7934,
+ "theoretical_loss": 4.709183364303258,
+ "tokens_seen": 115146752
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000779875626880642,
+ "loss": 1.7876,
+ "theoretical_loss": 4.708821452836388,
+ "tokens_seen": 115212288
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007798595787362086,
+ "loss": 1.8766,
+ "theoretical_loss": 4.708459804781414,
+ "tokens_seen": 115277824
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007798435305917753,
+ "loss": 1.8795,
+ "theoretical_loss": 4.708098419797031,
+ "tokens_seen": 115343360
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.000779827482447342,
+ "loss": 1.8992,
+ "theoretical_loss": 4.707737297542563,
+ "tokens_seen": 115408896
+ },
+ {
+ "epoch": 0.03,
+ "learning_rate": 0.0007798114343029087,
+ "loss": 1.7714,
+ "theoretical_loss": 4.707376437677974,
+ "tokens_seen": 115474432
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797953861584754,
+ "loss": 1.8703,
+ "theoretical_loss": 4.707015839863859,
+ "tokens_seen": 115539968
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797793380140421,
+ "loss": 1.8995,
+ "theoretical_loss": 4.706655503761443,
+ "tokens_seen": 115605504
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797632898696088,
+ "loss": 1.8673,
+ "theoretical_loss": 4.706295429032584,
+ "tokens_seen": 115671040
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797472417251755,
+ "loss": 1.8975,
+ "theoretical_loss": 4.705935615339763,
+ "tokens_seen": 115736576
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797311935807422,
+ "loss": 1.964,
+ "theoretical_loss": 4.705576062346094,
+ "tokens_seen": 115802112
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007797151454363089,
+ "loss": 1.7302,
+ "theoretical_loss": 4.70521676971531,
+ "tokens_seen": 115867648
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796990972918756,
+ "loss": 1.8957,
+ "theoretical_loss": 4.704857737111773,
+ "tokens_seen": 115933184
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796830491474423,
+ "loss": 1.8279,
+ "theoretical_loss": 4.704498964200463,
+ "tokens_seen": 115998720
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779667001003009,
+ "loss": 1.9176,
+ "theoretical_loss": 4.70414045064698,
+ "tokens_seen": 116064256
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796509528585757,
+ "loss": 1.9368,
+ "theoretical_loss": 4.703782196117549,
+ "tokens_seen": 116129792
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796349047141424,
+ "loss": 1.9024,
+ "theoretical_loss": 4.703424200279007,
+ "tokens_seen": 116195328
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796188565697091,
+ "loss": 1.7334,
+ "theoretical_loss": 4.703066462798809,
+ "tokens_seen": 116260864
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007796028084252758,
+ "loss": 2.0384,
+ "theoretical_loss": 4.7027089833450235,
+ "tokens_seen": 116326400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007795867602808425,
+ "loss": 1.8372,
+ "theoretical_loss": 4.7023517615863355,
+ "tokens_seen": 116391936
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007795707121364092,
+ "loss": 1.7742,
+ "theoretical_loss": 4.701994797192038,
+ "tokens_seen": 116457472
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779554663991976,
+ "loss": 1.977,
+ "theoretical_loss": 4.701638089832034,
+ "tokens_seen": 116523008
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007795386158475426,
+ "loss": 1.9297,
+ "theoretical_loss": 4.701281639176839,
+ "tokens_seen": 116588544
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007795225677031094,
+ "loss": 1.8724,
+ "theoretical_loss": 4.700925444897575,
+ "tokens_seen": 116654080
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779506519558676,
+ "loss": 1.7618,
+ "theoretical_loss": 4.700569506665965,
+ "tokens_seen": 116719616
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794904714142428,
+ "loss": 1.8056,
+ "theoretical_loss": 4.700213824154341,
+ "tokens_seen": 116785152
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794744232698094,
+ "loss": 1.8923,
+ "theoretical_loss": 4.699858397035637,
+ "tokens_seen": 116850688
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794583751253762,
+ "loss": 1.9056,
+ "theoretical_loss": 4.699503224983388,
+ "tokens_seen": 116916224
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794423269809428,
+ "loss": 1.9701,
+ "theoretical_loss": 4.699148307671729,
+ "tokens_seen": 116981760
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794262788365096,
+ "loss": 1.8863,
+ "theoretical_loss": 4.698793644775394,
+ "tokens_seen": 117047296
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007794102306920762,
+ "loss": 1.9355,
+ "theoretical_loss": 4.698439235969713,
+ "tokens_seen": 117112832
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779394182547643,
+ "loss": 1.9582,
+ "theoretical_loss": 4.698085080930612,
+ "tokens_seen": 117178368
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007793781344032096,
+ "loss": 1.8616,
+ "theoretical_loss": 4.697731179334614,
+ "tokens_seen": 117243904
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007793620862587764,
+ "loss": 1.7266,
+ "theoretical_loss": 4.69737753085883,
+ "tokens_seen": 117309440
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779346038114343,
+ "loss": 1.9112,
+ "theoretical_loss": 4.697024135180968,
+ "tokens_seen": 117374976
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007793299899699098,
+ "loss": 1.8236,
+ "theoretical_loss": 4.696670991979321,
+ "tokens_seen": 117440512
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007793139418254765,
+ "loss": 1.8411,
+ "theoretical_loss": 4.696318100932773,
+ "tokens_seen": 117506048
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792978936810432,
+ "loss": 1.7059,
+ "theoretical_loss": 4.695965461720796,
+ "tokens_seen": 117571584
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792818455366099,
+ "loss": 1.822,
+ "theoretical_loss": 4.695613074023446,
+ "tokens_seen": 117637120
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792657973921766,
+ "loss": 1.894,
+ "theoretical_loss": 4.695260937521364,
+ "tokens_seen": 117702656
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792497492477433,
+ "loss": 1.7513,
+ "theoretical_loss": 4.694909051895775,
+ "tokens_seen": 117768192
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.00077923370110331,
+ "loss": 1.8627,
+ "theoretical_loss": 4.694557416828484,
+ "tokens_seen": 117833728
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792176529588767,
+ "loss": 1.7953,
+ "theoretical_loss": 4.694206032001879,
+ "tokens_seen": 117899264
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 46257,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9019111394882202,
+ "objective/train/theoretical_loss": 4.693854897098923,
+ "objective/train/tokens_used": 138424800,
+ "theoretical_loss": 4.693854897098923,
+ "tokens_seen": 117964800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007792016048144434,
+ "loss": 1.9417,
+ "theoretical_loss": 4.693854897098923,
+ "tokens_seen": 117964800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791855566700101,
+ "loss": 1.8928,
+ "theoretical_loss": 4.693504011803159,
+ "tokens_seen": 118030336
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791695085255768,
+ "loss": 1.8751,
+ "theoretical_loss": 4.6931533757987065,
+ "tokens_seen": 118095872
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791534603811435,
+ "loss": 1.9013,
+ "theoretical_loss": 4.692802988770258,
+ "tokens_seen": 118161408
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791374122367102,
+ "loss": 1.8634,
+ "theoretical_loss": 4.692452850403081,
+ "tokens_seen": 118226944
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791213640922769,
+ "loss": 2.0152,
+ "theoretical_loss": 4.692102960383014,
+ "tokens_seen": 118292480
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007791053159478436,
+ "loss": 1.7348,
+ "theoretical_loss": 4.691753318396467,
+ "tokens_seen": 118358016
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007790892678034103,
+ "loss": 1.8863,
+ "theoretical_loss": 4.691403924130418,
+ "tokens_seen": 118423552
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000779073219658977,
+ "loss": 1.9038,
+ "theoretical_loss": 4.691054777272415,
+ "tokens_seen": 118489088
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007790571715145437,
+ "loss": 1.8282,
+ "theoretical_loss": 4.69070587751057,
+ "tokens_seen": 118554624
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007790411233701104,
+ "loss": 1.9085,
+ "theoretical_loss": 4.690357224533564,
+ "tokens_seen": 118620160
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007790250752256771,
+ "loss": 1.8251,
+ "theoretical_loss": 4.690008818030638,
+ "tokens_seen": 118685696
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007790090270812438,
+ "loss": 1.6978,
+ "theoretical_loss": 4.689660657691598,
+ "tokens_seen": 118751232
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007789929789368105,
+ "loss": 1.9915,
+ "theoretical_loss": 4.689312743206811,
+ "tokens_seen": 118816768
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007789769307923772,
+ "loss": 1.9805,
+ "theoretical_loss": 4.688965074267203,
+ "tokens_seen": 118882304
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007789608826479439,
+ "loss": 1.8472,
+ "theoretical_loss": 4.688617650564259,
+ "tokens_seen": 118947840
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007789448345035106,
+ "loss": 1.9364,
+ "theoretical_loss": 4.688270471790023,
+ "tokens_seen": 119013376
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007789287863590773,
+ "loss": 1.8973,
+ "theoretical_loss": 4.687923537637094,
+ "tokens_seen": 119078912
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000778912738214644,
+ "loss": 1.8099,
+ "theoretical_loss": 4.687576847798622,
+ "tokens_seen": 119144448
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788966900702107,
+ "loss": 1.8415,
+ "theoretical_loss": 4.687230401968319,
+ "tokens_seen": 119209984
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788806419257774,
+ "loss": 1.9467,
+ "theoretical_loss": 4.68688419984044,
+ "tokens_seen": 119275520
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788645937813441,
+ "loss": 1.8803,
+ "theoretical_loss": 4.686538241109796,
+ "tokens_seen": 119341056
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788485456369108,
+ "loss": 1.8097,
+ "theoretical_loss": 4.686192525471746,
+ "tokens_seen": 119406592
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788324974924775,
+ "loss": 1.8324,
+ "theoretical_loss": 4.685847052622199,
+ "tokens_seen": 119472128
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788164493480442,
+ "loss": 1.8055,
+ "theoretical_loss": 4.685501822257608,
+ "tokens_seen": 119537664
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007788004012036109,
+ "loss": 1.7789,
+ "theoretical_loss": 4.685156834074972,
+ "tokens_seen": 119603200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007787843530591776,
+ "loss": 1.763,
+ "theoretical_loss": 4.684812087771839,
+ "tokens_seen": 119668736
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007787683049147443,
+ "loss": 1.8904,
+ "theoretical_loss": 4.684467583046292,
+ "tokens_seen": 119734272
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000778752256770311,
+ "loss": 1.84,
+ "theoretical_loss": 4.6841233195969645,
+ "tokens_seen": 119799808
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007787362086258777,
+ "loss": 1.9018,
+ "theoretical_loss": 4.683779297123024,
+ "tokens_seen": 119865344
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007787201604814444,
+ "loss": 1.804,
+ "theoretical_loss": 4.683435515324181,
+ "tokens_seen": 119930880
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007787041123370111,
+ "loss": 1.9663,
+ "theoretical_loss": 4.683091973900682,
+ "tokens_seen": 119996416
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786880641925778,
+ "loss": 1.849,
+ "theoretical_loss": 4.682748672553313,
+ "tokens_seen": 120061952
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786720160481445,
+ "loss": 1.8176,
+ "theoretical_loss": 4.68240561098339,
+ "tokens_seen": 120127488
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786559679037112,
+ "loss": 1.7518,
+ "theoretical_loss": 4.68206278889277,
+ "tokens_seen": 120193024
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786399197592779,
+ "loss": 1.8349,
+ "theoretical_loss": 4.681720205983839,
+ "tokens_seen": 120258560
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786238716148446,
+ "loss": 1.8489,
+ "theoretical_loss": 4.681377861959516,
+ "tokens_seen": 120324096
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007786078234704113,
+ "loss": 1.8931,
+ "theoretical_loss": 4.681035756523249,
+ "tokens_seen": 120389632
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785917753259779,
+ "loss": 1.9067,
+ "theoretical_loss": 4.680693889379018,
+ "tokens_seen": 120455168
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785757271815447,
+ "loss": 1.855,
+ "theoretical_loss": 4.680352260231329,
+ "tokens_seen": 120520704
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785596790371113,
+ "loss": 1.8698,
+ "theoretical_loss": 4.680010868785216,
+ "tokens_seen": 120586240
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785436308926781,
+ "loss": 1.83,
+ "theoretical_loss": 4.679669714746238,
+ "tokens_seen": 120651776
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785275827482447,
+ "loss": 1.9867,
+ "theoretical_loss": 4.679328797820478,
+ "tokens_seen": 120717312
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007785115346038115,
+ "loss": 1.944,
+ "theoretical_loss": 4.678988117714544,
+ "tokens_seen": 120782848
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784954864593781,
+ "loss": 1.7626,
+ "theoretical_loss": 4.678647674135563,
+ "tokens_seen": 120848384
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784794383149449,
+ "loss": 1.8657,
+ "theoretical_loss": 4.6783074667911855,
+ "tokens_seen": 120913920
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784633901705115,
+ "loss": 1.8642,
+ "theoretical_loss": 4.677967495389581,
+ "tokens_seen": 120979456
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784473420260782,
+ "loss": 1.7655,
+ "theoretical_loss": 4.677627759639435,
+ "tokens_seen": 121044992
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784312938816449,
+ "loss": 1.8806,
+ "theoretical_loss": 4.677288259249954,
+ "tokens_seen": 121110528
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007784152457372116,
+ "loss": 1.8441,
+ "theoretical_loss": 4.676948993930857,
+ "tokens_seen": 121176064
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 47440,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.9147517681121826,
+ "objective/train/theoretical_loss": 4.67660996339238,
+ "objective/train/tokens_used": 141701600,
+ "theoretical_loss": 4.67660996339238,
+ "tokens_seen": 121241600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783991975927783,
+ "loss": 1.9058,
+ "theoretical_loss": 4.67660996339238,
+ "tokens_seen": 121241600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000778383149448345,
+ "loss": 1.9118,
+ "theoretical_loss": 4.676271167345272,
+ "tokens_seen": 121307136
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783671013039117,
+ "loss": 1.9903,
+ "theoretical_loss": 4.675932605500794,
+ "tokens_seen": 121372672
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783510531594785,
+ "loss": 1.846,
+ "theoretical_loss": 4.675594277570718,
+ "tokens_seen": 121438208
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783350050150451,
+ "loss": 1.7874,
+ "theoretical_loss": 4.675256183267327,
+ "tokens_seen": 121503744
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783189568706119,
+ "loss": 1.8911,
+ "theoretical_loss": 4.674918322303411,
+ "tokens_seen": 121569280
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007783029087261785,
+ "loss": 1.7944,
+ "theoretical_loss": 4.6745806943922705,
+ "tokens_seen": 121634816
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782868605817453,
+ "loss": 1.7367,
+ "theoretical_loss": 4.67424329924771,
+ "tokens_seen": 121700352
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782708124373119,
+ "loss": 1.8909,
+ "theoretical_loss": 4.673906136584041,
+ "tokens_seen": 121765888
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782547642928787,
+ "loss": 1.9456,
+ "theoretical_loss": 4.673569206116078,
+ "tokens_seen": 121831424
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782387161484453,
+ "loss": 1.8463,
+ "theoretical_loss": 4.6732325075591366,
+ "tokens_seen": 121896960
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782226680040121,
+ "loss": 1.9333,
+ "theoretical_loss": 4.67289604062904,
+ "tokens_seen": 121962496
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007782066198595787,
+ "loss": 1.7753,
+ "theoretical_loss": 4.672559805042107,
+ "tokens_seen": 122028032
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781905717151455,
+ "loss": 1.8157,
+ "theoretical_loss": 4.672223800515159,
+ "tokens_seen": 122093568
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781745235707121,
+ "loss": 1.8541,
+ "theoretical_loss": 4.671888026765512,
+ "tokens_seen": 122159104
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781584754262789,
+ "loss": 1.7951,
+ "theoretical_loss": 4.6715524835109825,
+ "tokens_seen": 122224640
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781424272818455,
+ "loss": 1.8092,
+ "theoretical_loss": 4.671217170469884,
+ "tokens_seen": 122290176
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781263791374123,
+ "loss": 1.7207,
+ "theoretical_loss": 4.67088208736102,
+ "tokens_seen": 122355712
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007781103309929789,
+ "loss": 1.9078,
+ "theoretical_loss": 4.670547233903694,
+ "tokens_seen": 122421248
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780942828485457,
+ "loss": 1.8607,
+ "theoretical_loss": 4.670212609817698,
+ "tokens_seen": 122486784
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780782347041123,
+ "loss": 1.8305,
+ "theoretical_loss": 4.669878214823316,
+ "tokens_seen": 122552320
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780621865596791,
+ "loss": 1.9033,
+ "theoretical_loss": 4.669544048641325,
+ "tokens_seen": 122617856
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780461384152457,
+ "loss": 1.9115,
+ "theoretical_loss": 4.669210110992989,
+ "tokens_seen": 122683392
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780300902708125,
+ "loss": 1.9634,
+ "theoretical_loss": 4.668876401600059,
+ "tokens_seen": 122748928
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007780140421263791,
+ "loss": 1.7661,
+ "theoretical_loss": 4.668542920184779,
+ "tokens_seen": 122814464
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779979939819459,
+ "loss": 1.8726,
+ "theoretical_loss": 4.6682096664698705,
+ "tokens_seen": 122880000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779819458375126,
+ "loss": 1.9475,
+ "theoretical_loss": 4.667876640178546,
+ "tokens_seen": 122945536
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779658976930793,
+ "loss": 1.9562,
+ "theoretical_loss": 4.667543841034502,
+ "tokens_seen": 123011072
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777949849548646,
+ "loss": 1.9686,
+ "theoretical_loss": 4.667211268761914,
+ "tokens_seen": 123076608
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779338014042127,
+ "loss": 1.8776,
+ "theoretical_loss": 4.66687892308544,
+ "tokens_seen": 123142144
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779177532597794,
+ "loss": 1.9679,
+ "theoretical_loss": 4.6665468037302205,
+ "tokens_seen": 123207680
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007779017051153461,
+ "loss": 1.804,
+ "theoretical_loss": 4.666214910421875,
+ "tokens_seen": 123273216
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778856569709128,
+ "loss": 1.8628,
+ "theoretical_loss": 4.665883242886498,
+ "tokens_seen": 123338752
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778696088264795,
+ "loss": 1.8482,
+ "theoretical_loss": 4.6655518008506665,
+ "tokens_seen": 123404288
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778535606820462,
+ "loss": 1.8696,
+ "theoretical_loss": 4.665220584041428,
+ "tokens_seen": 123469824
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778375125376129,
+ "loss": 1.9316,
+ "theoretical_loss": 4.664889592186309,
+ "tokens_seen": 123535360
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778214643931796,
+ "loss": 1.9424,
+ "theoretical_loss": 4.6645588250133105,
+ "tokens_seen": 123600896
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007778054162487463,
+ "loss": 1.9075,
+ "theoretical_loss": 4.664228282250902,
+ "tokens_seen": 123666432
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777789368104313,
+ "loss": 1.8398,
+ "theoretical_loss": 4.663897963628029,
+ "tokens_seen": 123731968
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007777733199598797,
+ "loss": 1.8985,
+ "theoretical_loss": 4.6635678688741065,
+ "tokens_seen": 123797504
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007777572718154464,
+ "loss": 1.9671,
+ "theoretical_loss": 4.663237997719017,
+ "tokens_seen": 123863040
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007777412236710131,
+ "loss": 1.8071,
+ "theoretical_loss": 4.662908349893116,
+ "tokens_seen": 123928576
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007777251755265798,
+ "loss": 1.9107,
+ "theoretical_loss": 4.662578925127223,
+ "tokens_seen": 123994112
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007777091273821465,
+ "loss": 1.7927,
+ "theoretical_loss": 4.662249723152624,
+ "tokens_seen": 124059648
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007776930792377132,
+ "loss": 1.9306,
+ "theoretical_loss": 4.661920743701073,
+ "tokens_seen": 124125184
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007776770310932799,
+ "loss": 2.021,
+ "theoretical_loss": 4.661591986504786,
+ "tokens_seen": 124190720
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007776609829488466,
+ "loss": 1.9438,
+ "theoretical_loss": 4.661263451296444,
+ "tokens_seen": 124256256
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007776449348044133,
+ "loss": 1.8106,
+ "theoretical_loss": 4.660935137809188,
+ "tokens_seen": 124321792
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.00077762888665998,
+ "loss": 1.8744,
+ "theoretical_loss": 4.660607045776624,
+ "tokens_seen": 124387328
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007776128385155467,
+ "loss": 1.7867,
+ "theoretical_loss": 4.660279174932814,
+ "tokens_seen": 124452864
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 48338,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7055823802947998,
+ "objective/train/theoretical_loss": 4.659951525012284,
+ "objective/train/tokens_used": 144978400,
+ "theoretical_loss": 4.659951525012284,
+ "tokens_seen": 124518400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775967903711134,
+ "loss": 1.8029,
+ "theoretical_loss": 4.659951525012284,
+ "tokens_seen": 124518400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775807422266801,
+ "loss": 1.8085,
+ "theoretical_loss": 4.6596240957500115,
+ "tokens_seen": 124583936
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775646940822468,
+ "loss": 1.8204,
+ "theoretical_loss": 4.659296886881438,
+ "tokens_seen": 124649472
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775486459378135,
+ "loss": 1.8372,
+ "theoretical_loss": 4.658969898142456,
+ "tokens_seen": 124715008
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775325977933802,
+ "loss": 1.7415,
+ "theoretical_loss": 4.658643129269416,
+ "tokens_seen": 124780544
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775165496489469,
+ "loss": 1.7994,
+ "theoretical_loss": 4.658316579999122,
+ "tokens_seen": 124846080
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007775005015045136,
+ "loss": 1.8632,
+ "theoretical_loss": 4.65799025006883,
+ "tokens_seen": 124911616
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007774844533600803,
+ "loss": 1.8766,
+ "theoretical_loss": 4.65766413921625,
+ "tokens_seen": 124977152
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777468405215647,
+ "loss": 1.8277,
+ "theoretical_loss": 4.657338247179538,
+ "tokens_seen": 125042688
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007774523570712137,
+ "loss": 1.8551,
+ "theoretical_loss": 4.657012573697307,
+ "tokens_seen": 125108224
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007774363089267804,
+ "loss": 1.8749,
+ "theoretical_loss": 4.656687118508616,
+ "tokens_seen": 125173760
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007774202607823471,
+ "loss": 1.7936,
+ "theoretical_loss": 4.656361881352968,
+ "tokens_seen": 125239296
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007774042126379138,
+ "loss": 1.7392,
+ "theoretical_loss": 4.656036861970319,
+ "tokens_seen": 125304832
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007773881644934805,
+ "loss": 1.8527,
+ "theoretical_loss": 4.655712060101068,
+ "tokens_seen": 125370368
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007773721163490472,
+ "loss": 1.854,
+ "theoretical_loss": 4.65538747548606,
+ "tokens_seen": 125435904
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007773560682046139,
+ "loss": 1.8442,
+ "theoretical_loss": 4.655063107866582,
+ "tokens_seen": 125501440
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007773400200601806,
+ "loss": 2.0244,
+ "theoretical_loss": 4.654738956984366,
+ "tokens_seen": 125566976
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007773239719157473,
+ "loss": 1.7517,
+ "theoretical_loss": 4.654415022581585,
+ "tokens_seen": 125632512
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777307923771314,
+ "loss": 1.8149,
+ "theoretical_loss": 4.654091304400853,
+ "tokens_seen": 125698048
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772918756268807,
+ "loss": 1.9215,
+ "theoretical_loss": 4.653767802185225,
+ "tokens_seen": 125763584
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772758274824474,
+ "loss": 1.8353,
+ "theoretical_loss": 4.653444515678195,
+ "tokens_seen": 125829120
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772597793380141,
+ "loss": 1.8363,
+ "theoretical_loss": 4.653121444623693,
+ "tokens_seen": 125894656
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772437311935808,
+ "loss": 1.7447,
+ "theoretical_loss": 4.652798588766088,
+ "tokens_seen": 125960192
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772276830491475,
+ "loss": 1.7671,
+ "theoretical_loss": 4.652475947850185,
+ "tokens_seen": 126025728
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007772116349047142,
+ "loss": 1.753,
+ "theoretical_loss": 4.652153521621223,
+ "tokens_seen": 126091264
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007771955867602808,
+ "loss": 1.9541,
+ "theoretical_loss": 4.651831309824875,
+ "tokens_seen": 126156800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007771795386158476,
+ "loss": 1.7825,
+ "theoretical_loss": 4.65150931220725,
+ "tokens_seen": 126222336
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007771634904714142,
+ "loss": 1.9339,
+ "theoretical_loss": 4.651187528514887,
+ "tokens_seen": 126287872
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777147442326981,
+ "loss": 1.8252,
+ "theoretical_loss": 4.650865958494755,
+ "tokens_seen": 126353408
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007771313941825476,
+ "loss": 1.8909,
+ "theoretical_loss": 4.650544601894257,
+ "tokens_seen": 126418944
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007771153460381144,
+ "loss": 1.9024,
+ "theoretical_loss": 4.650223458461221,
+ "tokens_seen": 126484480
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000777099297893681,
+ "loss": 1.862,
+ "theoretical_loss": 4.649902527943908,
+ "tokens_seen": 126550016
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770832497492478,
+ "loss": 1.8003,
+ "theoretical_loss": 4.649581810091003,
+ "tokens_seen": 126615552
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770672016048144,
+ "loss": 2.0104,
+ "theoretical_loss": 4.649261304651619,
+ "tokens_seen": 126681088
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770511534603812,
+ "loss": 1.7893,
+ "theoretical_loss": 4.6489410113752925,
+ "tokens_seen": 126746624
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770351053159478,
+ "loss": 1.814,
+ "theoretical_loss": 4.6486209300119885,
+ "tokens_seen": 126812160
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770190571715146,
+ "loss": 1.9046,
+ "theoretical_loss": 4.648301060312093,
+ "tokens_seen": 126877696
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007770030090270812,
+ "loss": 1.8817,
+ "theoretical_loss": 4.647981402026417,
+ "tokens_seen": 126943232
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776986960882648,
+ "loss": 1.8367,
+ "theoretical_loss": 4.647661954906188,
+ "tokens_seen": 127008768
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007769709127382146,
+ "loss": 1.7926,
+ "theoretical_loss": 4.647342718703062,
+ "tokens_seen": 127074304
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007769548645937814,
+ "loss": 1.9298,
+ "theoretical_loss": 4.647023693169107,
+ "tokens_seen": 127139840
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776938816449348,
+ "loss": 1.7827,
+ "theoretical_loss": 4.646704878056816,
+ "tokens_seen": 127205376
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007769227683049148,
+ "loss": 1.7387,
+ "theoretical_loss": 4.6463862731191,
+ "tokens_seen": 127270912
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007769067201604814,
+ "loss": 1.91,
+ "theoretical_loss": 4.6460678781092835,
+ "tokens_seen": 127336448
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007768906720160482,
+ "loss": 1.993,
+ "theoretical_loss": 4.6457496927811075,
+ "tokens_seen": 127401984
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007768746238716148,
+ "loss": 1.6914,
+ "theoretical_loss": 4.645431716888734,
+ "tokens_seen": 127467520
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007768585757271816,
+ "loss": 1.8668,
+ "theoretical_loss": 4.645113950186732,
+ "tokens_seen": 127533056
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007768425275827482,
+ "loss": 1.7944,
+ "theoretical_loss": 4.64479639243009,
+ "tokens_seen": 127598592
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776826479438315,
+ "loss": 1.7889,
+ "theoretical_loss": 4.644479043374204,
+ "tokens_seen": 127664128
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007768104312938816,
+ "loss": 1.8859,
+ "theoretical_loss": 4.644161902774886,
+ "tokens_seen": 127729664
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 49581,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.94822096824646,
+ "objective/train/theoretical_loss": 4.643844970388358,
+ "objective/train/tokens_used": 148255200,
+ "theoretical_loss": 4.643844970388358,
+ "tokens_seen": 127795200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007767943831494484,
+ "loss": 1.9133,
+ "theoretical_loss": 4.643844970388358,
+ "tokens_seen": 127795200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776778335005015,
+ "loss": 1.8609,
+ "theoretical_loss": 4.6435282459712495,
+ "tokens_seen": 127860736
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007767622868605818,
+ "loss": 1.8257,
+ "theoretical_loss": 4.643211729280601,
+ "tokens_seen": 127926272
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007767462387161484,
+ "loss": 1.7747,
+ "theoretical_loss": 4.64289542007386,
+ "tokens_seen": 127991808
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007767301905717152,
+ "loss": 1.8617,
+ "theoretical_loss": 4.642579318108883,
+ "tokens_seen": 128057344
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007767141424272818,
+ "loss": 1.8137,
+ "theoretical_loss": 4.642263423143932,
+ "tokens_seen": 128122880
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007766980942828486,
+ "loss": 1.8099,
+ "theoretical_loss": 4.641947734937673,
+ "tokens_seen": 128188416
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007766820461384152,
+ "loss": 1.8716,
+ "theoretical_loss": 4.641632253249179,
+ "tokens_seen": 128253952
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776665997993982,
+ "loss": 2.0769,
+ "theoretical_loss": 4.641316977837924,
+ "tokens_seen": 128319488
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007766499498495486,
+ "loss": 1.89,
+ "theoretical_loss": 4.641001908463787,
+ "tokens_seen": 128385024
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007766339017051154,
+ "loss": 1.8734,
+ "theoretical_loss": 4.6406870448870485,
+ "tokens_seen": 128450560
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776617853560682,
+ "loss": 1.8353,
+ "theoretical_loss": 4.64037238686839,
+ "tokens_seen": 128516096
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007766018054162488,
+ "loss": 1.8572,
+ "theoretical_loss": 4.640057934168891,
+ "tokens_seen": 128581632
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007765857572718155,
+ "loss": 1.7848,
+ "theoretical_loss": 4.639743686550034,
+ "tokens_seen": 128647168
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007765697091273822,
+ "loss": 1.8396,
+ "theoretical_loss": 4.639429643773697,
+ "tokens_seen": 128712704
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007765536609829489,
+ "loss": 1.8095,
+ "theoretical_loss": 4.639115805602156,
+ "tokens_seen": 128778240
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007765376128385156,
+ "loss": 1.8191,
+ "theoretical_loss": 4.638802171798086,
+ "tokens_seen": 128843776
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007765215646940823,
+ "loss": 1.9008,
+ "theoretical_loss": 4.638488742124554,
+ "tokens_seen": 128909312
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776505516549649,
+ "loss": 1.8786,
+ "theoretical_loss": 4.638175516345027,
+ "tokens_seen": 128974848
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764894684052157,
+ "loss": 1.8323,
+ "theoretical_loss": 4.637862494223359,
+ "tokens_seen": 129040384
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764734202607824,
+ "loss": 1.7773,
+ "theoretical_loss": 4.637549675523806,
+ "tokens_seen": 129105920
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764573721163491,
+ "loss": 1.8804,
+ "theoretical_loss": 4.637237060011007,
+ "tokens_seen": 129171456
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764413239719158,
+ "loss": 1.847,
+ "theoretical_loss": 4.636924647450002,
+ "tokens_seen": 129236992
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764252758274825,
+ "loss": 1.8726,
+ "theoretical_loss": 4.636612437606212,
+ "tokens_seen": 129302528
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007764092276830492,
+ "loss": 1.8728,
+ "theoretical_loss": 4.636300430245457,
+ "tokens_seen": 129368064
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007763931795386159,
+ "loss": 1.7916,
+ "theoretical_loss": 4.63598862513394,
+ "tokens_seen": 129433600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007763771313941826,
+ "loss": 1.8813,
+ "theoretical_loss": 4.635677022038254,
+ "tokens_seen": 129499136
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007763610832497493,
+ "loss": 1.8075,
+ "theoretical_loss": 4.63536562072538,
+ "tokens_seen": 129564672
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776345035105316,
+ "loss": 1.8598,
+ "theoretical_loss": 4.635054420962684,
+ "tokens_seen": 129630208
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007763289869608827,
+ "loss": 1.8148,
+ "theoretical_loss": 4.634743422517918,
+ "tokens_seen": 129695744
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007763129388164494,
+ "loss": 1.7992,
+ "theoretical_loss": 4.634432625159221,
+ "tokens_seen": 129761280
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762968906720161,
+ "loss": 1.8888,
+ "theoretical_loss": 4.634122028655112,
+ "tokens_seen": 129826816
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762808425275828,
+ "loss": 1.7781,
+ "theoretical_loss": 4.633811632774496,
+ "tokens_seen": 129892352
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762647943831495,
+ "loss": 1.8929,
+ "theoretical_loss": 4.633501437286659,
+ "tokens_seen": 129957888
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762487462387162,
+ "loss": 1.9065,
+ "theoretical_loss": 4.633191441961272,
+ "tokens_seen": 130023424
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762326980942829,
+ "loss": 1.7713,
+ "theoretical_loss": 4.63288164656838,
+ "tokens_seen": 130088960
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762166499498496,
+ "loss": 1.7753,
+ "theoretical_loss": 4.632572050878415,
+ "tokens_seen": 130154496
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007762006018054163,
+ "loss": 1.8997,
+ "theoretical_loss": 4.632262654662181,
+ "tokens_seen": 130220032
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000776184553660983,
+ "loss": 1.8561,
+ "theoretical_loss": 4.631953457690868,
+ "tokens_seen": 130285568
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007761685055165497,
+ "loss": 1.9013,
+ "theoretical_loss": 4.631644459736036,
+ "tokens_seen": 130351104
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007761524573721164,
+ "loss": 1.9393,
+ "theoretical_loss": 4.631335660569626,
+ "tokens_seen": 130416640
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007761364092276831,
+ "loss": 1.9441,
+ "theoretical_loss": 4.631027059963953,
+ "tokens_seen": 130482176
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007761203610832498,
+ "loss": 1.8176,
+ "theoretical_loss": 4.630718657691709,
+ "tokens_seen": 130547712
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007761043129388165,
+ "loss": 1.7216,
+ "theoretical_loss": 4.630410453525958,
+ "tokens_seen": 130613248
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007760882647943832,
+ "loss": 1.766,
+ "theoretical_loss": 4.630102447240138,
+ "tokens_seen": 130678784
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007760722166499499,
+ "loss": 1.8036,
+ "theoretical_loss": 4.629794638608059,
+ "tokens_seen": 130744320
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007760561685055166,
+ "loss": 1.851,
+ "theoretical_loss": 4.629487027403905,
+ "tokens_seen": 130809856
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007760401203610833,
+ "loss": 1.8026,
+ "theoretical_loss": 4.629179613402228,
+ "tokens_seen": 130875392
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.00077602407221665,
+ "loss": 1.8168,
+ "theoretical_loss": 4.628872396377953,
+ "tokens_seen": 130940928
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007760080240722167,
+ "loss": 1.872,
+ "theoretical_loss": 4.628565376106371,
+ "tokens_seen": 131006464
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 50603,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7769169807434082,
+ "objective/train/theoretical_loss": 4.628258552363146,
+ "objective/train/tokens_used": 151532000,
+ "theoretical_loss": 4.628258552363146,
+ "tokens_seen": 131072000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759919759277834,
+ "loss": 1.8162,
+ "theoretical_loss": 4.628258552363146,
+ "tokens_seen": 131072000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759759277833502,
+ "loss": 1.8492,
+ "theoretical_loss": 4.627951924924305,
+ "tokens_seen": 131137536
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759598796389168,
+ "loss": 1.8724,
+ "theoretical_loss": 4.627645493566245,
+ "tokens_seen": 131203072
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759438314944836,
+ "loss": 1.8344,
+ "theoretical_loss": 4.62733925806573,
+ "tokens_seen": 131268608
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759277833500502,
+ "loss": 1.8899,
+ "theoretical_loss": 4.627033218199886,
+ "tokens_seen": 131334144
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007759117352056169,
+ "loss": 1.8562,
+ "theoretical_loss": 4.626727373746206,
+ "tokens_seen": 131399680
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007758956870611836,
+ "loss": 1.8066,
+ "theoretical_loss": 4.626421724482549,
+ "tokens_seen": 131465216
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007758796389167503,
+ "loss": 1.8461,
+ "theoretical_loss": 4.626116270187131,
+ "tokens_seen": 131530752
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000775863590772317,
+ "loss": 1.736,
+ "theoretical_loss": 4.625811010638538,
+ "tokens_seen": 131596288
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007758475426278837,
+ "loss": 1.8001,
+ "theoretical_loss": 4.62550594561571,
+ "tokens_seen": 131661824
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007758314944834504,
+ "loss": 1.823,
+ "theoretical_loss": 4.625201074897952,
+ "tokens_seen": 131727360
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007758154463390171,
+ "loss": 1.9525,
+ "theoretical_loss": 4.62489639826493,
+ "tokens_seen": 131792896
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757993981945837,
+ "loss": 1.8137,
+ "theoretical_loss": 4.6245919154966675,
+ "tokens_seen": 131858432
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757833500501505,
+ "loss": 1.8273,
+ "theoretical_loss": 4.624287626373545,
+ "tokens_seen": 131923968
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757673019057171,
+ "loss": 1.7471,
+ "theoretical_loss": 4.623983530676304,
+ "tokens_seen": 131989504
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757512537612839,
+ "loss": 1.8519,
+ "theoretical_loss": 4.62367962818604,
+ "tokens_seen": 132055040
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757352056168505,
+ "loss": 1.944,
+ "theoretical_loss": 4.623375918684207,
+ "tokens_seen": 132120576
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757191574724173,
+ "loss": 1.8725,
+ "theoretical_loss": 4.623072401952614,
+ "tokens_seen": 132186112
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007757031093279839,
+ "loss": 1.8953,
+ "theoretical_loss": 4.622769077773423,
+ "tokens_seen": 132251648
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756870611835507,
+ "loss": 1.879,
+ "theoretical_loss": 4.622465945929152,
+ "tokens_seen": 132317184
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756710130391173,
+ "loss": 1.7999,
+ "theoretical_loss": 4.622163006202671,
+ "tokens_seen": 132382720
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756549648946841,
+ "loss": 1.7466,
+ "theoretical_loss": 4.621860258377203,
+ "tokens_seen": 132448256
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756389167502507,
+ "loss": 1.8657,
+ "theoretical_loss": 4.621557702236323,
+ "tokens_seen": 132513792
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756228686058175,
+ "loss": 1.806,
+ "theoretical_loss": 4.621255337563955,
+ "tokens_seen": 132579328
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007756068204613841,
+ "loss": 1.8109,
+ "theoretical_loss": 4.620953164144376,
+ "tokens_seen": 132644864
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755907723169509,
+ "loss": 1.8321,
+ "theoretical_loss": 4.620651181762212,
+ "tokens_seen": 132710400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755747241725175,
+ "loss": 1.8282,
+ "theoretical_loss": 4.620349390202436,
+ "tokens_seen": 132775936
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755586760280843,
+ "loss": 1.8138,
+ "theoretical_loss": 4.6200477892503695,
+ "tokens_seen": 132841472
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755426278836509,
+ "loss": 1.8533,
+ "theoretical_loss": 4.619746378691682,
+ "tokens_seen": 132907008
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755265797392177,
+ "loss": 1.7597,
+ "theoretical_loss": 4.61944515831239,
+ "tokens_seen": 132972544
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007755105315947843,
+ "loss": 1.8646,
+ "theoretical_loss": 4.6191441278988545,
+ "tokens_seen": 133038080
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754944834503511,
+ "loss": 1.7799,
+ "theoretical_loss": 4.618843287237782,
+ "tokens_seen": 133103616
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754784353059177,
+ "loss": 1.8689,
+ "theoretical_loss": 4.618542636116224,
+ "tokens_seen": 133169152
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754623871614845,
+ "loss": 1.833,
+ "theoretical_loss": 4.618242174321575,
+ "tokens_seen": 133234688
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754463390170511,
+ "loss": 1.8088,
+ "theoretical_loss": 4.617941901641573,
+ "tokens_seen": 133300224
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754302908726179,
+ "loss": 1.8624,
+ "theoretical_loss": 4.617641817864296,
+ "tokens_seen": 133365760
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007754142427281845,
+ "loss": 1.9235,
+ "theoretical_loss": 4.617341922778168,
+ "tokens_seen": 133431296
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753981945837513,
+ "loss": 1.701,
+ "theoretical_loss": 4.617042216171949,
+ "tokens_seen": 133496832
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753821464393179,
+ "loss": 1.7201,
+ "theoretical_loss": 4.616742697834742,
+ "tokens_seen": 133562368
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753660982948847,
+ "loss": 1.8286,
+ "theoretical_loss": 4.616443367555988,
+ "tokens_seen": 133627904
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753500501504513,
+ "loss": 1.8104,
+ "theoretical_loss": 4.616144225125465,
+ "tokens_seen": 133693440
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753340020060181,
+ "loss": 1.8212,
+ "theoretical_loss": 4.615845270333294,
+ "tokens_seen": 133758976
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753179538615847,
+ "loss": 1.7838,
+ "theoretical_loss": 4.615546502969929,
+ "tokens_seen": 133824512
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007753019057171515,
+ "loss": 1.8179,
+ "theoretical_loss": 4.61524792282616,
+ "tokens_seen": 133890048
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007752858575727181,
+ "loss": 1.8488,
+ "theoretical_loss": 4.614949529693115,
+ "tokens_seen": 133955584
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007752698094282849,
+ "loss": 1.7209,
+ "theoretical_loss": 4.614651323362257,
+ "tokens_seen": 134021120
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007752537612838515,
+ "loss": 1.6942,
+ "theoretical_loss": 4.614353303625382,
+ "tokens_seen": 134086656
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007752377131394183,
+ "loss": 1.819,
+ "theoretical_loss": 4.6140554702746215,
+ "tokens_seen": 134152192
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000775221664994985,
+ "loss": 1.7107,
+ "theoretical_loss": 4.613757823102437,
+ "tokens_seen": 134217728
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007752056168505517,
+ "loss": 1.782,
+ "theoretical_loss": 4.613460361901627,
+ "tokens_seen": 134283264
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 51744,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.0655648708343506,
+ "objective/train/theoretical_loss": 4.613163086465315,
+ "objective/train/tokens_used": 154808800,
+ "theoretical_loss": 4.613163086465315,
+ "tokens_seen": 134348800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751895687061184,
+ "loss": 1.9222,
+ "theoretical_loss": 4.613163086465315,
+ "tokens_seen": 134348800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751735205616851,
+ "loss": 1.7955,
+ "theoretical_loss": 4.612865996586962,
+ "tokens_seen": 134414336
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751574724172518,
+ "loss": 1.7857,
+ "theoretical_loss": 4.612569092060355,
+ "tokens_seen": 134479872
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751414242728185,
+ "loss": 1.8141,
+ "theoretical_loss": 4.612272372679611,
+ "tokens_seen": 134545408
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751253761283852,
+ "loss": 1.9289,
+ "theoretical_loss": 4.611975838239179,
+ "tokens_seen": 134610944
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007751093279839519,
+ "loss": 1.6775,
+ "theoretical_loss": 4.611679488533832,
+ "tokens_seen": 134676480
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007750932798395186,
+ "loss": 1.8499,
+ "theoretical_loss": 4.611383323358671,
+ "tokens_seen": 134742016
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007750772316950853,
+ "loss": 1.8098,
+ "theoretical_loss": 4.61108734250913,
+ "tokens_seen": 134807552
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000775061183550652,
+ "loss": 1.8215,
+ "theoretical_loss": 4.610791545780959,
+ "tokens_seen": 134873088
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007750451354062187,
+ "loss": 1.6756,
+ "theoretical_loss": 4.610495932970241,
+ "tokens_seen": 134938624
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007750290872617854,
+ "loss": 1.7515,
+ "theoretical_loss": 4.61020050387338,
+ "tokens_seen": 135004160
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007750130391173521,
+ "loss": 1.7896,
+ "theoretical_loss": 4.609905258287107,
+ "tokens_seen": 135069696
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749969909729188,
+ "loss": 1.8716,
+ "theoretical_loss": 4.609610196008473,
+ "tokens_seen": 135135232
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749809428284855,
+ "loss": 1.7909,
+ "theoretical_loss": 4.609315316834855,
+ "tokens_seen": 135200768
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749648946840522,
+ "loss": 1.7184,
+ "theoretical_loss": 4.60902062056395,
+ "tokens_seen": 135266304
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749488465396189,
+ "loss": 1.8287,
+ "theoretical_loss": 4.608726106993776,
+ "tokens_seen": 135331840
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749327983951856,
+ "loss": 1.9142,
+ "theoretical_loss": 4.608431775922673,
+ "tokens_seen": 135397376
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007749167502507523,
+ "loss": 1.7733,
+ "theoretical_loss": 4.608137627149302,
+ "tokens_seen": 135462912
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000774900702106319,
+ "loss": 1.8085,
+ "theoretical_loss": 4.607843660472642,
+ "tokens_seen": 135528448
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748846539618857,
+ "loss": 1.8051,
+ "theoretical_loss": 4.607549875691989,
+ "tokens_seen": 135593984
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748686058174524,
+ "loss": 1.8313,
+ "theoretical_loss": 4.607256272606961,
+ "tokens_seen": 135659520
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748525576730191,
+ "loss": 1.8189,
+ "theoretical_loss": 4.606962851017489,
+ "tokens_seen": 135725056
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748365095285858,
+ "loss": 1.905,
+ "theoretical_loss": 4.606669610723825,
+ "tokens_seen": 135790592
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748204613841525,
+ "loss": 1.8924,
+ "theoretical_loss": 4.606376551526536,
+ "tokens_seen": 135856128
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007748044132397192,
+ "loss": 1.7682,
+ "theoretical_loss": 4.6060836732265,
+ "tokens_seen": 135921664
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747883650952859,
+ "loss": 1.7632,
+ "theoretical_loss": 4.605790975624917,
+ "tokens_seen": 135987200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747723169508527,
+ "loss": 1.8681,
+ "theoretical_loss": 4.605498458523296,
+ "tokens_seen": 136052736
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747562688064193,
+ "loss": 1.7845,
+ "theoretical_loss": 4.605206121723461,
+ "tokens_seen": 136118272
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747402206619861,
+ "loss": 1.7568,
+ "theoretical_loss": 4.604913965027551,
+ "tokens_seen": 136183808
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747241725175527,
+ "loss": 1.8752,
+ "theoretical_loss": 4.604621988238012,
+ "tokens_seen": 136249344
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007747081243731195,
+ "loss": 1.9572,
+ "theoretical_loss": 4.604330191157608,
+ "tokens_seen": 136314880
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746920762286861,
+ "loss": 1.7565,
+ "theoretical_loss": 4.604038573589408,
+ "tokens_seen": 136380416
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746760280842529,
+ "loss": 1.783,
+ "theoretical_loss": 4.603747135336796,
+ "tokens_seen": 136445952
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746599799398195,
+ "loss": 1.8388,
+ "theoretical_loss": 4.603455876203463,
+ "tokens_seen": 136511488
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746439317953863,
+ "loss": 1.7053,
+ "theoretical_loss": 4.603164795993409,
+ "tokens_seen": 136577024
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746278836509529,
+ "loss": 1.7839,
+ "theoretical_loss": 4.602873894510945,
+ "tokens_seen": 136642560
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007746118355065197,
+ "loss": 1.7043,
+ "theoretical_loss": 4.602583171560686,
+ "tokens_seen": 136708096
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745957873620863,
+ "loss": 1.7794,
+ "theoretical_loss": 4.602292626947556,
+ "tokens_seen": 136773632
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745797392176531,
+ "loss": 1.8616,
+ "theoretical_loss": 4.602002260476787,
+ "tokens_seen": 136839168
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745636910732197,
+ "loss": 1.783,
+ "theoretical_loss": 4.601712071953914,
+ "tokens_seen": 136904704
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745476429287865,
+ "loss": 1.7862,
+ "theoretical_loss": 4.601422061184778,
+ "tokens_seen": 136970240
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745315947843531,
+ "loss": 1.6995,
+ "theoretical_loss": 4.601132227975527,
+ "tokens_seen": 137035776
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007745155466399198,
+ "loss": 2.0007,
+ "theoretical_loss": 4.60084257213261,
+ "tokens_seen": 137101312
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744994984954865,
+ "loss": 1.7742,
+ "theoretical_loss": 4.600553093462781,
+ "tokens_seen": 137166848
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744834503510532,
+ "loss": 1.9237,
+ "theoretical_loss": 4.600263791773097,
+ "tokens_seen": 137232384
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744674022066199,
+ "loss": 1.8399,
+ "theoretical_loss": 4.599974666870914,
+ "tokens_seen": 137297920
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744513540621866,
+ "loss": 1.7832,
+ "theoretical_loss": 4.599685718563894,
+ "tokens_seen": 137363456
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744353059177533,
+ "loss": 1.801,
+ "theoretical_loss": 4.599396946659997,
+ "tokens_seen": 137428992
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.00077441925777332,
+ "loss": 1.7621,
+ "theoretical_loss": 4.5991083509674855,
+ "tokens_seen": 137494528
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007744032096288867,
+ "loss": 1.7486,
+ "theoretical_loss": 4.598819931294919,
+ "tokens_seen": 137560064
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 52717,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7819393873214722,
+ "objective/train/theoretical_loss": 4.598531687451157,
+ "objective/train/tokens_used": 158085600,
+ "theoretical_loss": 4.598531687451157,
+ "tokens_seen": 137625600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007743871614844534,
+ "loss": 1.8069,
+ "theoretical_loss": 4.598531687451157,
+ "tokens_seen": 137625600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.00077437111334002,
+ "loss": 1.8632,
+ "theoretical_loss": 4.598243619245359,
+ "tokens_seen": 137691136
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007743550651955868,
+ "loss": 1.8544,
+ "theoretical_loss": 4.59795572648698,
+ "tokens_seen": 137756672
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007743390170511534,
+ "loss": 1.8115,
+ "theoretical_loss": 4.597668008985774,
+ "tokens_seen": 137822208
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007743229689067202,
+ "loss": 1.7225,
+ "theoretical_loss": 4.59738046655179,
+ "tokens_seen": 137887744
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007743069207622868,
+ "loss": 1.7712,
+ "theoretical_loss": 4.597093098995377,
+ "tokens_seen": 137953280
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007742908726178536,
+ "loss": 1.735,
+ "theoretical_loss": 4.596805906127173,
+ "tokens_seen": 138018816
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007742748244734202,
+ "loss": 1.7213,
+ "theoretical_loss": 4.596518887758117,
+ "tokens_seen": 138084352
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000774258776328987,
+ "loss": 1.867,
+ "theoretical_loss": 4.596232043699438,
+ "tokens_seen": 138149888
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007742427281845536,
+ "loss": 1.842,
+ "theoretical_loss": 4.59594537376266,
+ "tokens_seen": 138215424
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007742266800401204,
+ "loss": 1.8787,
+ "theoretical_loss": 4.595658877759602,
+ "tokens_seen": 138280960
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000774210631895687,
+ "loss": 1.9223,
+ "theoretical_loss": 4.595372555502371,
+ "tokens_seen": 138346496
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741945837512538,
+ "loss": 1.9055,
+ "theoretical_loss": 4.59508640680337,
+ "tokens_seen": 138412032
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741785356068204,
+ "loss": 1.8217,
+ "theoretical_loss": 4.594800431475292,
+ "tokens_seen": 138477568
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741624874623872,
+ "loss": 1.7421,
+ "theoretical_loss": 4.594514629331121,
+ "tokens_seen": 138543104
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741464393179538,
+ "loss": 1.7933,
+ "theoretical_loss": 4.594229000184128,
+ "tokens_seen": 138608640
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741303911735206,
+ "loss": 1.8861,
+ "theoretical_loss": 4.593943543847878,
+ "tokens_seen": 138674176
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007741143430290872,
+ "loss": 1.8354,
+ "theoretical_loss": 4.5936582601362215,
+ "tokens_seen": 138739712
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000774098294884654,
+ "loss": 1.8875,
+ "theoretical_loss": 4.593373148863301,
+ "tokens_seen": 138805248
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007740822467402206,
+ "loss": 1.8701,
+ "theoretical_loss": 4.593088209843543,
+ "tokens_seen": 138870784
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007740661985957874,
+ "loss": 1.8651,
+ "theoretical_loss": 4.592803442891663,
+ "tokens_seen": 138936320
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000774050150451354,
+ "loss": 1.8731,
+ "theoretical_loss": 4.592518847822661,
+ "tokens_seen": 139001856
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007740341023069208,
+ "loss": 1.7787,
+ "theoretical_loss": 4.592234424451828,
+ "tokens_seen": 139067392
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007740180541624874,
+ "loss": 1.8654,
+ "theoretical_loss": 4.591950172594734,
+ "tokens_seen": 139132928
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007740020060180542,
+ "loss": 1.728,
+ "theoretical_loss": 4.591666092067238,
+ "tokens_seen": 139198464
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007739859578736208,
+ "loss": 1.8389,
+ "theoretical_loss": 4.591382182685484,
+ "tokens_seen": 139264000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007739699097291876,
+ "loss": 1.7957,
+ "theoretical_loss": 4.591098444265896,
+ "tokens_seen": 139329536
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007739538615847542,
+ "loss": 1.7924,
+ "theoretical_loss": 4.590814876625183,
+ "tokens_seen": 139395072
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773937813440321,
+ "loss": 1.8632,
+ "theoretical_loss": 4.590531479580338,
+ "tokens_seen": 139460608
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007739217652958876,
+ "loss": 1.747,
+ "theoretical_loss": 4.5902482529486335,
+ "tokens_seen": 139526144
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007739057171514544,
+ "loss": 1.8658,
+ "theoretical_loss": 4.589965196547625,
+ "tokens_seen": 139591680
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773889669007021,
+ "loss": 1.904,
+ "theoretical_loss": 4.5896823101951485,
+ "tokens_seen": 139657216
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007738736208625878,
+ "loss": 1.8037,
+ "theoretical_loss": 4.58939959370932,
+ "tokens_seen": 139722752
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007738575727181545,
+ "loss": 1.7745,
+ "theoretical_loss": 4.5891170469085365,
+ "tokens_seen": 139788288
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007738415245737212,
+ "loss": 1.7752,
+ "theoretical_loss": 4.588834669611472,
+ "tokens_seen": 139853824
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007738254764292879,
+ "loss": 1.8438,
+ "theoretical_loss": 4.5885524616370805,
+ "tokens_seen": 139919360
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007738094282848546,
+ "loss": 1.8438,
+ "theoretical_loss": 4.5882704228045945,
+ "tokens_seen": 139984896
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007737933801404213,
+ "loss": 1.916,
+ "theoretical_loss": 4.587988552933523,
+ "tokens_seen": 140050432
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773777331995988,
+ "loss": 1.7058,
+ "theoretical_loss": 4.587706851843652,
+ "tokens_seen": 140115968
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007737612838515547,
+ "loss": 1.8099,
+ "theoretical_loss": 4.587425319355044,
+ "tokens_seen": 140181504
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007737452357071214,
+ "loss": 1.7592,
+ "theoretical_loss": 4.587143955288038,
+ "tokens_seen": 140247040
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007737291875626881,
+ "loss": 1.8013,
+ "theoretical_loss": 4.586862759463248,
+ "tokens_seen": 140312576
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007737131394182548,
+ "loss": 1.8653,
+ "theoretical_loss": 4.586581731701562,
+ "tokens_seen": 140378112
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736970912738215,
+ "loss": 1.8956,
+ "theoretical_loss": 4.586300871824143,
+ "tokens_seen": 140443648
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736810431293882,
+ "loss": 1.8632,
+ "theoretical_loss": 4.586020179652427,
+ "tokens_seen": 140509184
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736649949849549,
+ "loss": 1.7402,
+ "theoretical_loss": 4.585739655008123,
+ "tokens_seen": 140574720
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736489468405216,
+ "loss": 1.8236,
+ "theoretical_loss": 4.585459297713215,
+ "tokens_seen": 140640256
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736328986960883,
+ "loss": 1.8042,
+ "theoretical_loss": 4.585179107589955,
+ "tokens_seen": 140705792
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773616850551655,
+ "loss": 1.8166,
+ "theoretical_loss": 4.58489908446087,
+ "tokens_seen": 140771328
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007736008024072217,
+ "loss": 1.7575,
+ "theoretical_loss": 4.584619228148755,
+ "tokens_seen": 140836864
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 53917,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8665790557861328,
+ "objective/train/theoretical_loss": 4.584339538476678,
+ "objective/train/tokens_used": 161362400,
+ "theoretical_loss": 4.584339538476678,
+ "tokens_seen": 140902400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007735847542627884,
+ "loss": 1.8941,
+ "theoretical_loss": 4.584339538476678,
+ "tokens_seen": 140902400
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007735687061183552,
+ "loss": 1.6903,
+ "theoretical_loss": 4.5840600152679745,
+ "tokens_seen": 140967936
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007735526579739218,
+ "loss": 1.8307,
+ "theoretical_loss": 4.583780658346252,
+ "tokens_seen": 141033472
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007735366098294886,
+ "loss": 1.7627,
+ "theoretical_loss": 4.583501467535383,
+ "tokens_seen": 141099008
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007735205616850552,
+ "loss": 1.6755,
+ "theoretical_loss": 4.583222442659514,
+ "tokens_seen": 141164544
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773504513540622,
+ "loss": 1.8388,
+ "theoretical_loss": 4.5829435835430505,
+ "tokens_seen": 141230080
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007734884653961886,
+ "loss": 1.833,
+ "theoretical_loss": 4.582664890010674,
+ "tokens_seen": 141295616
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007734724172517554,
+ "loss": 1.9159,
+ "theoretical_loss": 4.582386361887329,
+ "tokens_seen": 141361152
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773456369107322,
+ "loss": 1.7725,
+ "theoretical_loss": 4.582107998998225,
+ "tokens_seen": 141426688
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007734403209628888,
+ "loss": 1.8281,
+ "theoretical_loss": 4.581829801168839,
+ "tokens_seen": 141492224
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007734242728184554,
+ "loss": 1.7452,
+ "theoretical_loss": 4.5815517682249105,
+ "tokens_seen": 141557760
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007734082246740222,
+ "loss": 1.8201,
+ "theoretical_loss": 4.581273899992447,
+ "tokens_seen": 141623296
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007733921765295888,
+ "loss": 1.8788,
+ "theoretical_loss": 4.580996196297718,
+ "tokens_seen": 141688832
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007733761283851556,
+ "loss": 1.8429,
+ "theoretical_loss": 4.580718656967257,
+ "tokens_seen": 141754368
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007733600802407222,
+ "loss": 1.8612,
+ "theoretical_loss": 4.580441281827858,
+ "tokens_seen": 141819904
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773344032096289,
+ "loss": 1.7208,
+ "theoretical_loss": 4.580164070706584,
+ "tokens_seen": 141885440
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007733279839518556,
+ "loss": 1.7845,
+ "theoretical_loss": 4.579887023430752,
+ "tokens_seen": 141950976
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007733119358074224,
+ "loss": 1.7119,
+ "theoretical_loss": 4.579610139827945,
+ "tokens_seen": 142016512
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773295887662989,
+ "loss": 1.7464,
+ "theoretical_loss": 4.579333419726007,
+ "tokens_seen": 142082048
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007732798395185558,
+ "loss": 1.8061,
+ "theoretical_loss": 4.579056862953039,
+ "tokens_seen": 142147584
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007732637913741224,
+ "loss": 1.8128,
+ "theoretical_loss": 4.578780469337407,
+ "tokens_seen": 142213120
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007732477432296892,
+ "loss": 1.77,
+ "theoretical_loss": 4.5785042387077315,
+ "tokens_seen": 142278656
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007732316950852558,
+ "loss": 1.8202,
+ "theoretical_loss": 4.578228170892896,
+ "tokens_seen": 142344192
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007732156469408226,
+ "loss": 1.9073,
+ "theoretical_loss": 4.577952265722038,
+ "tokens_seen": 142409728
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007731995987963892,
+ "loss": 1.8577,
+ "theoretical_loss": 4.5776765230245555,
+ "tokens_seen": 142475264
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773183550651956,
+ "loss": 1.8928,
+ "theoretical_loss": 4.5774009426301046,
+ "tokens_seen": 142540800
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007731675025075226,
+ "loss": 1.8294,
+ "theoretical_loss": 4.577125524368595,
+ "tokens_seen": 142606336
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007731514543630894,
+ "loss": 1.7549,
+ "theoretical_loss": 4.5768502680701975,
+ "tokens_seen": 142671872
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000773135406218656,
+ "loss": 1.8273,
+ "theoretical_loss": 4.576575173565334,
+ "tokens_seen": 142737408
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007731193580742227,
+ "loss": 1.8096,
+ "theoretical_loss": 4.5763002406846836,
+ "tokens_seen": 142802944
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007731033099297894,
+ "loss": 1.8756,
+ "theoretical_loss": 4.57602546925918,
+ "tokens_seen": 142868480
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730872617853561,
+ "loss": 1.6733,
+ "theoretical_loss": 4.575750859120014,
+ "tokens_seen": 142934016
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730712136409228,
+ "loss": 1.8921,
+ "theoretical_loss": 4.575476410098624,
+ "tokens_seen": 142999552
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730551654964895,
+ "loss": 1.7704,
+ "theoretical_loss": 4.575202122026706,
+ "tokens_seen": 143065088
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730391173520562,
+ "loss": 1.773,
+ "theoretical_loss": 4.5749279947362105,
+ "tokens_seen": 143130624
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730230692076229,
+ "loss": 1.8325,
+ "theoretical_loss": 4.574654028059336,
+ "tokens_seen": 143196160
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007730070210631896,
+ "loss": 1.8279,
+ "theoretical_loss": 4.574380221828535,
+ "tokens_seen": 143261696
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729909729187563,
+ "loss": 1.7535,
+ "theoretical_loss": 4.574106575876511,
+ "tokens_seen": 143327232
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729749247743229,
+ "loss": 1.8514,
+ "theoretical_loss": 4.573833090036219,
+ "tokens_seen": 143392768
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729588766298897,
+ "loss": 1.8637,
+ "theoretical_loss": 4.573559764140864,
+ "tokens_seen": 143458304
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729428284854563,
+ "loss": 1.6962,
+ "theoretical_loss": 4.573286598023898,
+ "tokens_seen": 143523840
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729267803410231,
+ "loss": 1.8597,
+ "theoretical_loss": 4.573013591519029,
+ "tokens_seen": 143589376
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007729107321965897,
+ "loss": 1.9061,
+ "theoretical_loss": 4.572740744460207,
+ "tokens_seen": 143654912
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728946840521565,
+ "loss": 1.9639,
+ "theoretical_loss": 4.572468056681634,
+ "tokens_seen": 143720448
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728786359077231,
+ "loss": 1.8062,
+ "theoretical_loss": 4.57219552801776,
+ "tokens_seen": 143785984
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728625877632899,
+ "loss": 1.791,
+ "theoretical_loss": 4.57192315830328,
+ "tokens_seen": 143851520
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728465396188565,
+ "loss": 1.7563,
+ "theoretical_loss": 4.571650947373138,
+ "tokens_seen": 143917056
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728304914744233,
+ "loss": 1.8727,
+ "theoretical_loss": 4.571378895062525,
+ "tokens_seen": 143982592
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007728144433299899,
+ "loss": 1.7582,
+ "theoretical_loss": 4.571107001206875,
+ "tokens_seen": 144048128
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727983951855567,
+ "loss": 1.885,
+ "theoretical_loss": 4.570835265641872,
+ "tokens_seen": 144113664
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 54793,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8981997966766357,
+ "objective/train/theoretical_loss": 4.57056368820344,
+ "objective/train/tokens_used": 164639200,
+ "theoretical_loss": 4.57056368820344,
+ "tokens_seen": 144179200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727823470411233,
+ "loss": 1.7713,
+ "theoretical_loss": 4.57056368820344,
+ "tokens_seen": 144179200
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727662988966901,
+ "loss": 1.722,
+ "theoretical_loss": 4.570292268727751,
+ "tokens_seen": 144244736
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727502507522567,
+ "loss": 1.7943,
+ "theoretical_loss": 4.570021007051221,
+ "tokens_seen": 144310272
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727342026078235,
+ "loss": 1.7481,
+ "theoretical_loss": 4.569749903010507,
+ "tokens_seen": 144375808
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727181544633901,
+ "loss": 1.7384,
+ "theoretical_loss": 4.569478956442513,
+ "tokens_seen": 144441344
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007727021063189569,
+ "loss": 2.0361,
+ "theoretical_loss": 4.569208167184382,
+ "tokens_seen": 144506880
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726860581745235,
+ "loss": 1.7303,
+ "theoretical_loss": 4.5689375350735,
+ "tokens_seen": 144572416
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726700100300903,
+ "loss": 1.7676,
+ "theoretical_loss": 4.568667059947496,
+ "tokens_seen": 144637952
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726539618856569,
+ "loss": 1.8165,
+ "theoretical_loss": 4.568396741644241,
+ "tokens_seen": 144703488
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726379137412237,
+ "loss": 1.8475,
+ "theoretical_loss": 4.568126580001843,
+ "tokens_seen": 144769024
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726218655967903,
+ "loss": 1.6271,
+ "theoretical_loss": 4.567856574858656,
+ "tokens_seen": 144834560
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007726058174523571,
+ "loss": 1.9387,
+ "theoretical_loss": 4.567586726053266,
+ "tokens_seen": 144900096
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725897693079237,
+ "loss": 1.7635,
+ "theoretical_loss": 4.567317033424507,
+ "tokens_seen": 144965632
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725737211634905,
+ "loss": 1.7297,
+ "theoretical_loss": 4.567047496811445,
+ "tokens_seen": 145031168
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725576730190571,
+ "loss": 1.8948,
+ "theoretical_loss": 4.566778116053389,
+ "tokens_seen": 145096704
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725416248746239,
+ "loss": 1.9421,
+ "theoretical_loss": 4.566508890989883,
+ "tokens_seen": 145162240
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725255767301905,
+ "loss": 1.8752,
+ "theoretical_loss": 4.566239821460712,
+ "tokens_seen": 145227776
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007725095285857573,
+ "loss": 1.7984,
+ "theoretical_loss": 4.5659709073058945,
+ "tokens_seen": 145293312
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.000772493480441324,
+ "loss": 1.8638,
+ "theoretical_loss": 4.565702148365686,
+ "tokens_seen": 145358848
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007724774322968907,
+ "loss": 1.7664,
+ "theoretical_loss": 4.56543354448058,
+ "tokens_seen": 145424384
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007724613841524574,
+ "loss": 1.842,
+ "theoretical_loss": 4.565165095491306,
+ "tokens_seen": 145489920
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007724453360080241,
+ "loss": 1.7626,
+ "theoretical_loss": 4.5648968012388265,
+ "tokens_seen": 145555456
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007724292878635908,
+ "loss": 1.7205,
+ "theoretical_loss": 4.56462866156434,
+ "tokens_seen": 145620992
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007724132397191575,
+ "loss": 1.7857,
+ "theoretical_loss": 4.564360676309279,
+ "tokens_seen": 145686528
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723971915747242,
+ "loss": 1.8482,
+ "theoretical_loss": 4.564092845315312,
+ "tokens_seen": 145752064
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723811434302909,
+ "loss": 1.7422,
+ "theoretical_loss": 4.563825168424337,
+ "tokens_seen": 145817600
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723650952858577,
+ "loss": 1.7175,
+ "theoretical_loss": 4.563557645478488,
+ "tokens_seen": 145883136
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723490471414243,
+ "loss": 1.8653,
+ "theoretical_loss": 4.56329027632013,
+ "tokens_seen": 145948672
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723329989969911,
+ "loss": 1.7809,
+ "theoretical_loss": 4.563023060791862,
+ "tokens_seen": 146014208
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723169508525577,
+ "loss": 1.729,
+ "theoretical_loss": 4.562755998736511,
+ "tokens_seen": 146079744
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007723009027081245,
+ "loss": 1.7853,
+ "theoretical_loss": 4.562489089997141,
+ "tokens_seen": 146145280
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722848545636911,
+ "loss": 1.7846,
+ "theoretical_loss": 4.562222334417041,
+ "tokens_seen": 146210816
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722688064192579,
+ "loss": 1.7191,
+ "theoretical_loss": 4.561955731839734,
+ "tokens_seen": 146276352
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722527582748245,
+ "loss": 1.8433,
+ "theoretical_loss": 4.561689282108971,
+ "tokens_seen": 146341888
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722367101303913,
+ "loss": 1.818,
+ "theoretical_loss": 4.561422985068733,
+ "tokens_seen": 146407424
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722206619859579,
+ "loss": 1.7675,
+ "theoretical_loss": 4.561156840563231,
+ "tokens_seen": 146472960
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007722046138415247,
+ "loss": 1.836,
+ "theoretical_loss": 4.560890848436902,
+ "tokens_seen": 146538496
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721885656970913,
+ "loss": 1.8614,
+ "theoretical_loss": 4.560625008534414,
+ "tokens_seen": 146604032
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721725175526581,
+ "loss": 1.824,
+ "theoretical_loss": 4.560359320700662,
+ "tokens_seen": 146669568
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721564694082247,
+ "loss": 1.7436,
+ "theoretical_loss": 4.560093784780766,
+ "tokens_seen": 146735104
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721404212637915,
+ "loss": 1.8323,
+ "theoretical_loss": 4.559828400620075,
+ "tokens_seen": 146800640
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721243731193581,
+ "loss": 1.89,
+ "theoretical_loss": 4.559563168064165,
+ "tokens_seen": 146866176
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007721083249749249,
+ "loss": 1.9487,
+ "theoretical_loss": 4.559298086958837,
+ "tokens_seen": 146931712
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720922768304915,
+ "loss": 1.6885,
+ "theoretical_loss": 4.559033157150115,
+ "tokens_seen": 146997248
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720762286860583,
+ "loss": 1.8212,
+ "theoretical_loss": 4.5587683784842525,
+ "tokens_seen": 147062784
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720601805416249,
+ "loss": 1.7993,
+ "theoretical_loss": 4.558503750807725,
+ "tokens_seen": 147128320
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720441323971917,
+ "loss": 1.7828,
+ "theoretical_loss": 4.558239273967234,
+ "tokens_seen": 147193856
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720280842527583,
+ "loss": 1.8936,
+ "theoretical_loss": 4.5579749478097025,
+ "tokens_seen": 147259392
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007720120361083251,
+ "loss": 1.8044,
+ "theoretical_loss": 4.557710772182278,
+ "tokens_seen": 147324928
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719959879638917,
+ "loss": 1.8211,
+ "theoretical_loss": 4.5574467469323325,
+ "tokens_seen": 147390464
+ },
+ {
+ "epoch": 0.04,
+ "objective/train/docs_used": 56056,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8968627452850342,
+ "objective/train/theoretical_loss": 4.557182871907457,
+ "objective/train/tokens_used": 167916000,
+ "theoretical_loss": 4.557182871907457,
+ "tokens_seen": 147456000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719799398194585,
+ "loss": 1.8249,
+ "theoretical_loss": 4.557182871907457,
+ "tokens_seen": 147456000
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719638916750251,
+ "loss": 1.7435,
+ "theoretical_loss": 4.556919146955469,
+ "tokens_seen": 147521536
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719478435305919,
+ "loss": 1.7664,
+ "theoretical_loss": 4.556655571924404,
+ "tokens_seen": 147587072
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719317953861585,
+ "loss": 1.8627,
+ "theoretical_loss": 4.556392146662521,
+ "tokens_seen": 147652608
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007719157472417253,
+ "loss": 1.8259,
+ "theoretical_loss": 4.556128871018297,
+ "tokens_seen": 147718144
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718996990972919,
+ "loss": 2.0033,
+ "theoretical_loss": 4.555865744840434,
+ "tokens_seen": 147783680
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718836509528587,
+ "loss": 1.698,
+ "theoretical_loss": 4.55560276797785,
+ "tokens_seen": 147849216
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718676028084253,
+ "loss": 1.7972,
+ "theoretical_loss": 4.5553399402796835,
+ "tokens_seen": 147914752
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718515546639921,
+ "loss": 1.7489,
+ "theoretical_loss": 4.5550772615952955,
+ "tokens_seen": 147980288
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718355065195587,
+ "loss": 1.8099,
+ "theoretical_loss": 4.554814731774259,
+ "tokens_seen": 148045824
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718194583751255,
+ "loss": 1.8885,
+ "theoretical_loss": 4.554552350666373,
+ "tokens_seen": 148111360
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007718034102306921,
+ "loss": 1.8405,
+ "theoretical_loss": 4.554290118121649,
+ "tokens_seen": 148176896
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007717873620862589,
+ "loss": 1.8405,
+ "theoretical_loss": 4.554028033990319,
+ "tokens_seen": 148242432
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007717713139418255,
+ "loss": 1.81,
+ "theoretical_loss": 4.5537660981228285,
+ "tokens_seen": 148307968
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007717552657973923,
+ "loss": 1.8125,
+ "theoretical_loss": 4.553504310369844,
+ "tokens_seen": 148373504
+ },
+ {
+ "epoch": 0.04,
+ "learning_rate": 0.0007717392176529589,
+ "loss": 1.8753,
+ "theoretical_loss": 4.553242670582245,
+ "tokens_seen": 148439040
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007717231695085256,
+ "loss": 1.9341,
+ "theoretical_loss": 4.55298117861113,
+ "tokens_seen": 148504576
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007717071213640923,
+ "loss": 1.7652,
+ "theoretical_loss": 4.55271983430781,
+ "tokens_seen": 148570112
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771691073219659,
+ "loss": 1.9011,
+ "theoretical_loss": 4.552458637523815,
+ "tokens_seen": 148635648
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007716750250752257,
+ "loss": 1.7245,
+ "theoretical_loss": 4.552197588110882,
+ "tokens_seen": 148701184
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007716589769307924,
+ "loss": 1.736,
+ "theoretical_loss": 4.551936685920971,
+ "tokens_seen": 148766720
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007716429287863591,
+ "loss": 1.6701,
+ "theoretical_loss": 4.551675930806251,
+ "tokens_seen": 148832256
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007716268806419258,
+ "loss": 1.8453,
+ "theoretical_loss": 4.551415322619107,
+ "tokens_seen": 148897792
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007716108324974925,
+ "loss": 1.8299,
+ "theoretical_loss": 4.551154861212133,
+ "tokens_seen": 148963328
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007715947843530592,
+ "loss": 1.97,
+ "theoretical_loss": 4.550894546438139,
+ "tokens_seen": 149028864
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007715787362086258,
+ "loss": 1.7303,
+ "theoretical_loss": 4.550634378150148,
+ "tokens_seen": 149094400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007715626880641926,
+ "loss": 1.8074,
+ "theoretical_loss": 4.550374356201392,
+ "tokens_seen": 149159936
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007715466399197592,
+ "loss": 1.8377,
+ "theoretical_loss": 4.550114480445316,
+ "tokens_seen": 149225472
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771530591775326,
+ "loss": 1.8604,
+ "theoretical_loss": 4.549854750735575,
+ "tokens_seen": 149291008
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007715145436308926,
+ "loss": 1.9376,
+ "theoretical_loss": 4.5495951669260375,
+ "tokens_seen": 149356544
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714984954864594,
+ "loss": 1.8481,
+ "theoretical_loss": 4.54933572887078,
+ "tokens_seen": 149422080
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771482447342026,
+ "loss": 1.8748,
+ "theoretical_loss": 4.5490764364240865,
+ "tokens_seen": 149487616
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714663991975928,
+ "loss": 1.7571,
+ "theoretical_loss": 4.548817289440455,
+ "tokens_seen": 149553152
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714503510531594,
+ "loss": 1.7867,
+ "theoretical_loss": 4.548558287774592,
+ "tokens_seen": 149618688
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714343029087262,
+ "loss": 1.857,
+ "theoretical_loss": 4.548299431281409,
+ "tokens_seen": 149684224
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714182547642928,
+ "loss": 1.773,
+ "theoretical_loss": 4.548040719816029,
+ "tokens_seen": 149749760
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007714022066198596,
+ "loss": 1.8777,
+ "theoretical_loss": 4.547782153233783,
+ "tokens_seen": 149815296
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007713861584754262,
+ "loss": 1.7381,
+ "theoretical_loss": 4.547523731390206,
+ "tokens_seen": 149880832
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771370110330993,
+ "loss": 1.7556,
+ "theoretical_loss": 4.547265454141046,
+ "tokens_seen": 149946368
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007713540621865596,
+ "loss": 1.7892,
+ "theoretical_loss": 4.547007321342251,
+ "tokens_seen": 150011904
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007713380140421264,
+ "loss": 1.8294,
+ "theoretical_loss": 4.546749332849982,
+ "tokens_seen": 150077440
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771321965897693,
+ "loss": 1.7976,
+ "theoretical_loss": 4.546491488520601,
+ "tokens_seen": 150142976
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007713059177532598,
+ "loss": 1.7639,
+ "theoretical_loss": 4.546233788210679,
+ "tokens_seen": 150208512
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007712898696088264,
+ "loss": 1.7777,
+ "theoretical_loss": 4.545976231776988,
+ "tokens_seen": 150274048
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007712738214643932,
+ "loss": 1.8345,
+ "theoretical_loss": 4.545718819076509,
+ "tokens_seen": 150339584
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007712577733199598,
+ "loss": 1.8569,
+ "theoretical_loss": 4.545461549966426,
+ "tokens_seen": 150405120
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007712417251755266,
+ "loss": 1.7723,
+ "theoretical_loss": 4.545204424304126,
+ "tokens_seen": 150470656
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007712256770310932,
+ "loss": 1.7852,
+ "theoretical_loss": 4.544947441947201,
+ "tokens_seen": 150536192
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.00077120962888666,
+ "loss": 1.7586,
+ "theoretical_loss": 4.544690602753445,
+ "tokens_seen": 150601728
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711935807422266,
+ "loss": 1.8189,
+ "theoretical_loss": 4.544433906580857,
+ "tokens_seen": 150667264
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 56915,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7465159893035889,
+ "objective/train/theoretical_loss": 4.544177353287637,
+ "objective/train/tokens_used": 171192800,
+ "theoretical_loss": 4.544177353287637,
+ "tokens_seen": 150732800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711775325977934,
+ "loss": 1.7764,
+ "theoretical_loss": 4.544177353287637,
+ "tokens_seen": 150732800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711614844533602,
+ "loss": 1.8765,
+ "theoretical_loss": 4.543920942732187,
+ "tokens_seen": 150798336
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711454363089268,
+ "loss": 1.7625,
+ "theoretical_loss": 4.543664674773112,
+ "tokens_seen": 150863872
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711293881644936,
+ "loss": 1.8125,
+ "theoretical_loss": 4.543408549269218,
+ "tokens_seen": 150929408
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007711133400200602,
+ "loss": 1.7914,
+ "theoretical_loss": 4.54315256607951,
+ "tokens_seen": 150994944
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771097291875627,
+ "loss": 1.8573,
+ "theoretical_loss": 4.542896725063195,
+ "tokens_seen": 151060480
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007710812437311936,
+ "loss": 1.6647,
+ "theoretical_loss": 4.542641026079683,
+ "tokens_seen": 151126016
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007710651955867604,
+ "loss": 1.8338,
+ "theoretical_loss": 4.542385468988581,
+ "tokens_seen": 151191552
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000771049147442327,
+ "loss": 1.8271,
+ "theoretical_loss": 4.542130053649695,
+ "tokens_seen": 151257088
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007710330992978938,
+ "loss": 1.6977,
+ "theoretical_loss": 4.541874779923031,
+ "tokens_seen": 151322624
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007710170511534604,
+ "loss": 1.7345,
+ "theoretical_loss": 4.541619647668796,
+ "tokens_seen": 151388160
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007710010030090272,
+ "loss": 1.9003,
+ "theoretical_loss": 4.541364656747392,
+ "tokens_seen": 151453696
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007709849548645938,
+ "loss": 1.7489,
+ "theoretical_loss": 4.541109807019422,
+ "tokens_seen": 151519232
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007709689067201606,
+ "loss": 1.7527,
+ "theoretical_loss": 4.540855098345683,
+ "tokens_seen": 151584768
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007709528585757272,
+ "loss": 1.784,
+ "theoretical_loss": 4.540600530587175,
+ "tokens_seen": 151650304
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770936810431294,
+ "loss": 1.7235,
+ "theoretical_loss": 4.5403461036050885,
+ "tokens_seen": 151715840
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007709207622868606,
+ "loss": 1.838,
+ "theoretical_loss": 4.540091817260815,
+ "tokens_seen": 151781376
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007709047141424274,
+ "loss": 1.8029,
+ "theoretical_loss": 4.539837671415941,
+ "tokens_seen": 151846912
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770888665997994,
+ "loss": 1.7071,
+ "theoretical_loss": 4.539583665932249,
+ "tokens_seen": 151912448
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007708726178535608,
+ "loss": 1.7128,
+ "theoretical_loss": 4.539329800671718,
+ "tokens_seen": 151977984
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007708565697091274,
+ "loss": 1.7997,
+ "theoretical_loss": 4.539076075496519,
+ "tokens_seen": 152043520
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007708405215646942,
+ "loss": 1.9835,
+ "theoretical_loss": 4.538822490269021,
+ "tokens_seen": 152109056
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007708244734202608,
+ "loss": 1.7025,
+ "theoretical_loss": 4.538569044851785,
+ "tokens_seen": 152174592
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007708084252758276,
+ "loss": 1.7776,
+ "theoretical_loss": 4.538315739107571,
+ "tokens_seen": 152240128
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007707923771313942,
+ "loss": 1.63,
+ "theoretical_loss": 4.5380625728993245,
+ "tokens_seen": 152305664
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770776328986961,
+ "loss": 1.736,
+ "theoretical_loss": 4.537809546090193,
+ "tokens_seen": 152371200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007707602808425276,
+ "loss": 1.8733,
+ "theoretical_loss": 4.53755665854351,
+ "tokens_seen": 152436736
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007707442326980944,
+ "loss": 1.73,
+ "theoretical_loss": 4.537303910122807,
+ "tokens_seen": 152502272
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770728184553661,
+ "loss": 1.753,
+ "theoretical_loss": 4.537051300691805,
+ "tokens_seen": 152567808
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007707121364092278,
+ "loss": 1.7725,
+ "theoretical_loss": 4.5367988301144155,
+ "tokens_seen": 152633344
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007706960882647944,
+ "loss": 1.7438,
+ "theoretical_loss": 4.536546498254746,
+ "tokens_seen": 152698880
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007706800401203612,
+ "loss": 1.8448,
+ "theoretical_loss": 4.536294304977092,
+ "tokens_seen": 152764416
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007706639919759278,
+ "loss": 1.76,
+ "theoretical_loss": 4.536042250145941,
+ "tokens_seen": 152829952
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007706479438314946,
+ "loss": 1.5725,
+ "theoretical_loss": 4.53579033362597,
+ "tokens_seen": 152895488
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007706318956870612,
+ "loss": 1.8505,
+ "theoretical_loss": 4.535538555282048,
+ "tokens_seen": 152961024
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770615847542628,
+ "loss": 1.9077,
+ "theoretical_loss": 4.535286914979232,
+ "tokens_seen": 153026560
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705997993981946,
+ "loss": 1.7547,
+ "theoretical_loss": 4.5350354125827685,
+ "tokens_seen": 153092096
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705837512537614,
+ "loss": 1.747,
+ "theoretical_loss": 4.534784047958096,
+ "tokens_seen": 153157632
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770567703109328,
+ "loss": 1.7225,
+ "theoretical_loss": 4.534532820970839,
+ "tokens_seen": 153223168
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705516549648948,
+ "loss": 1.7695,
+ "theoretical_loss": 4.534281731486812,
+ "tokens_seen": 153288704
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705356068204614,
+ "loss": 1.7913,
+ "theoretical_loss": 4.5340307793720145,
+ "tokens_seen": 153354240
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705195586760282,
+ "loss": 1.703,
+ "theoretical_loss": 4.533779964492638,
+ "tokens_seen": 153419776
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007705035105315948,
+ "loss": 1.695,
+ "theoretical_loss": 4.533529286715059,
+ "tokens_seen": 153485312
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007704874623871616,
+ "loss": 1.7663,
+ "theoretical_loss": 4.533278745905843,
+ "tokens_seen": 153550848
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007704714142427282,
+ "loss": 1.6136,
+ "theoretical_loss": 4.533028341931738,
+ "tokens_seen": 153616384
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770455366098295,
+ "loss": 1.9402,
+ "theoretical_loss": 4.532778074659685,
+ "tokens_seen": 153681920
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007704393179538616,
+ "loss": 1.8058,
+ "theoretical_loss": 4.532527943956804,
+ "tokens_seen": 153747456
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007704232698094284,
+ "loss": 1.8964,
+ "theoretical_loss": 4.532277949690407,
+ "tokens_seen": 153812992
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770407221664995,
+ "loss": 1.6986,
+ "theoretical_loss": 4.532028091727986,
+ "tokens_seen": 153878528
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703911735205618,
+ "loss": 1.6689,
+ "theoretical_loss": 4.531778369937222,
+ "tokens_seen": 153944064
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 58041,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8966615200042725,
+ "objective/train/theoretical_loss": 4.53152878418598,
+ "objective/train/tokens_used": 174469600,
+ "theoretical_loss": 4.53152878418598,
+ "tokens_seen": 154009600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703751253761284,
+ "loss": 1.7864,
+ "theoretical_loss": 4.53152878418598,
+ "tokens_seen": 154009600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703590772316952,
+ "loss": 1.724,
+ "theoretical_loss": 4.531279334342306,
+ "tokens_seen": 154075136
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703430290872618,
+ "loss": 1.7493,
+ "theoretical_loss": 4.531030020274436,
+ "tokens_seen": 154140672
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703269809428285,
+ "loss": 1.7452,
+ "theoretical_loss": 4.530780841850783,
+ "tokens_seen": 154206208
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007703109327983952,
+ "loss": 1.7928,
+ "theoretical_loss": 4.5305317989399505,
+ "tokens_seen": 154271744
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007702948846539619,
+ "loss": 1.7797,
+ "theoretical_loss": 4.530282891410717,
+ "tokens_seen": 154337280
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007702788365095286,
+ "loss": 1.7927,
+ "theoretical_loss": 4.530034119132052,
+ "tokens_seen": 154402816
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007702627883650953,
+ "loss": 1.7336,
+ "theoretical_loss": 4.529785481973102,
+ "tokens_seen": 154468352
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000770246740220662,
+ "loss": 1.7642,
+ "theoretical_loss": 4.529536979803195,
+ "tokens_seen": 154533888
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007702306920762287,
+ "loss": 1.9063,
+ "theoretical_loss": 4.529288612491846,
+ "tokens_seen": 154599424
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007702146439317954,
+ "loss": 1.6956,
+ "theoretical_loss": 4.5290403799087455,
+ "tokens_seen": 154664960
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701985957873621,
+ "loss": 1.9776,
+ "theoretical_loss": 4.528792281923769,
+ "tokens_seen": 154730496
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701825476429287,
+ "loss": 1.8414,
+ "theoretical_loss": 4.52854431840697,
+ "tokens_seen": 154796032
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701664994984955,
+ "loss": 1.8142,
+ "theoretical_loss": 4.528296489228585,
+ "tokens_seen": 154861568
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701504513540621,
+ "loss": 1.6601,
+ "theoretical_loss": 4.528048794259028,
+ "tokens_seen": 154927104
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701344032096289,
+ "loss": 1.8983,
+ "theoretical_loss": 4.527801233368896,
+ "tokens_seen": 154992640
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701183550651955,
+ "loss": 1.7475,
+ "theoretical_loss": 4.527553806428962,
+ "tokens_seen": 155058176
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007701023069207623,
+ "loss": 1.8392,
+ "theoretical_loss": 4.5273065133101795,
+ "tokens_seen": 155123712
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700862587763289,
+ "loss": 1.7844,
+ "theoretical_loss": 4.527059353883682,
+ "tokens_seen": 155189248
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700702106318957,
+ "loss": 1.8887,
+ "theoretical_loss": 4.526812328020778,
+ "tokens_seen": 155254784
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700541624874623,
+ "loss": 1.736,
+ "theoretical_loss": 4.52656543559296,
+ "tokens_seen": 155320320
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700381143430291,
+ "loss": 1.7517,
+ "theoretical_loss": 4.526318676471891,
+ "tokens_seen": 155385856
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700220661985957,
+ "loss": 1.7906,
+ "theoretical_loss": 4.526072050529417,
+ "tokens_seen": 155451392
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007700060180541625,
+ "loss": 1.8339,
+ "theoretical_loss": 4.52582555763756,
+ "tokens_seen": 155516928
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007699899699097291,
+ "loss": 1.8402,
+ "theoretical_loss": 4.525579197668517,
+ "tokens_seen": 155582464
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007699739217652959,
+ "loss": 1.8406,
+ "theoretical_loss": 4.525332970494663,
+ "tokens_seen": 155648000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007699578736208627,
+ "loss": 1.793,
+ "theoretical_loss": 4.525086875988549,
+ "tokens_seen": 155713536
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007699418254764293,
+ "loss": 1.8599,
+ "theoretical_loss": 4.524840914022902,
+ "tokens_seen": 155779072
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000769925777331996,
+ "loss": 1.7589,
+ "theoretical_loss": 4.524595084470625,
+ "tokens_seen": 155844608
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007699097291875627,
+ "loss": 1.8455,
+ "theoretical_loss": 4.524349387204794,
+ "tokens_seen": 155910144
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698936810431295,
+ "loss": 1.6923,
+ "theoretical_loss": 4.524103822098664,
+ "tokens_seen": 155975680
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698776328986961,
+ "loss": 1.7622,
+ "theoretical_loss": 4.52385838902566,
+ "tokens_seen": 156041216
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698615847542629,
+ "loss": 1.6998,
+ "theoretical_loss": 4.523613087859386,
+ "tokens_seen": 156106752
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698455366098295,
+ "loss": 1.7923,
+ "theoretical_loss": 4.523367918473614,
+ "tokens_seen": 156172288
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698294884653963,
+ "loss": 1.814,
+ "theoretical_loss": 4.523122880742298,
+ "tokens_seen": 156237824
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007698134403209629,
+ "loss": 1.7696,
+ "theoretical_loss": 4.522877974539556,
+ "tokens_seen": 156303360
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697973921765297,
+ "loss": 1.8467,
+ "theoretical_loss": 4.522633199739687,
+ "tokens_seen": 156368896
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697813440320963,
+ "loss": 1.7256,
+ "theoretical_loss": 4.522388556217157,
+ "tokens_seen": 156434432
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697652958876631,
+ "loss": 1.7527,
+ "theoretical_loss": 4.522144043846609,
+ "tokens_seen": 156499968
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697492477432297,
+ "loss": 1.784,
+ "theoretical_loss": 4.521899662502856,
+ "tokens_seen": 156565504
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697331995987965,
+ "loss": 1.7029,
+ "theoretical_loss": 4.5216554120608805,
+ "tokens_seen": 156631040
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697171514543631,
+ "loss": 1.7115,
+ "theoretical_loss": 4.521411292395841,
+ "tokens_seen": 156696576
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007697011033099299,
+ "loss": 1.7872,
+ "theoretical_loss": 4.521167303383065,
+ "tokens_seen": 156762112
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696850551654965,
+ "loss": 1.6923,
+ "theoretical_loss": 4.520923444898051,
+ "tokens_seen": 156827648
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696690070210633,
+ "loss": 1.7266,
+ "theoretical_loss": 4.520679716816467,
+ "tokens_seen": 156893184
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696529588766299,
+ "loss": 1.7961,
+ "theoretical_loss": 4.520436119014155,
+ "tokens_seen": 156958720
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696369107321967,
+ "loss": 1.7192,
+ "theoretical_loss": 4.520192651367121,
+ "tokens_seen": 157024256
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696208625877633,
+ "loss": 1.5909,
+ "theoretical_loss": 4.519949313751547,
+ "tokens_seen": 157089792
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007696048144433301,
+ "loss": 1.8818,
+ "theoretical_loss": 4.519706106043779,
+ "tokens_seen": 157155328
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695887662988967,
+ "loss": 1.7198,
+ "theoretical_loss": 4.519463028120338,
+ "tokens_seen": 157220864
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 59100,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.76185941696167,
+ "objective/train/theoretical_loss": 4.519220079857908,
+ "objective/train/tokens_used": 177746400,
+ "theoretical_loss": 4.519220079857908,
+ "tokens_seen": 157286400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695727181544635,
+ "loss": 1.7576,
+ "theoretical_loss": 4.519220079857908,
+ "tokens_seen": 157286400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695566700100301,
+ "loss": 1.7981,
+ "theoretical_loss": 4.518977261133343,
+ "tokens_seen": 157351936
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695406218655969,
+ "loss": 1.6934,
+ "theoretical_loss": 4.518734571823668,
+ "tokens_seen": 157417472
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695245737211635,
+ "loss": 1.7905,
+ "theoretical_loss": 4.518492011806073,
+ "tokens_seen": 157483008
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007695085255767303,
+ "loss": 1.7677,
+ "theoretical_loss": 4.518249580957917,
+ "tokens_seen": 157548544
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694924774322969,
+ "loss": 1.8178,
+ "theoretical_loss": 4.518007279156725,
+ "tokens_seen": 157614080
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694764292878637,
+ "loss": 1.8327,
+ "theoretical_loss": 4.5177651062801925,
+ "tokens_seen": 157679616
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694603811434303,
+ "loss": 1.8089,
+ "theoretical_loss": 4.517523062206175,
+ "tokens_seen": 157745152
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694443329989971,
+ "loss": 1.7748,
+ "theoretical_loss": 4.517281146812701,
+ "tokens_seen": 157810688
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694282848545637,
+ "loss": 1.8415,
+ "theoretical_loss": 4.517039359977961,
+ "tokens_seen": 157876224
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007694122367101305,
+ "loss": 1.8614,
+ "theoretical_loss": 4.516797701580314,
+ "tokens_seen": 157941760
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693961885656971,
+ "loss": 1.6887,
+ "theoretical_loss": 4.516556171498284,
+ "tokens_seen": 158007296
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693801404212639,
+ "loss": 1.6983,
+ "theoretical_loss": 4.516314769610557,
+ "tokens_seen": 158072832
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693640922768305,
+ "loss": 1.693,
+ "theoretical_loss": 4.516073495795988,
+ "tokens_seen": 158138368
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693480441323973,
+ "loss": 1.8146,
+ "theoretical_loss": 4.5158323499335955,
+ "tokens_seen": 158203904
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693319959879639,
+ "loss": 1.7686,
+ "theoretical_loss": 4.515591331902561,
+ "tokens_seen": 158269440
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007693159478435307,
+ "loss": 1.6949,
+ "theoretical_loss": 4.515350441582231,
+ "tokens_seen": 158334976
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692998996990973,
+ "loss": 1.8541,
+ "theoretical_loss": 4.515109678852117,
+ "tokens_seen": 158400512
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692838515546641,
+ "loss": 1.9401,
+ "theoretical_loss": 4.514869043591891,
+ "tokens_seen": 158466048
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692678034102307,
+ "loss": 1.7639,
+ "theoretical_loss": 4.514628535681391,
+ "tokens_seen": 158531584
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692517552657975,
+ "loss": 1.7224,
+ "theoretical_loss": 4.514388155000615,
+ "tokens_seen": 158597120
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692357071213641,
+ "loss": 1.76,
+ "theoretical_loss": 4.5141479014297285,
+ "tokens_seen": 158662656
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692196589769309,
+ "loss": 1.8276,
+ "theoretical_loss": 4.513907774849054,
+ "tokens_seen": 158728192
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007692036108324975,
+ "loss": 1.8055,
+ "theoretical_loss": 4.513667775139078,
+ "tokens_seen": 158793728
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691875626880643,
+ "loss": 1.7416,
+ "theoretical_loss": 4.5134279021804495,
+ "tokens_seen": 158859264
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691715145436309,
+ "loss": 1.675,
+ "theoretical_loss": 4.513188155853978,
+ "tokens_seen": 158924800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691554663991977,
+ "loss": 1.8391,
+ "theoretical_loss": 4.512948536040636,
+ "tokens_seen": 158990336
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691394182547643,
+ "loss": 1.8972,
+ "theoretical_loss": 4.512709042621554,
+ "tokens_seen": 159055872
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691233701103311,
+ "loss": 1.7549,
+ "theoretical_loss": 4.512469675478026,
+ "tokens_seen": 159121408
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007691073219658977,
+ "loss": 1.7728,
+ "theoretical_loss": 4.512230434491503,
+ "tokens_seen": 159186944
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690912738214645,
+ "loss": 1.8985,
+ "theoretical_loss": 4.511991319543599,
+ "tokens_seen": 159252480
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690752256770311,
+ "loss": 1.7964,
+ "theoretical_loss": 4.511752330516086,
+ "tokens_seen": 159318016
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690591775325979,
+ "loss": 1.658,
+ "theoretical_loss": 4.5115134672908965,
+ "tokens_seen": 159383552
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690431293881645,
+ "loss": 1.9224,
+ "theoretical_loss": 4.511274729750123,
+ "tokens_seen": 159449088
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690270812437313,
+ "loss": 1.7319,
+ "theoretical_loss": 4.5110361177760145,
+ "tokens_seen": 159514624
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007690110330992979,
+ "loss": 1.7229,
+ "theoretical_loss": 4.5107976312509805,
+ "tokens_seen": 159580160
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689949849548647,
+ "loss": 1.7498,
+ "theoretical_loss": 4.510559270057587,
+ "tokens_seen": 159645696
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689789368104313,
+ "loss": 1.7295,
+ "theoretical_loss": 4.51032103407856,
+ "tokens_seen": 159711232
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689628886659981,
+ "loss": 1.7314,
+ "theoretical_loss": 4.510082923196784,
+ "tokens_seen": 159776768
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689468405215647,
+ "loss": 1.7616,
+ "theoretical_loss": 4.5098449372952985,
+ "tokens_seen": 159842304
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689307923771315,
+ "loss": 1.7165,
+ "theoretical_loss": 4.5096070762573,
+ "tokens_seen": 159907840
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007689147442326981,
+ "loss": 1.7341,
+ "theoretical_loss": 4.509369339966146,
+ "tokens_seen": 159973376
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688986960882648,
+ "loss": 1.7801,
+ "theoretical_loss": 4.509131728305347,
+ "tokens_seen": 160038912
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688826479438315,
+ "loss": 1.9064,
+ "theoretical_loss": 4.50889424115857,
+ "tokens_seen": 160104448
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688665997993982,
+ "loss": 1.6784,
+ "theoretical_loss": 4.50865687840964,
+ "tokens_seen": 160169984
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688505516549649,
+ "loss": 1.7526,
+ "theoretical_loss": 4.508419639942538,
+ "tokens_seen": 160235520
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688345035105316,
+ "loss": 1.7459,
+ "theoretical_loss": 4.5081825256413985,
+ "tokens_seen": 160301056
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007688184553660983,
+ "loss": 1.8808,
+ "theoretical_loss": 4.507945535390512,
+ "tokens_seen": 160366592
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768802407221665,
+ "loss": 1.816,
+ "theoretical_loss": 4.507708669074326,
+ "tokens_seen": 160432128
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687863590772316,
+ "loss": 1.7083,
+ "theoretical_loss": 4.5074719265774394,
+ "tokens_seen": 160497664
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 60314,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.1015968322753906,
+ "objective/train/theoretical_loss": 4.507235307784608,
+ "objective/train/tokens_used": 181023200,
+ "theoretical_loss": 4.507235307784608,
+ "tokens_seen": 160563200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687703109327984,
+ "loss": 1.9357,
+ "theoretical_loss": 4.507235307784608,
+ "tokens_seen": 160563200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687542627883651,
+ "loss": 1.6547,
+ "theoretical_loss": 4.5069988125807425,
+ "tokens_seen": 160628736
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687382146439318,
+ "loss": 1.7709,
+ "theoretical_loss": 4.506762440850906,
+ "tokens_seen": 160694272
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687221664994985,
+ "loss": 1.7327,
+ "theoretical_loss": 4.506526192480315,
+ "tokens_seen": 160759808
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007687061183550652,
+ "loss": 1.7753,
+ "theoretical_loss": 4.506290067354341,
+ "tokens_seen": 160825344
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768690070210632,
+ "loss": 1.7335,
+ "theoretical_loss": 4.506054065358508,
+ "tokens_seen": 160890880
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007686740220661986,
+ "loss": 1.7166,
+ "theoretical_loss": 4.505818186378491,
+ "tokens_seen": 160956416
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007686579739217653,
+ "loss": 1.6904,
+ "theoretical_loss": 4.505582430300121,
+ "tokens_seen": 161021952
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768641925777332,
+ "loss": 1.8413,
+ "theoretical_loss": 4.505346797009381,
+ "tokens_seen": 161087488
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007686258776328987,
+ "loss": 1.7868,
+ "theoretical_loss": 4.505111286392402,
+ "tokens_seen": 161153024
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007686098294884654,
+ "loss": 1.6634,
+ "theoretical_loss": 4.50487589833547,
+ "tokens_seen": 161218560
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007685937813440322,
+ "loss": 1.6353,
+ "theoretical_loss": 4.504640632725025,
+ "tokens_seen": 161284096
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007685777331995988,
+ "loss": 1.7892,
+ "theoretical_loss": 4.504405489447654,
+ "tokens_seen": 161349632
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007685616850551656,
+ "loss": 1.7234,
+ "theoretical_loss": 4.504170468390096,
+ "tokens_seen": 161415168
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007685456369107322,
+ "loss": 1.6762,
+ "theoretical_loss": 4.503935569439242,
+ "tokens_seen": 161480704
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768529588766299,
+ "loss": 1.8172,
+ "theoretical_loss": 4.5037007924821335,
+ "tokens_seen": 161546240
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007685135406218656,
+ "loss": 1.7786,
+ "theoretical_loss": 4.503466137405961,
+ "tokens_seen": 161611776
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684974924774324,
+ "loss": 1.732,
+ "theoretical_loss": 4.503231604098065,
+ "tokens_seen": 161677312
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768481444332999,
+ "loss": 1.8209,
+ "theoretical_loss": 4.50299719244594,
+ "tokens_seen": 161742848
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684653961885658,
+ "loss": 1.8043,
+ "theoretical_loss": 4.502762902337222,
+ "tokens_seen": 161808384
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684493480441324,
+ "loss": 1.8455,
+ "theoretical_loss": 4.502528733659704,
+ "tokens_seen": 161873920
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684332998996992,
+ "loss": 1.7825,
+ "theoretical_loss": 4.502294686301323,
+ "tokens_seen": 161939456
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684172517552658,
+ "loss": 1.6838,
+ "theoretical_loss": 4.502060760150167,
+ "tokens_seen": 162004992
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007684012036108326,
+ "loss": 1.8061,
+ "theoretical_loss": 4.501826955094472,
+ "tokens_seen": 162070528
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007683851554663992,
+ "loss": 1.6355,
+ "theoretical_loss": 4.501593271022623,
+ "tokens_seen": 162136064
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768369107321966,
+ "loss": 1.7908,
+ "theoretical_loss": 4.501359707823149,
+ "tokens_seen": 162201600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007683530591775326,
+ "loss": 1.7182,
+ "theoretical_loss": 4.501126265384732,
+ "tokens_seen": 162267136
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007683370110330994,
+ "loss": 1.9053,
+ "theoretical_loss": 4.500892943596199,
+ "tokens_seen": 162332672
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768320962888666,
+ "loss": 1.8499,
+ "theoretical_loss": 4.500659742346524,
+ "tokens_seen": 162398208
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007683049147442328,
+ "loss": 1.7459,
+ "theoretical_loss": 4.500426661524828,
+ "tokens_seen": 162463744
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007682888665997994,
+ "loss": 1.774,
+ "theoretical_loss": 4.500193701020379,
+ "tokens_seen": 162529280
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007682728184553662,
+ "loss": 1.8263,
+ "theoretical_loss": 4.499960860722592,
+ "tokens_seen": 162594816
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007682567703109328,
+ "loss": 1.7356,
+ "theoretical_loss": 4.499728140521025,
+ "tokens_seen": 162660352
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007682407221664996,
+ "loss": 1.8387,
+ "theoretical_loss": 4.499495540305388,
+ "tokens_seen": 162725888
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007682246740220662,
+ "loss": 1.6331,
+ "theoretical_loss": 4.49926305996553,
+ "tokens_seen": 162791424
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768208625877633,
+ "loss": 1.7426,
+ "theoretical_loss": 4.499030699391449,
+ "tokens_seen": 162856960
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007681925777331996,
+ "loss": 1.617,
+ "theoretical_loss": 4.49879845847329,
+ "tokens_seen": 162922496
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007681765295887664,
+ "loss": 1.7124,
+ "theoretical_loss": 4.498566337101337,
+ "tokens_seen": 162988032
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768160481444333,
+ "loss": 1.7712,
+ "theoretical_loss": 4.4983343351660245,
+ "tokens_seen": 163053568
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007681444332998998,
+ "loss": 1.9108,
+ "theoretical_loss": 4.498102452557928,
+ "tokens_seen": 163119104
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007681283851554664,
+ "loss": 1.6287,
+ "theoretical_loss": 4.497870689167771,
+ "tokens_seen": 163184640
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007681123370110332,
+ "loss": 1.8195,
+ "theoretical_loss": 4.497639044886412,
+ "tokens_seen": 163250176
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680962888665998,
+ "loss": 1.7055,
+ "theoretical_loss": 4.497407519604866,
+ "tokens_seen": 163315712
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680802407221666,
+ "loss": 1.8599,
+ "theoretical_loss": 4.49717611321428,
+ "tokens_seen": 163381248
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680641925777332,
+ "loss": 1.6922,
+ "theoretical_loss": 4.496944825605951,
+ "tokens_seen": 163446784
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680481444333,
+ "loss": 1.6333,
+ "theoretical_loss": 4.496713656671317,
+ "tokens_seen": 163512320
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680320962888666,
+ "loss": 1.8145,
+ "theoretical_loss": 4.496482606301957,
+ "tokens_seen": 163577856
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007680160481444334,
+ "loss": 1.7066,
+ "theoretical_loss": 4.496251674389596,
+ "tokens_seen": 163643392
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000768,
+ "loss": 1.7063,
+ "theoretical_loss": 4.496020860826098,
+ "tokens_seen": 163708928
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679839518555668,
+ "loss": 1.7411,
+ "theoretical_loss": 4.49579016550347,
+ "tokens_seen": 163774464
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 61072,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7547411918640137,
+ "objective/train/theoretical_loss": 4.49555958831386,
+ "objective/train/tokens_used": 184300000,
+ "theoretical_loss": 4.49555958831386,
+ "tokens_seen": 163840000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679679037111334,
+ "loss": 1.7758,
+ "theoretical_loss": 4.49555958831386,
+ "tokens_seen": 163840000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679518555667002,
+ "loss": 1.7571,
+ "theoretical_loss": 4.49532912914956,
+ "tokens_seen": 163905536
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679358074222668,
+ "loss": 1.7834,
+ "theoretical_loss": 4.495098787903,
+ "tokens_seen": 163971072
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679197592778336,
+ "loss": 1.7879,
+ "theoretical_loss": 4.4948685644667545,
+ "tokens_seen": 164036608
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007679037111334002,
+ "loss": 1.6408,
+ "theoretical_loss": 4.494638458733535,
+ "tokens_seen": 164102144
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767887662988967,
+ "loss": 1.6375,
+ "theoretical_loss": 4.494408470596195,
+ "tokens_seen": 164167680
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007678716148445336,
+ "loss": 1.7134,
+ "theoretical_loss": 4.494178599947729,
+ "tokens_seen": 164233216
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007678555667001004,
+ "loss": 1.8468,
+ "theoretical_loss": 4.4939488466812705,
+ "tokens_seen": 164298752
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767839518555667,
+ "loss": 1.8571,
+ "theoretical_loss": 4.493719210690093,
+ "tokens_seen": 164364288
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007678234704112338,
+ "loss": 1.711,
+ "theoretical_loss": 4.493489691867612,
+ "tokens_seen": 164429824
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007678074222668004,
+ "loss": 1.6744,
+ "theoretical_loss": 4.493260290107379,
+ "tokens_seen": 164495360
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007677913741223672,
+ "loss": 1.6627,
+ "theoretical_loss": 4.493031005303084,
+ "tokens_seen": 164560896
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007677753259779338,
+ "loss": 1.7317,
+ "theoretical_loss": 4.49280183734856,
+ "tokens_seen": 164626432
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007677592778335006,
+ "loss": 1.8705,
+ "theoretical_loss": 4.492572786137774,
+ "tokens_seen": 164691968
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007677432296890672,
+ "loss": 1.7851,
+ "theoretical_loss": 4.492343851564835,
+ "tokens_seen": 164757504
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767727181544634,
+ "loss": 1.8575,
+ "theoretical_loss": 4.492115033523987,
+ "tokens_seen": 164823040
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007677111334002006,
+ "loss": 1.7289,
+ "theoretical_loss": 4.491886331909615,
+ "tokens_seen": 164888576
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007676950852557674,
+ "loss": 1.7442,
+ "theoretical_loss": 4.491657746616241,
+ "tokens_seen": 164954112
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767679037111334,
+ "loss": 1.6633,
+ "theoretical_loss": 4.49142927753852,
+ "tokens_seen": 165019648
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007676629889669008,
+ "loss": 1.6377,
+ "theoretical_loss": 4.491200924571251,
+ "tokens_seen": 165085184
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007676469408224674,
+ "loss": 1.5454,
+ "theoretical_loss": 4.490972687609364,
+ "tokens_seen": 165150720
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007676308926780342,
+ "loss": 1.7009,
+ "theoretical_loss": 4.4907445665479315,
+ "tokens_seen": 165216256
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007676148445336008,
+ "loss": 1.7974,
+ "theoretical_loss": 4.490516561282157,
+ "tokens_seen": 165281792
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007675987963891676,
+ "loss": 1.6644,
+ "theoretical_loss": 4.4902886717073835,
+ "tokens_seen": 165347328
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007675827482447342,
+ "loss": 1.7853,
+ "theoretical_loss": 4.49006089771909,
+ "tokens_seen": 165412864
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767566700100301,
+ "loss": 1.5877,
+ "theoretical_loss": 4.489833239212888,
+ "tokens_seen": 165478400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007675506519558676,
+ "loss": 1.6106,
+ "theoretical_loss": 4.48960569608453,
+ "tokens_seen": 165543936
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007675346038114344,
+ "loss": 1.7594,
+ "theoretical_loss": 4.489378268229897,
+ "tokens_seen": 165609472
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767518555667001,
+ "loss": 1.8262,
+ "theoretical_loss": 4.489150955545012,
+ "tokens_seen": 165675008
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007675025075225677,
+ "loss": 1.7828,
+ "theoretical_loss": 4.488923757926031,
+ "tokens_seen": 165740544
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674864593781344,
+ "loss": 1.6994,
+ "theoretical_loss": 4.4886966752692405,
+ "tokens_seen": 165806080
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674704112337011,
+ "loss": 1.6913,
+ "theoretical_loss": 4.488469707471066,
+ "tokens_seen": 165871616
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674543630892678,
+ "loss": 1.6736,
+ "theoretical_loss": 4.488242854428064,
+ "tokens_seen": 165937152
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674383149448345,
+ "loss": 1.6912,
+ "theoretical_loss": 4.488016116036929,
+ "tokens_seen": 166002688
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674222668004012,
+ "loss": 1.9221,
+ "theoretical_loss": 4.487789492194485,
+ "tokens_seen": 166068224
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007674062186559679,
+ "loss": 1.7546,
+ "theoretical_loss": 4.487562982797693,
+ "tokens_seen": 166133760
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007673901705115346,
+ "loss": 1.7376,
+ "theoretical_loss": 4.487336587743643,
+ "tokens_seen": 166199296
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007673741223671013,
+ "loss": 1.7413,
+ "theoretical_loss": 4.487110306929562,
+ "tokens_seen": 166264832
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767358074222668,
+ "loss": 1.7142,
+ "theoretical_loss": 4.48688414025281,
+ "tokens_seen": 166330368
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007673420260782347,
+ "loss": 1.8665,
+ "theoretical_loss": 4.486658087610875,
+ "tokens_seen": 166395904
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007673259779338014,
+ "loss": 1.6341,
+ "theoretical_loss": 4.486432148901384,
+ "tokens_seen": 166461440
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007673099297893681,
+ "loss": 1.7621,
+ "theoretical_loss": 4.48620632402209,
+ "tokens_seen": 166526976
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672938816449348,
+ "loss": 1.7723,
+ "theoretical_loss": 4.485980612870883,
+ "tokens_seen": 166592512
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672778335005015,
+ "loss": 1.7573,
+ "theoretical_loss": 4.48575501534578,
+ "tokens_seen": 166658048
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672617853560682,
+ "loss": 1.7641,
+ "theoretical_loss": 4.485529531344932,
+ "tokens_seen": 166723584
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672457372116349,
+ "loss": 1.8136,
+ "theoretical_loss": 4.485304160766624,
+ "tokens_seen": 166789120
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672296890672016,
+ "loss": 1.7361,
+ "theoretical_loss": 4.485078903509266,
+ "tokens_seen": 166854656
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007672136409227683,
+ "loss": 1.8344,
+ "theoretical_loss": 4.484853759471403,
+ "tokens_seen": 166920192
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000767197592778335,
+ "loss": 1.833,
+ "theoretical_loss": 4.484628728551711,
+ "tokens_seen": 166985728
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671815446339017,
+ "loss": 1.7914,
+ "theoretical_loss": 4.484403810648994,
+ "tokens_seen": 167051264
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 62380,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6307178735733032,
+ "objective/train/theoretical_loss": 4.484179005662186,
+ "objective/train/tokens_used": 187576800,
+ "theoretical_loss": 4.484179005662186,
+ "tokens_seen": 167116800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671654964894685,
+ "loss": 1.7346,
+ "theoretical_loss": 4.484179005662186,
+ "tokens_seen": 167116800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671494483450351,
+ "loss": 1.651,
+ "theoretical_loss": 4.483954313490354,
+ "tokens_seen": 167182336
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671334002006019,
+ "loss": 1.7858,
+ "theoretical_loss": 4.483729734032694,
+ "tokens_seen": 167247872
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671173520561685,
+ "loss": 1.7515,
+ "theoretical_loss": 4.483505267188528,
+ "tokens_seen": 167313408
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007671013039117353,
+ "loss": 1.7517,
+ "theoretical_loss": 4.48328091285731,
+ "tokens_seen": 167378944
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670852557673019,
+ "loss": 1.8486,
+ "theoretical_loss": 4.483056670938625,
+ "tokens_seen": 167444480
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670692076228687,
+ "loss": 1.796,
+ "theoretical_loss": 4.482832541332183,
+ "tokens_seen": 167510016
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670531594784353,
+ "loss": 1.7272,
+ "theoretical_loss": 4.482608523937824,
+ "tokens_seen": 167575552
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670371113340021,
+ "loss": 1.6783,
+ "theoretical_loss": 4.4823846186555185,
+ "tokens_seen": 167641088
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670210631895687,
+ "loss": 1.658,
+ "theoretical_loss": 4.482160825385363,
+ "tokens_seen": 167706624
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007670050150451355,
+ "loss": 1.6597,
+ "theoretical_loss": 4.481937144027583,
+ "tokens_seen": 167772160
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669889669007021,
+ "loss": 1.7993,
+ "theoretical_loss": 4.48171357448253,
+ "tokens_seen": 167837696
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669729187562689,
+ "loss": 1.5537,
+ "theoretical_loss": 4.481490116650686,
+ "tokens_seen": 167903232
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669568706118355,
+ "loss": 1.6828,
+ "theoretical_loss": 4.48126677043266,
+ "tokens_seen": 167968768
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669408224674023,
+ "loss": 1.7082,
+ "theoretical_loss": 4.481043535729183,
+ "tokens_seen": 168034304
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669247743229689,
+ "loss": 1.75,
+ "theoretical_loss": 4.480820412441122,
+ "tokens_seen": 168099840
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007669087261785357,
+ "loss": 1.8383,
+ "theoretical_loss": 4.4805974004694615,
+ "tokens_seen": 168165376
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668926780341023,
+ "loss": 1.7588,
+ "theoretical_loss": 4.48037449971532,
+ "tokens_seen": 168230912
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668766298896691,
+ "loss": 1.7177,
+ "theoretical_loss": 4.480151710079937,
+ "tokens_seen": 168296448
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668605817452357,
+ "loss": 1.722,
+ "theoretical_loss": 4.479929031464682,
+ "tokens_seen": 168361984
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668445336008025,
+ "loss": 1.7248,
+ "theoretical_loss": 4.479706463771047,
+ "tokens_seen": 168427520
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668284854563691,
+ "loss": 1.6552,
+ "theoretical_loss": 4.479484006900652,
+ "tokens_seen": 168493056
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007668124373119359,
+ "loss": 1.6428,
+ "theoretical_loss": 4.479261660755242,
+ "tokens_seen": 168558592
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667963891675025,
+ "loss": 1.8342,
+ "theoretical_loss": 4.479039425236687,
+ "tokens_seen": 168624128
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667803410230693,
+ "loss": 1.7043,
+ "theoretical_loss": 4.478817300246982,
+ "tokens_seen": 168689664
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667642928786359,
+ "loss": 1.7463,
+ "theoretical_loss": 4.478595285688247,
+ "tokens_seen": 168755200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667482447342027,
+ "loss": 1.6112,
+ "theoretical_loss": 4.478373381462728,
+ "tokens_seen": 168820736
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667321965897693,
+ "loss": 1.689,
+ "theoretical_loss": 4.478151587472791,
+ "tokens_seen": 168886272
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667161484453361,
+ "loss": 1.7633,
+ "theoretical_loss": 4.477929903620931,
+ "tokens_seen": 168951808
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007667001003009027,
+ "loss": 1.7455,
+ "theoretical_loss": 4.477708329809766,
+ "tokens_seen": 169017344
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666840521564695,
+ "loss": 1.7587,
+ "theoretical_loss": 4.477486865942036,
+ "tokens_seen": 169082880
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666680040120361,
+ "loss": 1.8086,
+ "theoretical_loss": 4.477265511920607,
+ "tokens_seen": 169148416
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666519558676029,
+ "loss": 1.7175,
+ "theoretical_loss": 4.477044267648466,
+ "tokens_seen": 169213952
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666359077231695,
+ "loss": 1.8273,
+ "theoretical_loss": 4.476823133028725,
+ "tokens_seen": 169279488
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666198595787363,
+ "loss": 1.7291,
+ "theoretical_loss": 4.47660210796462,
+ "tokens_seen": 169345024
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007666038114343029,
+ "loss": 1.7855,
+ "theoretical_loss": 4.476381192359504,
+ "tokens_seen": 169410560
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665877632898697,
+ "loss": 1.8167,
+ "theoretical_loss": 4.476160386116861,
+ "tokens_seen": 169476096
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665717151454363,
+ "loss": 1.8135,
+ "theoretical_loss": 4.475939689140291,
+ "tokens_seen": 169541632
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665556670010031,
+ "loss": 1.6252,
+ "theoretical_loss": 4.475719101333519,
+ "tokens_seen": 169607168
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665396188565697,
+ "loss": 1.851,
+ "theoretical_loss": 4.475498622600391,
+ "tokens_seen": 169672704
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665235707121365,
+ "loss": 1.642,
+ "theoretical_loss": 4.475278252844875,
+ "tokens_seen": 169738240
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007665075225677031,
+ "loss": 1.7247,
+ "theoretical_loss": 4.475057991971062,
+ "tokens_seen": 169803776
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664914744232699,
+ "loss": 1.729,
+ "theoretical_loss": 4.474837839883162,
+ "tokens_seen": 169869312
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664754262788365,
+ "loss": 1.7534,
+ "theoretical_loss": 4.474617796485507,
+ "tokens_seen": 169934848
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664593781344033,
+ "loss": 1.7936,
+ "theoretical_loss": 4.474397861682552,
+ "tokens_seen": 170000384
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664433299899699,
+ "loss": 1.747,
+ "theoretical_loss": 4.474178035378869,
+ "tokens_seen": 170065920
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664272818455367,
+ "loss": 1.8412,
+ "theoretical_loss": 4.473958317479154,
+ "tokens_seen": 170131456
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007664112337011033,
+ "loss": 1.5907,
+ "theoretical_loss": 4.473738707888221,
+ "tokens_seen": 170196992
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663951855566701,
+ "loss": 1.778,
+ "theoretical_loss": 4.473519206511006,
+ "tokens_seen": 170262528
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663791374122367,
+ "loss": 1.8488,
+ "theoretical_loss": 4.473299813252565,
+ "tokens_seen": 170328064
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 63171,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6936179399490356,
+ "objective/train/theoretical_loss": 4.473080528018071,
+ "objective/train/tokens_used": 190853600,
+ "theoretical_loss": 4.473080528018071,
+ "tokens_seen": 170393600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663630892678035,
+ "loss": 1.7195,
+ "theoretical_loss": 4.473080528018071,
+ "tokens_seen": 170393600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663470411233702,
+ "loss": 1.7942,
+ "theoretical_loss": 4.4728613507128205,
+ "tokens_seen": 170459136
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663309929789369,
+ "loss": 1.673,
+ "theoretical_loss": 4.472642281242226,
+ "tokens_seen": 170524672
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007663149448345036,
+ "loss": 1.6945,
+ "theoretical_loss": 4.472423319511822,
+ "tokens_seen": 170590208
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662988966900703,
+ "loss": 1.7537,
+ "theoretical_loss": 4.472204465427261,
+ "tokens_seen": 170655744
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000766282848545637,
+ "loss": 1.7052,
+ "theoretical_loss": 4.471985718894312,
+ "tokens_seen": 170721280
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662668004012037,
+ "loss": 1.7989,
+ "theoretical_loss": 4.471767079818866,
+ "tokens_seen": 170786816
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662507522567703,
+ "loss": 1.7626,
+ "theoretical_loss": 4.471548548106932,
+ "tokens_seen": 170852352
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662347041123371,
+ "loss": 1.7287,
+ "theoretical_loss": 4.471330123664634,
+ "tokens_seen": 170917888
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662186559679037,
+ "loss": 1.5974,
+ "theoretical_loss": 4.471111806398218,
+ "tokens_seen": 170983424
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007662026078234705,
+ "loss": 1.7094,
+ "theoretical_loss": 4.4708935962140455,
+ "tokens_seen": 171048960
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661865596790371,
+ "loss": 1.7946,
+ "theoretical_loss": 4.4706754930185975,
+ "tokens_seen": 171114496
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661705115346039,
+ "loss": 1.677,
+ "theoretical_loss": 4.47045749671847,
+ "tokens_seen": 171180032
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661544633901705,
+ "loss": 1.6607,
+ "theoretical_loss": 4.470239607220377,
+ "tokens_seen": 171245568
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661384152457373,
+ "loss": 1.7783,
+ "theoretical_loss": 4.470021824431152,
+ "tokens_seen": 171311104
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661223671013039,
+ "loss": 1.7274,
+ "theoretical_loss": 4.469804148257742,
+ "tokens_seen": 171376640
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007661063189568706,
+ "loss": 1.8638,
+ "theoretical_loss": 4.469586578607213,
+ "tokens_seen": 171442176
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007660902708124373,
+ "loss": 1.7533,
+ "theoretical_loss": 4.469369115386746,
+ "tokens_seen": 171507712
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000766074222668004,
+ "loss": 1.7417,
+ "theoretical_loss": 4.46915175850364,
+ "tokens_seen": 171573248
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007660581745235707,
+ "loss": 1.6826,
+ "theoretical_loss": 4.468934507865308,
+ "tokens_seen": 171638784
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007660421263791374,
+ "loss": 1.7231,
+ "theoretical_loss": 4.46871736337928,
+ "tokens_seen": 171704320
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007660260782347041,
+ "loss": 1.7589,
+ "theoretical_loss": 4.468500324953202,
+ "tokens_seen": 171769856
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007660100300902708,
+ "loss": 1.8372,
+ "theoretical_loss": 4.468283392494836,
+ "tokens_seen": 171835392
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007659939819458375,
+ "loss": 1.8463,
+ "theoretical_loss": 4.468066565912057,
+ "tokens_seen": 171900928
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007659779338014042,
+ "loss": 1.7143,
+ "theoretical_loss": 4.467849845112859,
+ "tokens_seen": 171966464
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765961885656971,
+ "loss": 1.7111,
+ "theoretical_loss": 4.467633230005347,
+ "tokens_seen": 172032000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007659458375125376,
+ "loss": 1.7905,
+ "theoretical_loss": 4.467416720497743,
+ "tokens_seen": 172097536
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007659297893681043,
+ "loss": 1.7305,
+ "theoretical_loss": 4.467200316498385,
+ "tokens_seen": 172163072
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765913741223671,
+ "loss": 1.6837,
+ "theoretical_loss": 4.4669840179157205,
+ "tokens_seen": 172228608
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658976930792377,
+ "loss": 1.6952,
+ "theoretical_loss": 4.466767824658317,
+ "tokens_seen": 172294144
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658816449348044,
+ "loss": 1.6608,
+ "theoretical_loss": 4.466551736634852,
+ "tokens_seen": 172359680
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658655967903711,
+ "loss": 1.7433,
+ "theoretical_loss": 4.466335753754118,
+ "tokens_seen": 172425216
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658495486459378,
+ "loss": 1.7105,
+ "theoretical_loss": 4.466119875925022,
+ "tokens_seen": 172490752
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658335005015046,
+ "loss": 1.6972,
+ "theoretical_loss": 4.465904103056586,
+ "tokens_seen": 172556288
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007658174523570712,
+ "loss": 1.7239,
+ "theoretical_loss": 4.465688435057938,
+ "tokens_seen": 172621824
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765801404212638,
+ "loss": 1.6552,
+ "theoretical_loss": 4.46547287183833,
+ "tokens_seen": 172687360
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007657853560682046,
+ "loss": 1.5746,
+ "theoretical_loss": 4.465257413307119,
+ "tokens_seen": 172752896
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007657693079237714,
+ "loss": 1.7696,
+ "theoretical_loss": 4.465042059373776,
+ "tokens_seen": 172818432
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765753259779338,
+ "loss": 1.7059,
+ "theoretical_loss": 4.464826809947887,
+ "tokens_seen": 172883968
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007657372116349048,
+ "loss": 1.7048,
+ "theoretical_loss": 4.464611664939149,
+ "tokens_seen": 172949504
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007657211634904714,
+ "loss": 1.6989,
+ "theoretical_loss": 4.46439662425737,
+ "tokens_seen": 173015040
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007657051153460382,
+ "loss": 1.6436,
+ "theoretical_loss": 4.464181687812474,
+ "tokens_seen": 173080576
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007656890672016048,
+ "loss": 1.7972,
+ "theoretical_loss": 4.4639668555144905,
+ "tokens_seen": 173146112
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007656730190571716,
+ "loss": 1.7853,
+ "theoretical_loss": 4.463752127273567,
+ "tokens_seen": 173211648
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007656569709127382,
+ "loss": 1.795,
+ "theoretical_loss": 4.463537502999959,
+ "tokens_seen": 173277184
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765640922768305,
+ "loss": 1.6484,
+ "theoretical_loss": 4.4633229826040335,
+ "tokens_seen": 173342720
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007656248746238716,
+ "loss": 1.6078,
+ "theoretical_loss": 4.46310856599627,
+ "tokens_seen": 173408256
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007656088264794384,
+ "loss": 1.7552,
+ "theoretical_loss": 4.462894253087258,
+ "tokens_seen": 173473792
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765592778335005,
+ "loss": 1.8263,
+ "theoretical_loss": 4.462680043787699,
+ "tokens_seen": 173539328
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007655767301905718,
+ "loss": 1.794,
+ "theoretical_loss": 4.4624659380084015,
+ "tokens_seen": 173604864
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 64355,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7965178489685059,
+ "objective/train/theoretical_loss": 4.462251935660287,
+ "objective/train/tokens_used": 194130400,
+ "theoretical_loss": 4.462251935660287,
+ "tokens_seen": 173670400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007655606820461384,
+ "loss": 1.6888,
+ "theoretical_loss": 4.462251935660287,
+ "tokens_seen": 173670400
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007655446339017052,
+ "loss": 1.722,
+ "theoretical_loss": 4.46203803665439,
+ "tokens_seen": 173735936
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007655285857572718,
+ "loss": 1.8336,
+ "theoretical_loss": 4.4618242409018505,
+ "tokens_seen": 173801472
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007655125376128386,
+ "loss": 1.7571,
+ "theoretical_loss": 4.461610548313919,
+ "tokens_seen": 173867008
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007654964894684052,
+ "loss": 1.7694,
+ "theoretical_loss": 4.461396958801959,
+ "tokens_seen": 173932544
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765480441323972,
+ "loss": 1.7376,
+ "theoretical_loss": 4.461183472277439,
+ "tokens_seen": 173998080
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007654643931795386,
+ "loss": 1.6581,
+ "theoretical_loss": 4.460970088651941,
+ "tokens_seen": 174063616
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007654483450351054,
+ "loss": 1.6625,
+ "theoretical_loss": 4.460756807837154,
+ "tokens_seen": 174129152
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765432296890672,
+ "loss": 1.6933,
+ "theoretical_loss": 4.460543629744874,
+ "tokens_seen": 174194688
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007654162487462388,
+ "loss": 1.4361,
+ "theoretical_loss": 4.460330554287011,
+ "tokens_seen": 174260224
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007654002006018054,
+ "loss": 1.7742,
+ "theoretical_loss": 4.460117581375579,
+ "tokens_seen": 174325760
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007653841524573722,
+ "loss": 1.6683,
+ "theoretical_loss": 4.459904710922704,
+ "tokens_seen": 174391296
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007653681043129388,
+ "loss": 1.7873,
+ "theoretical_loss": 4.459691942840616,
+ "tokens_seen": 174456832
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007653520561685056,
+ "loss": 1.7836,
+ "theoretical_loss": 4.459479277041657,
+ "tokens_seen": 174522368
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007653360080240722,
+ "loss": 1.616,
+ "theoretical_loss": 4.459266713438277,
+ "tokens_seen": 174587904
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765319959879639,
+ "loss": 1.7366,
+ "theoretical_loss": 4.459054251943029,
+ "tokens_seen": 174653440
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007653039117352056,
+ "loss": 1.7021,
+ "theoretical_loss": 4.4588418924685795,
+ "tokens_seen": 174718976
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007652878635907724,
+ "loss": 1.9371,
+ "theoretical_loss": 4.458629634927698,
+ "tokens_seen": 174784512
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765271815446339,
+ "loss": 1.6323,
+ "theoretical_loss": 4.458417479233265,
+ "tokens_seen": 174850048
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007652557673019058,
+ "loss": 1.7393,
+ "theoretical_loss": 4.4582054252982655,
+ "tokens_seen": 174915584
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007652397191574724,
+ "loss": 1.7396,
+ "theoretical_loss": 4.457993473035792,
+ "tokens_seen": 174981120
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007652236710130392,
+ "loss": 1.6274,
+ "theoretical_loss": 4.457781622359043,
+ "tokens_seen": 175046656
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007652076228686058,
+ "loss": 1.6795,
+ "theoretical_loss": 4.457569873181326,
+ "tokens_seen": 175112192
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007651915747241726,
+ "loss": 1.7442,
+ "theoretical_loss": 4.457358225416052,
+ "tokens_seen": 175177728
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007651755265797392,
+ "loss": 1.6656,
+ "theoretical_loss": 4.4571466789767396,
+ "tokens_seen": 175243264
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000765159478435306,
+ "loss": 1.6225,
+ "theoretical_loss": 4.456935233777013,
+ "tokens_seen": 175308800
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007651434302908727,
+ "loss": 1.7749,
+ "theoretical_loss": 4.456723889730605,
+ "tokens_seen": 175374336
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007651273821464394,
+ "loss": 1.7267,
+ "theoretical_loss": 4.456512646751349,
+ "tokens_seen": 175439872
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007651113340020061,
+ "loss": 1.7734,
+ "theoretical_loss": 4.456301504753188,
+ "tokens_seen": 175505408
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650952858575728,
+ "loss": 1.7111,
+ "theoretical_loss": 4.456090463650169,
+ "tokens_seen": 175570944
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650792377131395,
+ "loss": 1.6561,
+ "theoretical_loss": 4.455879523356444,
+ "tokens_seen": 175636480
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650631895687062,
+ "loss": 1.77,
+ "theoretical_loss": 4.4556686837862705,
+ "tokens_seen": 175702016
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650471414242729,
+ "loss": 1.6699,
+ "theoretical_loss": 4.455457944854011,
+ "tokens_seen": 175767552
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650310932798396,
+ "loss": 1.7107,
+ "theoretical_loss": 4.455247306474131,
+ "tokens_seen": 175833088
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007650150451354063,
+ "loss": 1.8197,
+ "theoretical_loss": 4.455036768561204,
+ "tokens_seen": 175898624
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000764998996990973,
+ "loss": 1.7647,
+ "theoretical_loss": 4.4548263310299046,
+ "tokens_seen": 175964160
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649829488465397,
+ "loss": 1.7439,
+ "theoretical_loss": 4.454615993795015,
+ "tokens_seen": 176029696
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649669007021064,
+ "loss": 1.6774,
+ "theoretical_loss": 4.454405756771415,
+ "tokens_seen": 176095232
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649508525576731,
+ "loss": 1.822,
+ "theoretical_loss": 4.454195619874096,
+ "tokens_seen": 176160768
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649348044132398,
+ "loss": 1.7231,
+ "theoretical_loss": 4.453985583018149,
+ "tokens_seen": 176226304
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649187562688065,
+ "loss": 1.7635,
+ "theoretical_loss": 4.4537756461187685,
+ "tokens_seen": 176291840
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007649027081243732,
+ "loss": 1.7327,
+ "theoretical_loss": 4.453565809091254,
+ "tokens_seen": 176357376
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007648866599799399,
+ "loss": 1.7449,
+ "theoretical_loss": 4.453356071851006,
+ "tokens_seen": 176422912
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007648706118355066,
+ "loss": 1.7211,
+ "theoretical_loss": 4.453146434313531,
+ "tokens_seen": 176488448
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007648545636910732,
+ "loss": 1.851,
+ "theoretical_loss": 4.452936896394435,
+ "tokens_seen": 176553984
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.00076483851554664,
+ "loss": 1.7872,
+ "theoretical_loss": 4.452727458009431,
+ "tokens_seen": 176619520
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007648224674022066,
+ "loss": 1.7427,
+ "theoretical_loss": 4.45251811907433,
+ "tokens_seen": 176685056
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007648064192577734,
+ "loss": 1.6532,
+ "theoretical_loss": 4.452308879505047,
+ "tokens_seen": 176750592
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.00076479037111334,
+ "loss": 1.8032,
+ "theoretical_loss": 4.452099739217601,
+ "tokens_seen": 176816128
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007647743229689068,
+ "loss": 1.6829,
+ "theoretical_loss": 4.451890698128112,
+ "tokens_seen": 176881664
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 65665,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5994759798049927,
+ "objective/train/theoretical_loss": 4.4516817561528015,
+ "objective/train/tokens_used": 197407200,
+ "theoretical_loss": 4.4516817561528015,
+ "tokens_seen": 176947200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007647582748244734,
+ "loss": 1.6761,
+ "theoretical_loss": 4.4516817561528015,
+ "tokens_seen": 176947200
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007647422266800402,
+ "loss": 1.6801,
+ "theoretical_loss": 4.451472913207992,
+ "tokens_seen": 177012736
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007647261785356068,
+ "loss": 1.7034,
+ "theoretical_loss": 4.451264169210107,
+ "tokens_seen": 177078272
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007647101303911735,
+ "loss": 1.651,
+ "theoretical_loss": 4.451055524075677,
+ "tokens_seen": 177143808
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007646940822467402,
+ "loss": 1.6878,
+ "theoretical_loss": 4.450846977721326,
+ "tokens_seen": 177209344
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007646780341023069,
+ "loss": 1.7719,
+ "theoretical_loss": 4.450638530063786,
+ "tokens_seen": 177274880
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007646619859578736,
+ "loss": 1.7237,
+ "theoretical_loss": 4.450430181019884,
+ "tokens_seen": 177340416
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007646459378134403,
+ "loss": 1.7421,
+ "theoretical_loss": 4.450221930506553,
+ "tokens_seen": 177405952
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000764629889669007,
+ "loss": 1.6159,
+ "theoretical_loss": 4.450013778440822,
+ "tokens_seen": 177471488
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007646138415245737,
+ "loss": 1.6349,
+ "theoretical_loss": 4.449805724739824,
+ "tokens_seen": 177537024
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645977933801404,
+ "loss": 1.6987,
+ "theoretical_loss": 4.449597769320791,
+ "tokens_seen": 177602560
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645817452357071,
+ "loss": 1.7311,
+ "theoretical_loss": 4.4493899121010525,
+ "tokens_seen": 177668096
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645656970912738,
+ "loss": 1.7738,
+ "theoretical_loss": 4.449182152998044,
+ "tokens_seen": 177733632
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645496489468405,
+ "loss": 1.6177,
+ "theoretical_loss": 4.448974491929297,
+ "tokens_seen": 177799168
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645336008024072,
+ "loss": 1.8036,
+ "theoretical_loss": 4.448766928812441,
+ "tokens_seen": 177864704
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645175526579739,
+ "loss": 1.7037,
+ "theoretical_loss": 4.448559463565209,
+ "tokens_seen": 177930240
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007645015045135406,
+ "loss": 1.6341,
+ "theoretical_loss": 4.44835209610543,
+ "tokens_seen": 177995776
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007644854563691073,
+ "loss": 1.7024,
+ "theoretical_loss": 4.448144826351036,
+ "tokens_seen": 178061312
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000764469408224674,
+ "loss": 1.6771,
+ "theoretical_loss": 4.447937654220054,
+ "tokens_seen": 178126848
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007644533600802407,
+ "loss": 1.7449,
+ "theoretical_loss": 4.447730579630612,
+ "tokens_seen": 178192384
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007644373119358075,
+ "loss": 1.5651,
+ "theoretical_loss": 4.447523602500936,
+ "tokens_seen": 178257920
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007644212637913741,
+ "loss": 1.6465,
+ "theoretical_loss": 4.447316722749351,
+ "tokens_seen": 178323456
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007644052156469409,
+ "loss": 1.6778,
+ "theoretical_loss": 4.447109940294283,
+ "tokens_seen": 178388992
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643891675025075,
+ "loss": 1.7214,
+ "theoretical_loss": 4.44690325505425,
+ "tokens_seen": 178454528
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643731193580743,
+ "loss": 1.6484,
+ "theoretical_loss": 4.446696666947874,
+ "tokens_seen": 178520064
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643570712136409,
+ "loss": 1.733,
+ "theoretical_loss": 4.446490175893872,
+ "tokens_seen": 178585600
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643410230692077,
+ "loss": 1.6846,
+ "theoretical_loss": 4.44628378181106,
+ "tokens_seen": 178651136
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643249749247743,
+ "loss": 1.7986,
+ "theoretical_loss": 4.4460774846183515,
+ "tokens_seen": 178716672
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007643089267803411,
+ "loss": 1.6765,
+ "theoretical_loss": 4.445871284234757,
+ "tokens_seen": 178782208
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642928786359077,
+ "loss": 1.8968,
+ "theoretical_loss": 4.445665180579383,
+ "tokens_seen": 178847744
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642768304914745,
+ "loss": 1.6697,
+ "theoretical_loss": 4.445459173571438,
+ "tokens_seen": 178913280
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642607823470411,
+ "loss": 1.731,
+ "theoretical_loss": 4.445253263130223,
+ "tokens_seen": 178978816
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642447342026079,
+ "loss": 1.668,
+ "theoretical_loss": 4.445047449175136,
+ "tokens_seen": 179044352
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642286860581745,
+ "loss": 1.6233,
+ "theoretical_loss": 4.444841731625674,
+ "tokens_seen": 179109888
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007642126379137413,
+ "loss": 1.8112,
+ "theoretical_loss": 4.444636110401431,
+ "tokens_seen": 179175424
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641965897693079,
+ "loss": 1.7487,
+ "theoretical_loss": 4.444430585422094,
+ "tokens_seen": 179240960
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641805416248747,
+ "loss": 1.5881,
+ "theoretical_loss": 4.444225156607449,
+ "tokens_seen": 179306496
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641644934804413,
+ "loss": 1.7401,
+ "theoretical_loss": 4.444019823877378,
+ "tokens_seen": 179372032
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641484453360081,
+ "loss": 1.6739,
+ "theoretical_loss": 4.4438145871518575,
+ "tokens_seen": 179437568
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641323971915747,
+ "loss": 1.7123,
+ "theoretical_loss": 4.443609446350962,
+ "tokens_seen": 179503104
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641163490471415,
+ "loss": 1.697,
+ "theoretical_loss": 4.44340440139486,
+ "tokens_seen": 179568640
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007641003009027081,
+ "loss": 1.7086,
+ "theoretical_loss": 4.443199452203816,
+ "tokens_seen": 179634176
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640842527582749,
+ "loss": 1.6203,
+ "theoretical_loss": 4.442994598698189,
+ "tokens_seen": 179699712
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640682046138415,
+ "loss": 1.7007,
+ "theoretical_loss": 4.4427898407984365,
+ "tokens_seen": 179765248
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640521564694083,
+ "loss": 1.6982,
+ "theoretical_loss": 4.442585178425107,
+ "tokens_seen": 179830784
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640361083249749,
+ "loss": 1.6712,
+ "theoretical_loss": 4.442380611498846,
+ "tokens_seen": 179896320
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640200601805417,
+ "loss": 1.7155,
+ "theoretical_loss": 4.442176139940393,
+ "tokens_seen": 179961856
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007640040120361083,
+ "loss": 1.8174,
+ "theoretical_loss": 4.441971763670585,
+ "tokens_seen": 180027392
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639879638916751,
+ "loss": 1.8372,
+ "theoretical_loss": 4.441767482610349,
+ "tokens_seen": 180092928
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639719157472417,
+ "loss": 1.7136,
+ "theoretical_loss": 4.441563296680709,
+ "tokens_seen": 180158464
+ },
+ {
+ "epoch": 0.05,
+ "objective/train/docs_used": 66635,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7017289400100708,
+ "objective/train/theoretical_loss": 4.441359205802783,
+ "objective/train/tokens_used": 200684000,
+ "theoretical_loss": 4.441359205802783,
+ "tokens_seen": 180224000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639558676028085,
+ "loss": 1.6671,
+ "theoretical_loss": 4.441359205802783,
+ "tokens_seen": 180224000
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639398194583752,
+ "loss": 1.74,
+ "theoretical_loss": 4.441155209897782,
+ "tokens_seen": 180289536
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639237713139419,
+ "loss": 1.7344,
+ "theoretical_loss": 4.440951308887014,
+ "tokens_seen": 180355072
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007639077231695086,
+ "loss": 1.7618,
+ "theoretical_loss": 4.440747502691876,
+ "tokens_seen": 180420608
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007638916750250753,
+ "loss": 1.7888,
+ "theoretical_loss": 4.440543791233862,
+ "tokens_seen": 180486144
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000763875626880642,
+ "loss": 1.6206,
+ "theoretical_loss": 4.440340174434558,
+ "tokens_seen": 180551680
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007638595787362087,
+ "loss": 1.7611,
+ "theoretical_loss": 4.440136652215646,
+ "tokens_seen": 180617216
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007638435305917754,
+ "loss": 1.7102,
+ "theoretical_loss": 4.439933224498898,
+ "tokens_seen": 180682752
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007638274824473421,
+ "loss": 1.7108,
+ "theoretical_loss": 4.43972989120618,
+ "tokens_seen": 180748288
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007638114343029088,
+ "loss": 1.8015,
+ "theoretical_loss": 4.439526652259451,
+ "tokens_seen": 180813824
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007637953861584755,
+ "loss": 1.6381,
+ "theoretical_loss": 4.439323507580761,
+ "tokens_seen": 180879360
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007637793380140422,
+ "loss": 1.633,
+ "theoretical_loss": 4.439120457092258,
+ "tokens_seen": 180944896
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007637632898696089,
+ "loss": 1.6394,
+ "theoretical_loss": 4.438917500716178,
+ "tokens_seen": 181010432
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007637472417251756,
+ "loss": 1.658,
+ "theoretical_loss": 4.438714638374849,
+ "tokens_seen": 181075968
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007637311935807423,
+ "loss": 1.7647,
+ "theoretical_loss": 4.438511869990693,
+ "tokens_seen": 181141504
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.000763715145436309,
+ "loss": 1.7654,
+ "theoretical_loss": 4.438309195486223,
+ "tokens_seen": 181207040
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007636990972918757,
+ "loss": 1.7345,
+ "theoretical_loss": 4.438106614784045,
+ "tokens_seen": 181272576
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007636830491474424,
+ "loss": 1.7755,
+ "theoretical_loss": 4.437904127806856,
+ "tokens_seen": 181338112
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007636670010030091,
+ "loss": 1.7887,
+ "theoretical_loss": 4.437701734477445,
+ "tokens_seen": 181403648
+ },
+ {
+ "epoch": 0.05,
+ "learning_rate": 0.0007636509528585758,
+ "loss": 1.7479,
+ "theoretical_loss": 4.437499434718692,
+ "tokens_seen": 181469184
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007636349047141425,
+ "loss": 1.8136,
+ "theoretical_loss": 4.437297228453567,
+ "tokens_seen": 181534720
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007636188565697092,
+ "loss": 1.6026,
+ "theoretical_loss": 4.4370951156051355,
+ "tokens_seen": 181600256
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007636028084252759,
+ "loss": 1.7126,
+ "theoretical_loss": 4.43689309609655,
+ "tokens_seen": 181665792
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007635867602808426,
+ "loss": 1.7019,
+ "theoretical_loss": 4.436691169851055,
+ "tokens_seen": 181731328
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007635707121364093,
+ "loss": 1.5814,
+ "theoretical_loss": 4.436489336791985,
+ "tokens_seen": 181796864
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000763554663991976,
+ "loss": 1.7454,
+ "theoretical_loss": 4.436287596842769,
+ "tokens_seen": 181862400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007635386158475427,
+ "loss": 1.6315,
+ "theoretical_loss": 4.4360859499269205,
+ "tokens_seen": 181927936
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007635225677031094,
+ "loss": 1.7988,
+ "theoretical_loss": 4.435884395968047,
+ "tokens_seen": 181993472
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007635065195586761,
+ "loss": 1.763,
+ "theoretical_loss": 4.435682934889847,
+ "tokens_seen": 182059008
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634904714142428,
+ "loss": 1.7406,
+ "theoretical_loss": 4.435481566616108,
+ "tokens_seen": 182124544
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634744232698095,
+ "loss": 1.605,
+ "theoretical_loss": 4.435280291070705,
+ "tokens_seen": 182190080
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634583751253762,
+ "loss": 1.7259,
+ "theoretical_loss": 4.435079108177606,
+ "tokens_seen": 182255616
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634423269809429,
+ "loss": 1.7148,
+ "theoretical_loss": 4.434878017860869,
+ "tokens_seen": 182321152
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634262788365095,
+ "loss": 1.752,
+ "theoretical_loss": 4.434677020044637,
+ "tokens_seen": 182386688
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007634102306920763,
+ "loss": 1.7357,
+ "theoretical_loss": 4.4344761146531475,
+ "tokens_seen": 182452224
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633941825476429,
+ "loss": 1.712,
+ "theoretical_loss": 4.434275301610725,
+ "tokens_seen": 182517760
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633781344032097,
+ "loss": 1.7025,
+ "theoretical_loss": 4.434074580841784,
+ "tokens_seen": 182583296
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633620862587763,
+ "loss": 1.6831,
+ "theoretical_loss": 4.433873952270826,
+ "tokens_seen": 182648832
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633460381143431,
+ "loss": 1.6822,
+ "theoretical_loss": 4.433673415822443,
+ "tokens_seen": 182714368
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633299899699097,
+ "loss": 1.6375,
+ "theoretical_loss": 4.433472971421315,
+ "tokens_seen": 182779904
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007633139418254764,
+ "loss": 1.7446,
+ "theoretical_loss": 4.433272618992213,
+ "tokens_seen": 182845440
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632978936810431,
+ "loss": 1.6624,
+ "theoretical_loss": 4.433072358459991,
+ "tokens_seen": 182910976
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632818455366098,
+ "loss": 1.667,
+ "theoretical_loss": 4.432872189749596,
+ "tokens_seen": 182976512
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632657973921765,
+ "loss": 1.6365,
+ "theoretical_loss": 4.432672112786063,
+ "tokens_seen": 183042048
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632497492477432,
+ "loss": 1.7046,
+ "theoretical_loss": 4.4324721274945125,
+ "tokens_seen": 183107584
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632337011033099,
+ "loss": 1.6919,
+ "theoretical_loss": 4.432272233800154,
+ "tokens_seen": 183173120
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632176529588766,
+ "loss": 1.6129,
+ "theoretical_loss": 4.432072431628286,
+ "tokens_seen": 183238656
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007632016048144433,
+ "loss": 1.6262,
+ "theoretical_loss": 4.431872720904292,
+ "tokens_seen": 183304192
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.00076318555667001,
+ "loss": 1.6003,
+ "theoretical_loss": 4.4316731015536455,
+ "tokens_seen": 183369728
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007631695085255767,
+ "loss": 1.6435,
+ "theoretical_loss": 4.431473573501906,
+ "tokens_seen": 183435264
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 67570,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8887503147125244,
+ "objective/train/theoretical_loss": 4.431274136674721,
+ "objective/train/tokens_used": 203960800,
+ "theoretical_loss": 4.431274136674721,
+ "tokens_seen": 183500800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007631534603811434,
+ "loss": 1.8266,
+ "theoretical_loss": 4.431274136674721,
+ "tokens_seen": 183500800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007631374122367101,
+ "loss": 1.6985,
+ "theoretical_loss": 4.431074790997823,
+ "tokens_seen": 183566336
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007631213640922768,
+ "loss": 1.7259,
+ "theoretical_loss": 4.430875536397036,
+ "tokens_seen": 183631872
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007631053159478436,
+ "loss": 1.7227,
+ "theoretical_loss": 4.430676372798265,
+ "tokens_seen": 183697408
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007630892678034102,
+ "loss": 1.8484,
+ "theoretical_loss": 4.430477300127507,
+ "tokens_seen": 183762944
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000763073219658977,
+ "loss": 1.7766,
+ "theoretical_loss": 4.43027831831084,
+ "tokens_seen": 183828480
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007630571715145436,
+ "loss": 1.7096,
+ "theoretical_loss": 4.430079427274435,
+ "tokens_seen": 183894016
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007630411233701104,
+ "loss": 1.6482,
+ "theoretical_loss": 4.429880626944543,
+ "tokens_seen": 183959552
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000763025075225677,
+ "loss": 1.7298,
+ "theoretical_loss": 4.429681917247506,
+ "tokens_seen": 184025088
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007630090270812438,
+ "loss": 1.7268,
+ "theoretical_loss": 4.429483298109749,
+ "tokens_seen": 184090624
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007629929789368104,
+ "loss": 1.6922,
+ "theoretical_loss": 4.429284769457784,
+ "tokens_seen": 184156160
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007629769307923772,
+ "loss": 1.5606,
+ "theoretical_loss": 4.429086331218208,
+ "tokens_seen": 184221696
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007629608826479438,
+ "loss": 1.6067,
+ "theoretical_loss": 4.428887983317706,
+ "tokens_seen": 184287232
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007629448345035106,
+ "loss": 1.7639,
+ "theoretical_loss": 4.4286897256830455,
+ "tokens_seen": 184352768
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007629287863590772,
+ "loss": 1.7258,
+ "theoretical_loss": 4.428491558241081,
+ "tokens_seen": 184418304
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762912738214644,
+ "loss": 1.6622,
+ "theoretical_loss": 4.428293480918751,
+ "tokens_seen": 184483840
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628966900702106,
+ "loss": 1.7358,
+ "theoretical_loss": 4.428095493643081,
+ "tokens_seen": 184549376
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628806419257774,
+ "loss": 1.6846,
+ "theoretical_loss": 4.427897596341181,
+ "tokens_seen": 184614912
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762864593781344,
+ "loss": 1.6989,
+ "theoretical_loss": 4.427699788940243,
+ "tokens_seen": 184680448
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628485456369108,
+ "loss": 1.6336,
+ "theoretical_loss": 4.427502071367549,
+ "tokens_seen": 184745984
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628324974924774,
+ "loss": 1.684,
+ "theoretical_loss": 4.42730444355046,
+ "tokens_seen": 184811520
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628164493480442,
+ "loss": 1.7098,
+ "theoretical_loss": 4.4271069054164265,
+ "tokens_seen": 184877056
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007628004012036108,
+ "loss": 1.7049,
+ "theoretical_loss": 4.426909456892979,
+ "tokens_seen": 184942592
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007627843530591776,
+ "loss": 1.6428,
+ "theoretical_loss": 4.426712097907735,
+ "tokens_seen": 185008128
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007627683049147442,
+ "loss": 1.6611,
+ "theoretical_loss": 4.426514828388394,
+ "tokens_seen": 185073664
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762752256770311,
+ "loss": 1.5817,
+ "theoretical_loss": 4.426317648262743,
+ "tokens_seen": 185139200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007627362086258777,
+ "loss": 1.6968,
+ "theoretical_loss": 4.426120557458649,
+ "tokens_seen": 185204736
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007627201604814444,
+ "loss": 1.6527,
+ "theoretical_loss": 4.425923555904065,
+ "tokens_seen": 185270272
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007627041123370111,
+ "loss": 1.6712,
+ "theoretical_loss": 4.4257266435270255,
+ "tokens_seen": 185335808
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626880641925778,
+ "loss": 1.622,
+ "theoretical_loss": 4.425529820255651,
+ "tokens_seen": 185401344
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626720160481445,
+ "loss": 1.7908,
+ "theoretical_loss": 4.425333086018144,
+ "tokens_seen": 185466880
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626559679037112,
+ "loss": 1.7091,
+ "theoretical_loss": 4.425136440742788,
+ "tokens_seen": 185532416
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626399197592779,
+ "loss": 1.6781,
+ "theoretical_loss": 4.424939884357956,
+ "tokens_seen": 185597952
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626238716148446,
+ "loss": 1.697,
+ "theoretical_loss": 4.424743416792096,
+ "tokens_seen": 185663488
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007626078234704113,
+ "loss": 1.6791,
+ "theoretical_loss": 4.424547037973745,
+ "tokens_seen": 185729024
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762591775325978,
+ "loss": 1.699,
+ "theoretical_loss": 4.424350747831518,
+ "tokens_seen": 185794560
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007625757271815447,
+ "loss": 1.7917,
+ "theoretical_loss": 4.424154546294117,
+ "tokens_seen": 185860096
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007625596790371114,
+ "loss": 1.707,
+ "theoretical_loss": 4.4239584332903235,
+ "tokens_seen": 185925632
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007625436308926781,
+ "loss": 1.6794,
+ "theoretical_loss": 4.423762408749001,
+ "tokens_seen": 185991168
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007625275827482448,
+ "loss": 1.6936,
+ "theoretical_loss": 4.423566472599099,
+ "tokens_seen": 186056704
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007625115346038115,
+ "loss": 1.8014,
+ "theoretical_loss": 4.423370624769643,
+ "tokens_seen": 186122240
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007624954864593782,
+ "loss": 1.7071,
+ "theoretical_loss": 4.423174865189747,
+ "tokens_seen": 186187776
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007624794383149449,
+ "loss": 1.6442,
+ "theoretical_loss": 4.422979193788601,
+ "tokens_seen": 186253312
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007624633901705116,
+ "loss": 1.5889,
+ "theoretical_loss": 4.422783610495481,
+ "tokens_seen": 186318848
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007624473420260783,
+ "loss": 1.6439,
+ "theoretical_loss": 4.422588115239742,
+ "tokens_seen": 186384384
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762431293881645,
+ "loss": 1.7129,
+ "theoretical_loss": 4.422392707950824,
+ "tokens_seen": 186449920
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007624152457372117,
+ "loss": 1.7146,
+ "theoretical_loss": 4.42219738855824,
+ "tokens_seen": 186515456
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623991975927784,
+ "loss": 1.663,
+ "theoretical_loss": 4.422002156991597,
+ "tokens_seen": 186580992
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623831494483451,
+ "loss": 1.6474,
+ "theoretical_loss": 4.421807013180571,
+ "tokens_seen": 186646528
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623671013039118,
+ "loss": 1.7353,
+ "theoretical_loss": 4.421611957054925,
+ "tokens_seen": 186712064
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 68808,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6501104831695557,
+ "objective/train/theoretical_loss": 4.421416988544503,
+ "objective/train/tokens_used": 207237600,
+ "theoretical_loss": 4.421416988544503,
+ "tokens_seen": 186777600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623510531594785,
+ "loss": 1.7163,
+ "theoretical_loss": 4.421416988544503,
+ "tokens_seen": 186777600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623350050150452,
+ "loss": 1.7814,
+ "theoretical_loss": 4.4212221075792275,
+ "tokens_seen": 186843136
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623189568706119,
+ "loss": 1.7138,
+ "theoretical_loss": 4.421027314089103,
+ "tokens_seen": 186908672
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007623029087261786,
+ "loss": 1.6188,
+ "theoretical_loss": 4.420832608004215,
+ "tokens_seen": 186974208
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007622868605817453,
+ "loss": 1.6911,
+ "theoretical_loss": 4.420637989254726,
+ "tokens_seen": 187039744
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762270812437312,
+ "loss": 1.6176,
+ "theoretical_loss": 4.420443457770885,
+ "tokens_seen": 187105280
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007622547642928787,
+ "loss": 1.7182,
+ "theoretical_loss": 4.420249013483014,
+ "tokens_seen": 187170816
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007622387161484454,
+ "loss": 1.7003,
+ "theoretical_loss": 4.42005465632152,
+ "tokens_seen": 187236352
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007622226680040121,
+ "loss": 1.7183,
+ "theoretical_loss": 4.419860386216888,
+ "tokens_seen": 187301888
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007622066198595788,
+ "loss": 1.737,
+ "theoretical_loss": 4.419666203099683,
+ "tokens_seen": 187367424
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007621905717151455,
+ "loss": 1.7014,
+ "theoretical_loss": 4.41947210690055,
+ "tokens_seen": 187432960
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007621745235707122,
+ "loss": 1.7964,
+ "theoretical_loss": 4.419278097550212,
+ "tokens_seen": 187498496
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007621584754262789,
+ "loss": 1.6903,
+ "theoretical_loss": 4.419084174979473,
+ "tokens_seen": 187564032
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007621424272818456,
+ "loss": 1.7771,
+ "theoretical_loss": 4.4188903391192165,
+ "tokens_seen": 187629568
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007621263791374123,
+ "loss": 1.7877,
+ "theoretical_loss": 4.418696589900405,
+ "tokens_seen": 187695104
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000762110330992979,
+ "loss": 1.7553,
+ "theoretical_loss": 4.418502927254078,
+ "tokens_seen": 187760640
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620942828485457,
+ "loss": 1.7333,
+ "theoretical_loss": 4.418309351111356,
+ "tokens_seen": 187826176
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620782347041124,
+ "loss": 1.6975,
+ "theoretical_loss": 4.418115861403439,
+ "tokens_seen": 187891712
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620621865596791,
+ "loss": 1.5968,
+ "theoretical_loss": 4.417922458061603,
+ "tokens_seen": 187957248
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620461384152458,
+ "loss": 1.6263,
+ "theoretical_loss": 4.417729141017205,
+ "tokens_seen": 188022784
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620300902708124,
+ "loss": 1.7959,
+ "theoretical_loss": 4.4175359102016785,
+ "tokens_seen": 188088320
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007620140421263792,
+ "loss": 1.7892,
+ "theoretical_loss": 4.417342765546538,
+ "tokens_seen": 188153856
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007619979939819458,
+ "loss": 1.75,
+ "theoretical_loss": 4.417149706983373,
+ "tokens_seen": 188219392
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007619819458375126,
+ "loss": 1.6113,
+ "theoretical_loss": 4.416956734443854,
+ "tokens_seen": 188284928
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007619658976930792,
+ "loss": 1.5337,
+ "theoretical_loss": 4.416763847859728,
+ "tokens_seen": 188350464
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761949849548646,
+ "loss": 1.6184,
+ "theoretical_loss": 4.41657104716282,
+ "tokens_seen": 188416000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007619338014042126,
+ "loss": 1.7843,
+ "theoretical_loss": 4.416378332285031,
+ "tokens_seen": 188481536
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007619177532597794,
+ "loss": 1.6608,
+ "theoretical_loss": 4.4161857031583445,
+ "tokens_seen": 188547072
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761901705115346,
+ "loss": 1.7039,
+ "theoretical_loss": 4.415993159714818,
+ "tokens_seen": 188612608
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618856569709127,
+ "loss": 1.8617,
+ "theoretical_loss": 4.415800701886584,
+ "tokens_seen": 188678144
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618696088264794,
+ "loss": 1.813,
+ "theoretical_loss": 4.415608329605859,
+ "tokens_seen": 188743680
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618535606820461,
+ "loss": 1.5579,
+ "theoretical_loss": 4.415416042804931,
+ "tokens_seen": 188809216
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618375125376128,
+ "loss": 1.6717,
+ "theoretical_loss": 4.415223841416167,
+ "tokens_seen": 188874752
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618214643931795,
+ "loss": 1.7537,
+ "theoretical_loss": 4.415031725372011,
+ "tokens_seen": 188940288
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007618054162487462,
+ "loss": 1.6978,
+ "theoretical_loss": 4.414839694604985,
+ "tokens_seen": 189005824
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007617893681043129,
+ "loss": 1.6423,
+ "theoretical_loss": 4.414647749047685,
+ "tokens_seen": 189071360
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007617733199598796,
+ "loss": 1.6646,
+ "theoretical_loss": 4.414455888632786,
+ "tokens_seen": 189136896
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007617572718154463,
+ "loss": 1.7303,
+ "theoretical_loss": 4.414264113293038,
+ "tokens_seen": 189202432
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761741223671013,
+ "loss": 1.7068,
+ "theoretical_loss": 4.41407242296127,
+ "tokens_seen": 189267968
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007617251755265797,
+ "loss": 1.6995,
+ "theoretical_loss": 4.413880817570382,
+ "tokens_seen": 189333504
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007617091273821465,
+ "loss": 1.6645,
+ "theoretical_loss": 4.413689297053356,
+ "tokens_seen": 189399040
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616930792377131,
+ "loss": 1.6716,
+ "theoretical_loss": 4.413497861343247,
+ "tokens_seen": 189464576
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616770310932799,
+ "loss": 1.6816,
+ "theoretical_loss": 4.413306510373185,
+ "tokens_seen": 189530112
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616609829488465,
+ "loss": 1.62,
+ "theoretical_loss": 4.41311524407638,
+ "tokens_seen": 189595648
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616449348044133,
+ "loss": 1.715,
+ "theoretical_loss": 4.412924062386112,
+ "tokens_seen": 189661184
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616288866599799,
+ "loss": 1.8238,
+ "theoretical_loss": 4.412732965235742,
+ "tokens_seen": 189726720
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007616128385155467,
+ "loss": 1.8357,
+ "theoretical_loss": 4.412541952558703,
+ "tokens_seen": 189792256
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615967903711133,
+ "loss": 1.6682,
+ "theoretical_loss": 4.412351024288504,
+ "tokens_seen": 189857792
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615807422266801,
+ "loss": 1.6815,
+ "theoretical_loss": 4.412160180358731,
+ "tokens_seen": 189923328
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615646940822467,
+ "loss": 1.6328,
+ "theoretical_loss": 4.411969420703042,
+ "tokens_seen": 189988864
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 70105,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.612324595451355,
+ "objective/train/theoretical_loss": 4.411778745255174,
+ "objective/train/tokens_used": 210514400,
+ "theoretical_loss": 4.411778745255174,
+ "tokens_seen": 190054400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615486459378135,
+ "loss": 1.6841,
+ "theoretical_loss": 4.411778745255174,
+ "tokens_seen": 190054400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615325977933802,
+ "loss": 1.7375,
+ "theoretical_loss": 4.411588153948935,
+ "tokens_seen": 190119936
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615165496489469,
+ "loss": 1.8672,
+ "theoretical_loss": 4.411397646718211,
+ "tokens_seen": 190185472
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007615005015045136,
+ "loss": 1.6434,
+ "theoretical_loss": 4.411207223496962,
+ "tokens_seen": 190251008
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007614844533600803,
+ "loss": 1.6167,
+ "theoretical_loss": 4.41101688421922,
+ "tokens_seen": 190316544
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761468405215647,
+ "loss": 1.7425,
+ "theoretical_loss": 4.410826628819094,
+ "tokens_seen": 190382080
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007614523570712137,
+ "loss": 1.6488,
+ "theoretical_loss": 4.410636457230767,
+ "tokens_seen": 190447616
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007614363089267804,
+ "loss": 1.6163,
+ "theoretical_loss": 4.410446369388497,
+ "tokens_seen": 190513152
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007614202607823471,
+ "loss": 1.7284,
+ "theoretical_loss": 4.410256365226614,
+ "tokens_seen": 190578688
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007614042126379138,
+ "loss": 1.7268,
+ "theoretical_loss": 4.410066444679524,
+ "tokens_seen": 190644224
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007613881644934805,
+ "loss": 1.7091,
+ "theoretical_loss": 4.409876607681706,
+ "tokens_seen": 190709760
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007613721163490472,
+ "loss": 1.5607,
+ "theoretical_loss": 4.409686854167714,
+ "tokens_seen": 190775296
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007613560682046139,
+ "loss": 1.6649,
+ "theoretical_loss": 4.4094971840721735,
+ "tokens_seen": 190840832
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007613400200601806,
+ "loss": 1.7259,
+ "theoretical_loss": 4.409307597329786,
+ "tokens_seen": 190906368
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007613239719157473,
+ "loss": 1.8349,
+ "theoretical_loss": 4.409118093875324,
+ "tokens_seen": 190971904
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761307923771314,
+ "loss": 1.7556,
+ "theoretical_loss": 4.408928673643636,
+ "tokens_seen": 191037440
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612918756268807,
+ "loss": 1.8219,
+ "theoretical_loss": 4.408739336569644,
+ "tokens_seen": 191102976
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612758274824474,
+ "loss": 1.5849,
+ "theoretical_loss": 4.408550082588339,
+ "tokens_seen": 191168512
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612597793380141,
+ "loss": 1.716,
+ "theoretical_loss": 4.408360911634789,
+ "tokens_seen": 191234048
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612437311935808,
+ "loss": 1.6816,
+ "theoretical_loss": 4.408171823644135,
+ "tokens_seen": 191299584
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612276830491475,
+ "loss": 1.7805,
+ "theoretical_loss": 4.407982818551589,
+ "tokens_seen": 191365120
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007612116349047142,
+ "loss": 1.822,
+ "theoretical_loss": 4.407793896292437,
+ "tokens_seen": 191430656
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007611955867602809,
+ "loss": 1.7945,
+ "theoretical_loss": 4.407605056802035,
+ "tokens_seen": 191496192
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007611795386158476,
+ "loss": 1.7431,
+ "theoretical_loss": 4.407416300015816,
+ "tokens_seen": 191561728
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007611634904714143,
+ "loss": 1.562,
+ "theoretical_loss": 4.407227625869283,
+ "tokens_seen": 191627264
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000761147442326981,
+ "loss": 1.7065,
+ "theoretical_loss": 4.407039034298011,
+ "tokens_seen": 191692800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007611313941825477,
+ "loss": 1.55,
+ "theoretical_loss": 4.4068505252376475,
+ "tokens_seen": 191758336
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007611153460381144,
+ "loss": 1.8253,
+ "theoretical_loss": 4.406662098623913,
+ "tokens_seen": 191823872
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610992978936811,
+ "loss": 1.5841,
+ "theoretical_loss": 4.406473754392598,
+ "tokens_seen": 191889408
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610832497492478,
+ "loss": 1.7049,
+ "theoretical_loss": 4.406285492479569,
+ "tokens_seen": 191954944
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610672016048145,
+ "loss": 1.7289,
+ "theoretical_loss": 4.406097312820759,
+ "tokens_seen": 192020480
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610511534603812,
+ "loss": 1.5932,
+ "theoretical_loss": 4.4059092153521755,
+ "tokens_seen": 192086016
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610351053159479,
+ "loss": 1.7163,
+ "theoretical_loss": 4.405721200009898,
+ "tokens_seen": 192151552
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610190571715146,
+ "loss": 1.7145,
+ "theoretical_loss": 4.405533266730077,
+ "tokens_seen": 192217088
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007610030090270813,
+ "loss": 1.7585,
+ "theoretical_loss": 4.405345415448934,
+ "tokens_seen": 192282624
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760986960882648,
+ "loss": 1.809,
+ "theoretical_loss": 4.405157646102762,
+ "tokens_seen": 192348160
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007609709127382147,
+ "loss": 1.6969,
+ "theoretical_loss": 4.404969958627927,
+ "tokens_seen": 192413696
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007609548645937814,
+ "loss": 1.6836,
+ "theoretical_loss": 4.4047823529608605,
+ "tokens_seen": 192479232
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007609388164493481,
+ "loss": 1.6517,
+ "theoretical_loss": 4.404594829038071,
+ "tokens_seen": 192544768
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007609227683049148,
+ "loss": 1.8073,
+ "theoretical_loss": 4.404407386796137,
+ "tokens_seen": 192610304
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007609067201604815,
+ "loss": 1.6976,
+ "theoretical_loss": 4.404220026171704,
+ "tokens_seen": 192675840
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007608906720160482,
+ "loss": 1.8556,
+ "theoretical_loss": 4.404032747101493,
+ "tokens_seen": 192741376
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007608746238716149,
+ "loss": 1.6249,
+ "theoretical_loss": 4.4038455495222895,
+ "tokens_seen": 192806912
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007608585757271816,
+ "loss": 1.7009,
+ "theoretical_loss": 4.403658433370957,
+ "tokens_seen": 192872448
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007608425275827483,
+ "loss": 1.6373,
+ "theoretical_loss": 4.403471398584422,
+ "tokens_seen": 192937984
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760826479438315,
+ "loss": 1.6357,
+ "theoretical_loss": 4.403284445099688,
+ "tokens_seen": 193003520
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007608104312938817,
+ "loss": 1.7354,
+ "theoretical_loss": 4.403097572853823,
+ "tokens_seen": 193069056
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607943831494484,
+ "loss": 1.7761,
+ "theoretical_loss": 4.402910781783968,
+ "tokens_seen": 193134592
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607783350050151,
+ "loss": 1.7687,
+ "theoretical_loss": 4.4027240718273335,
+ "tokens_seen": 193200128
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607622868605818,
+ "loss": 1.6265,
+ "theoretical_loss": 4.4025374429212,
+ "tokens_seen": 193265664
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 71032,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6292566061019897,
+ "objective/train/theoretical_loss": 4.402350895002916,
+ "objective/train/tokens_used": 213791200,
+ "theoretical_loss": 4.402350895002916,
+ "tokens_seen": 193331200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607462387161485,
+ "loss": 1.6732,
+ "theoretical_loss": 4.402350895002916,
+ "tokens_seen": 193331200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607301905717152,
+ "loss": 1.7352,
+ "theoretical_loss": 4.402164428009904,
+ "tokens_seen": 193396736
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007607141424272819,
+ "loss": 1.7918,
+ "theoretical_loss": 4.40197804187965,
+ "tokens_seen": 193462272
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606980942828486,
+ "loss": 1.6467,
+ "theoretical_loss": 4.401791736549714,
+ "tokens_seen": 193527808
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606820461384153,
+ "loss": 1.7642,
+ "theoretical_loss": 4.401605511957724,
+ "tokens_seen": 193593344
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760665997993982,
+ "loss": 1.6585,
+ "theoretical_loss": 4.401419368041377,
+ "tokens_seen": 193658880
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606499498495487,
+ "loss": 1.6439,
+ "theoretical_loss": 4.401233304738438,
+ "tokens_seen": 193724416
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606339017051153,
+ "loss": 1.7171,
+ "theoretical_loss": 4.401047321986745,
+ "tokens_seen": 193789952
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606178535606821,
+ "loss": 1.7524,
+ "theoretical_loss": 4.4008614197242,
+ "tokens_seen": 193855488
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007606018054162487,
+ "loss": 1.7401,
+ "theoretical_loss": 4.400675597888777,
+ "tokens_seen": 193921024
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605857572718155,
+ "loss": 1.6179,
+ "theoretical_loss": 4.400489856418518,
+ "tokens_seen": 193986560
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605697091273821,
+ "loss": 1.6743,
+ "theoretical_loss": 4.4003041952515325,
+ "tokens_seen": 194052096
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605536609829489,
+ "loss": 1.7264,
+ "theoretical_loss": 4.400118614326001,
+ "tokens_seen": 194117632
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605376128385155,
+ "loss": 1.6531,
+ "theoretical_loss": 4.39993311358017,
+ "tokens_seen": 194183168
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605215646940823,
+ "loss": 1.5623,
+ "theoretical_loss": 4.399747692952355,
+ "tokens_seen": 194248704
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007605055165496489,
+ "loss": 1.7761,
+ "theoretical_loss": 4.399562352380941,
+ "tokens_seen": 194314240
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007604894684052156,
+ "loss": 1.7038,
+ "theoretical_loss": 4.399377091804379,
+ "tokens_seen": 194379776
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007604734202607823,
+ "loss": 1.7686,
+ "theoretical_loss": 4.399191911161191,
+ "tokens_seen": 194445312
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760457372116349,
+ "loss": 1.6672,
+ "theoretical_loss": 4.399006810389963,
+ "tokens_seen": 194510848
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007604413239719157,
+ "loss": 1.6675,
+ "theoretical_loss": 4.398821789429354,
+ "tokens_seen": 194576384
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007604252758274824,
+ "loss": 1.844,
+ "theoretical_loss": 4.398636848218084,
+ "tokens_seen": 194641920
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007604092276830491,
+ "loss": 1.7601,
+ "theoretical_loss": 4.398451986694948,
+ "tokens_seen": 194707456
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007603931795386158,
+ "loss": 1.7763,
+ "theoretical_loss": 4.398267204798803,
+ "tokens_seen": 194772992
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007603771313941825,
+ "loss": 1.6735,
+ "theoretical_loss": 4.398082502468577,
+ "tokens_seen": 194838528
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007603610832497492,
+ "loss": 1.5872,
+ "theoretical_loss": 4.397897879643262,
+ "tokens_seen": 194904064
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760345035105316,
+ "loss": 1.6719,
+ "theoretical_loss": 4.39771333626192,
+ "tokens_seen": 194969600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007603289869608827,
+ "loss": 1.7731,
+ "theoretical_loss": 4.397528872263679,
+ "tokens_seen": 195035136
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007603129388164494,
+ "loss": 1.6565,
+ "theoretical_loss": 4.397344487587736,
+ "tokens_seen": 195100672
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602968906720161,
+ "loss": 1.7334,
+ "theoretical_loss": 4.3971601821733515,
+ "tokens_seen": 195166208
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602808425275828,
+ "loss": 1.6964,
+ "theoretical_loss": 4.396975955959856,
+ "tokens_seen": 195231744
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602647943831495,
+ "loss": 1.7469,
+ "theoretical_loss": 4.396791808886643,
+ "tokens_seen": 195297280
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602487462387162,
+ "loss": 1.8126,
+ "theoretical_loss": 4.396607740893179,
+ "tokens_seen": 195362816
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602326980942829,
+ "loss": 1.6569,
+ "theoretical_loss": 4.396423751918991,
+ "tokens_seen": 195428352
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602166499498496,
+ "loss": 1.5221,
+ "theoretical_loss": 4.396239841903674,
+ "tokens_seen": 195493888
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007602006018054163,
+ "loss": 1.6202,
+ "theoretical_loss": 4.396056010786893,
+ "tokens_seen": 195559424
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000760184553660983,
+ "loss": 1.8545,
+ "theoretical_loss": 4.3958722585083745,
+ "tokens_seen": 195624960
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007601685055165497,
+ "loss": 1.7108,
+ "theoretical_loss": 4.395688585007913,
+ "tokens_seen": 195690496
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007601524573721164,
+ "loss": 1.7769,
+ "theoretical_loss": 4.395504990225371,
+ "tokens_seen": 195756032
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007601364092276831,
+ "loss": 1.8607,
+ "theoretical_loss": 4.395321474100673,
+ "tokens_seen": 195821568
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007601203610832498,
+ "loss": 1.7631,
+ "theoretical_loss": 4.395138036573814,
+ "tokens_seen": 195887104
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007601043129388165,
+ "loss": 1.7287,
+ "theoretical_loss": 4.3949546775848525,
+ "tokens_seen": 195952640
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007600882647943832,
+ "loss": 1.6529,
+ "theoretical_loss": 4.394771397073912,
+ "tokens_seen": 196018176
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007600722166499499,
+ "loss": 1.7395,
+ "theoretical_loss": 4.394588194981182,
+ "tokens_seen": 196083712
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007600561685055166,
+ "loss": 1.8479,
+ "theoretical_loss": 4.394405071246919,
+ "tokens_seen": 196149248
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007600401203610833,
+ "loss": 1.733,
+ "theoretical_loss": 4.394222025811445,
+ "tokens_seen": 196214784
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.00076002407221665,
+ "loss": 1.6186,
+ "theoretical_loss": 4.394039058615144,
+ "tokens_seen": 196280320
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007600080240722167,
+ "loss": 1.6977,
+ "theoretical_loss": 4.39385616959847,
+ "tokens_seen": 196345856
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599919759277834,
+ "loss": 1.7501,
+ "theoretical_loss": 4.393673358701939,
+ "tokens_seen": 196411392
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599759277833501,
+ "loss": 1.5986,
+ "theoretical_loss": 4.393490625866132,
+ "tokens_seen": 196476928
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599598796389168,
+ "loss": 1.777,
+ "theoretical_loss": 4.393307971031697,
+ "tokens_seen": 196542464
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 71932,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4793413877487183,
+ "objective/train/theoretical_loss": 4.3931253941393456,
+ "objective/train/tokens_used": 217068000,
+ "theoretical_loss": 4.3931253941393456,
+ "tokens_seen": 196608000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599438314944835,
+ "loss": 1.5309,
+ "theoretical_loss": 4.3931253941393456,
+ "tokens_seen": 196608000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599277833500502,
+ "loss": 1.7435,
+ "theoretical_loss": 4.392942895129854,
+ "tokens_seen": 196673536
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007599117352056169,
+ "loss": 1.6521,
+ "theoretical_loss": 4.392760473944064,
+ "tokens_seen": 196739072
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007598956870611836,
+ "loss": 1.708,
+ "theoretical_loss": 4.392578130522883,
+ "tokens_seen": 196804608
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007598796389167503,
+ "loss": 1.7905,
+ "theoretical_loss": 4.392395864807279,
+ "tokens_seen": 196870144
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759863590772317,
+ "loss": 1.7152,
+ "theoretical_loss": 4.392213676738288,
+ "tokens_seen": 196935680
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007598475426278837,
+ "loss": 1.8214,
+ "theoretical_loss": 4.3920315662570095,
+ "tokens_seen": 197001216
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007598314944834504,
+ "loss": 1.7028,
+ "theoretical_loss": 4.391849533304607,
+ "tokens_seen": 197066752
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007598154463390171,
+ "loss": 1.6381,
+ "theoretical_loss": 4.391667577822308,
+ "tokens_seen": 197132288
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597993981945838,
+ "loss": 1.7244,
+ "theoretical_loss": 4.391485699751406,
+ "tokens_seen": 197197824
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597833500501505,
+ "loss": 1.7536,
+ "theoretical_loss": 4.391303899033253,
+ "tokens_seen": 197263360
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597673019057172,
+ "loss": 1.6795,
+ "theoretical_loss": 4.391122175609272,
+ "tokens_seen": 197328896
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597512537612839,
+ "loss": 1.6457,
+ "theoretical_loss": 4.390940529420946,
+ "tokens_seen": 197394432
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597352056168506,
+ "loss": 1.6911,
+ "theoretical_loss": 4.39075896040982,
+ "tokens_seen": 197459968
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007597191574724173,
+ "loss": 1.5381,
+ "theoretical_loss": 4.390577468517508,
+ "tokens_seen": 197525504
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759703109327984,
+ "loss": 1.7558,
+ "theoretical_loss": 4.390396053685682,
+ "tokens_seen": 197591040
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596870611835507,
+ "loss": 1.7281,
+ "theoretical_loss": 4.39021471585608,
+ "tokens_seen": 197656576
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596710130391174,
+ "loss": 1.6057,
+ "theoretical_loss": 4.390033454970504,
+ "tokens_seen": 197722112
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596549648946841,
+ "loss": 1.6983,
+ "theoretical_loss": 4.389852270970818,
+ "tokens_seen": 197787648
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596389167502508,
+ "loss": 1.7131,
+ "theoretical_loss": 4.389671163798949,
+ "tokens_seen": 197853184
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596228686058175,
+ "loss": 1.7804,
+ "theoretical_loss": 4.3894901333968885,
+ "tokens_seen": 197918720
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007596068204613842,
+ "loss": 1.7999,
+ "theoretical_loss": 4.38930917970669,
+ "tokens_seen": 197984256
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007595907723169509,
+ "loss": 1.7063,
+ "theoretical_loss": 4.3891283026704695,
+ "tokens_seen": 198049792
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007595747241725176,
+ "loss": 1.7568,
+ "theoretical_loss": 4.388947502230407,
+ "tokens_seen": 198115328
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007595586760280843,
+ "loss": 1.6771,
+ "theoretical_loss": 4.388766778328744,
+ "tokens_seen": 198180864
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759542627883651,
+ "loss": 1.6316,
+ "theoretical_loss": 4.388586130907786,
+ "tokens_seen": 198246400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007595265797392177,
+ "loss": 1.7721,
+ "theoretical_loss": 4.388405559909899,
+ "tokens_seen": 198311936
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007595105315947844,
+ "loss": 1.672,
+ "theoretical_loss": 4.3882250652775125,
+ "tokens_seen": 198377472
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594944834503511,
+ "loss": 1.6478,
+ "theoretical_loss": 4.38804464695312,
+ "tokens_seen": 198443008
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594784353059178,
+ "loss": 1.648,
+ "theoretical_loss": 4.387864304879275,
+ "tokens_seen": 198508544
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594623871614845,
+ "loss": 1.7571,
+ "theoretical_loss": 4.387684038998595,
+ "tokens_seen": 198574080
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594463390170512,
+ "loss": 1.7198,
+ "theoretical_loss": 4.387503849253757,
+ "tokens_seen": 198639616
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594302908726179,
+ "loss": 1.7731,
+ "theoretical_loss": 4.387323735587502,
+ "tokens_seen": 198705152
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007594142427281846,
+ "loss": 1.7331,
+ "theoretical_loss": 4.387143697942634,
+ "tokens_seen": 198770688
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593981945837513,
+ "loss": 1.8219,
+ "theoretical_loss": 4.386963736262016,
+ "tokens_seen": 198836224
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759382146439318,
+ "loss": 1.685,
+ "theoretical_loss": 4.386783850488575,
+ "tokens_seen": 198901760
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593660982948847,
+ "loss": 1.7198,
+ "theoretical_loss": 4.386604040565299,
+ "tokens_seen": 198967296
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593500501504514,
+ "loss": 1.7957,
+ "theoretical_loss": 4.386424306435236,
+ "tokens_seen": 199032832
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593340020060181,
+ "loss": 1.7063,
+ "theoretical_loss": 4.386244648041498,
+ "tokens_seen": 199098368
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593179538615848,
+ "loss": 1.6459,
+ "theoretical_loss": 4.386065065327258,
+ "tokens_seen": 199163904
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007593019057171515,
+ "loss": 1.6441,
+ "theoretical_loss": 4.385885558235747,
+ "tokens_seen": 199229440
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007592858575727182,
+ "loss": 1.7275,
+ "theoretical_loss": 4.385706126710264,
+ "tokens_seen": 199294976
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759269809428285,
+ "loss": 1.7563,
+ "theoretical_loss": 4.385526770694161,
+ "tokens_seen": 199360512
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007592537612838516,
+ "loss": 1.6168,
+ "theoretical_loss": 4.385347490130858,
+ "tokens_seen": 199426048
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007592377131394182,
+ "loss": 1.678,
+ "theoretical_loss": 4.385168284963832,
+ "tokens_seen": 199491584
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759221664994985,
+ "loss": 1.7211,
+ "theoretical_loss": 4.384989155136621,
+ "tokens_seen": 199557120
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007592056168505516,
+ "loss": 1.6404,
+ "theoretical_loss": 4.384810100592826,
+ "tokens_seen": 199622656
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007591895687061184,
+ "loss": 1.7634,
+ "theoretical_loss": 4.384631121276108,
+ "tokens_seen": 199688192
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759173520561685,
+ "loss": 1.649,
+ "theoretical_loss": 4.384452217130187,
+ "tokens_seen": 199753728
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007591574724172518,
+ "loss": 1.6701,
+ "theoretical_loss": 4.384273388098845,
+ "tokens_seen": 199819264
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 73160,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7228140830993652,
+ "objective/train/theoretical_loss": 4.384094634125924,
+ "objective/train/tokens_used": 220344800,
+ "theoretical_loss": 4.384094634125924,
+ "tokens_seen": 199884800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007591414242728184,
+ "loss": 1.6614,
+ "theoretical_loss": 4.384094634125924,
+ "tokens_seen": 199884800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007591253761283852,
+ "loss": 1.6742,
+ "theoretical_loss": 4.383915955155326,
+ "tokens_seen": 199950336
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007591093279839518,
+ "loss": 1.6602,
+ "theoretical_loss": 4.383737351131016,
+ "tokens_seen": 200015872
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007590932798395186,
+ "loss": 1.7915,
+ "theoretical_loss": 4.383558821997014,
+ "tokens_seen": 200081408
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007590772316950852,
+ "loss": 1.7205,
+ "theoretical_loss": 4.383380367697406,
+ "tokens_seen": 200146944
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759061183550652,
+ "loss": 1.7392,
+ "theoretical_loss": 4.383201988176333,
+ "tokens_seen": 200212480
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007590451354062186,
+ "loss": 1.735,
+ "theoretical_loss": 4.383023683377999,
+ "tokens_seen": 200278016
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007590290872617854,
+ "loss": 1.796,
+ "theoretical_loss": 4.382845453246668,
+ "tokens_seen": 200343552
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000759013039117352,
+ "loss": 1.705,
+ "theoretical_loss": 4.382667297726661,
+ "tokens_seen": 200409088
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589969909729188,
+ "loss": 1.6234,
+ "theoretical_loss": 4.3824892167623615,
+ "tokens_seen": 200474624
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589809428284855,
+ "loss": 1.725,
+ "theoretical_loss": 4.382311210298212,
+ "tokens_seen": 200540160
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589648946840522,
+ "loss": 1.7048,
+ "theoretical_loss": 4.3821332782787135,
+ "tokens_seen": 200605696
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589488465396189,
+ "loss": 1.718,
+ "theoretical_loss": 4.381955420648428,
+ "tokens_seen": 200671232
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589327983951856,
+ "loss": 1.7411,
+ "theoretical_loss": 4.381777637351974,
+ "tokens_seen": 200736768
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007589167502507523,
+ "loss": 1.8566,
+ "theoretical_loss": 4.381599928334033,
+ "tokens_seen": 200802304
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000758900702106319,
+ "loss": 1.7141,
+ "theoretical_loss": 4.381422293539345,
+ "tokens_seen": 200867840
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588846539618857,
+ "loss": 1.695,
+ "theoretical_loss": 4.381244732912706,
+ "tokens_seen": 200933376
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588686058174524,
+ "loss": 1.6169,
+ "theoretical_loss": 4.381067246398975,
+ "tokens_seen": 200998912
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588525576730191,
+ "loss": 1.6962,
+ "theoretical_loss": 4.380889833943066,
+ "tokens_seen": 201064448
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588365095285858,
+ "loss": 1.6986,
+ "theoretical_loss": 4.380712495489957,
+ "tokens_seen": 201129984
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588204613841525,
+ "loss": 1.7617,
+ "theoretical_loss": 4.380535230984679,
+ "tokens_seen": 201195520
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007588044132397192,
+ "loss": 1.6525,
+ "theoretical_loss": 4.380358040372325,
+ "tokens_seen": 201261056
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007587883650952859,
+ "loss": 1.7725,
+ "theoretical_loss": 4.380180923598047,
+ "tokens_seen": 201326592
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007587723169508526,
+ "loss": 1.7453,
+ "theoretical_loss": 4.380003880607055,
+ "tokens_seen": 201392128
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007587562688064193,
+ "loss": 1.787,
+ "theoretical_loss": 4.379826911344617,
+ "tokens_seen": 201457664
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000758740220661986,
+ "loss": 1.7013,
+ "theoretical_loss": 4.3796500157560585,
+ "tokens_seen": 201523200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007587241725175527,
+ "loss": 1.7672,
+ "theoretical_loss": 4.379473193786765,
+ "tokens_seen": 201588736
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007587081243731194,
+ "loss": 1.7227,
+ "theoretical_loss": 4.37929644538218,
+ "tokens_seen": 201654272
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586920762286861,
+ "loss": 1.7301,
+ "theoretical_loss": 4.379119770487805,
+ "tokens_seen": 201719808
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586760280842528,
+ "loss": 1.7752,
+ "theoretical_loss": 4.378943169049198,
+ "tokens_seen": 201785344
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586599799398195,
+ "loss": 1.7993,
+ "theoretical_loss": 4.378766641011978,
+ "tokens_seen": 201850880
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586439317953862,
+ "loss": 1.6976,
+ "theoretical_loss": 4.378590186321819,
+ "tokens_seen": 201916416
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586278836509529,
+ "loss": 1.73,
+ "theoretical_loss": 4.378413804924454,
+ "tokens_seen": 201981952
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007586118355065196,
+ "loss": 1.6686,
+ "theoretical_loss": 4.378237496765674,
+ "tokens_seen": 202047488
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007585957873620863,
+ "loss": 1.6747,
+ "theoretical_loss": 4.378061261791328,
+ "tokens_seen": 202113024
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000758579739217653,
+ "loss": 1.7438,
+ "theoretical_loss": 4.377885099947321,
+ "tokens_seen": 202178560
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007585636910732197,
+ "loss": 1.6094,
+ "theoretical_loss": 4.3777090111796175,
+ "tokens_seen": 202244096
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007585476429287864,
+ "loss": 1.7414,
+ "theoretical_loss": 4.377532995434238,
+ "tokens_seen": 202309632
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007585315947843531,
+ "loss": 1.7977,
+ "theoretical_loss": 4.37735705265726,
+ "tokens_seen": 202375168
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007585155466399198,
+ "loss": 1.6395,
+ "theoretical_loss": 4.3771811827948195,
+ "tokens_seen": 202440704
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584994984954865,
+ "loss": 1.7897,
+ "theoretical_loss": 4.37700538579311,
+ "tokens_seen": 202506240
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584834503510532,
+ "loss": 1.6584,
+ "theoretical_loss": 4.37682966159838,
+ "tokens_seen": 202571776
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584674022066199,
+ "loss": 1.6479,
+ "theoretical_loss": 4.376654010156939,
+ "tokens_seen": 202637312
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584513540621866,
+ "loss": 1.6911,
+ "theoretical_loss": 4.376478431415148,
+ "tokens_seen": 202702848
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584353059177533,
+ "loss": 1.6819,
+ "theoretical_loss": 4.376302925319427,
+ "tokens_seen": 202768384
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.00075841925777332,
+ "loss": 1.7279,
+ "theoretical_loss": 4.3761274918162565,
+ "tokens_seen": 202833920
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007584032096288867,
+ "loss": 1.5278,
+ "theoretical_loss": 4.375952130852169,
+ "tokens_seen": 202899456
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583871614844534,
+ "loss": 1.5301,
+ "theoretical_loss": 4.375776842373755,
+ "tokens_seen": 202964992
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583711133400201,
+ "loss": 1.7144,
+ "theoretical_loss": 4.375601626327662,
+ "tokens_seen": 203030528
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583550651955868,
+ "loss": 1.6743,
+ "theoretical_loss": 4.375426482660595,
+ "tokens_seen": 203096064
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 74031,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6785008907318115,
+ "objective/train/theoretical_loss": 4.375251411319312,
+ "objective/train/tokens_used": 223621600,
+ "theoretical_loss": 4.375251411319312,
+ "tokens_seen": 203161600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583390170511535,
+ "loss": 1.7139,
+ "theoretical_loss": 4.375251411319312,
+ "tokens_seen": 203161600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583229689067202,
+ "loss": 1.6417,
+ "theoretical_loss": 4.375076412250632,
+ "tokens_seen": 203227136
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007583069207622869,
+ "loss": 1.6601,
+ "theoretical_loss": 4.374901485401426,
+ "tokens_seen": 203292672
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007582908726178536,
+ "loss": 1.6632,
+ "theoretical_loss": 4.374726630718624,
+ "tokens_seen": 203358208
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007582748244734203,
+ "loss": 1.6369,
+ "theoretical_loss": 4.37455184814921,
+ "tokens_seen": 203423744
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000758258776328987,
+ "loss": 1.5944,
+ "theoretical_loss": 4.374377137640225,
+ "tokens_seen": 203489280
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007582427281845537,
+ "loss": 1.633,
+ "theoretical_loss": 4.374202499138768,
+ "tokens_seen": 203554816
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007582266800401204,
+ "loss": 1.6212,
+ "theoretical_loss": 4.374027932591989,
+ "tokens_seen": 203620352
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007582106318956871,
+ "loss": 1.6938,
+ "theoretical_loss": 4.373853437947099,
+ "tokens_seen": 203685888
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581945837512538,
+ "loss": 1.7528,
+ "theoretical_loss": 4.37367901515136,
+ "tokens_seen": 203751424
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581785356068205,
+ "loss": 1.7577,
+ "theoretical_loss": 4.373504664152094,
+ "tokens_seen": 203816960
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581624874623872,
+ "loss": 1.5288,
+ "theoretical_loss": 4.373330384896676,
+ "tokens_seen": 203882496
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581464393179539,
+ "loss": 1.6905,
+ "theoretical_loss": 4.373156177332536,
+ "tokens_seen": 203948032
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581303911735206,
+ "loss": 1.7408,
+ "theoretical_loss": 4.372982041407161,
+ "tokens_seen": 204013568
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007581143430290873,
+ "loss": 1.64,
+ "theoretical_loss": 4.372807977068092,
+ "tokens_seen": 204079104
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000758098294884654,
+ "loss": 1.6473,
+ "theoretical_loss": 4.372633984262928,
+ "tokens_seen": 204144640
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580822467402207,
+ "loss": 1.6681,
+ "theoretical_loss": 4.372460062939318,
+ "tokens_seen": 204210176
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580661985957874,
+ "loss": 1.6786,
+ "theoretical_loss": 4.372286213044972,
+ "tokens_seen": 204275712
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580501504513541,
+ "loss": 1.7793,
+ "theoretical_loss": 4.37211243452765,
+ "tokens_seen": 204341248
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580341023069208,
+ "loss": 1.5185,
+ "theoretical_loss": 4.37193872733517,
+ "tokens_seen": 204406784
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580180541624875,
+ "loss": 1.7019,
+ "theoretical_loss": 4.371765091415404,
+ "tokens_seen": 204472320
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007580020060180542,
+ "loss": 1.7595,
+ "theoretical_loss": 4.371591526716279,
+ "tokens_seen": 204537856
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007579859578736209,
+ "loss": 1.7805,
+ "theoretical_loss": 4.371418033185777,
+ "tokens_seen": 204603392
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007579699097291876,
+ "loss": 1.5919,
+ "theoretical_loss": 4.3712446107719325,
+ "tokens_seen": 204668928
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007579538615847543,
+ "loss": 1.7362,
+ "theoretical_loss": 4.371071259422838,
+ "tokens_seen": 204734464
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000757937813440321,
+ "loss": 1.6875,
+ "theoretical_loss": 4.370897979086636,
+ "tokens_seen": 204800000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007579217652958877,
+ "loss": 1.8177,
+ "theoretical_loss": 4.370724769711529,
+ "tokens_seen": 204865536
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007579057171514544,
+ "loss": 1.7856,
+ "theoretical_loss": 4.370551631245769,
+ "tokens_seen": 204931072
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578896690070211,
+ "loss": 1.6602,
+ "theoretical_loss": 4.370378563637667,
+ "tokens_seen": 204996608
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578736208625878,
+ "loss": 1.7391,
+ "theoretical_loss": 4.3702055668355815,
+ "tokens_seen": 205062144
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578575727181545,
+ "loss": 1.7766,
+ "theoretical_loss": 4.370032640787931,
+ "tokens_seen": 205127680
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578415245737211,
+ "loss": 1.7982,
+ "theoretical_loss": 4.369859785443188,
+ "tokens_seen": 205193216
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578254764292879,
+ "loss": 1.6197,
+ "theoretical_loss": 4.369687000749873,
+ "tokens_seen": 205258752
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007578094282848545,
+ "loss": 1.7441,
+ "theoretical_loss": 4.3695142866565675,
+ "tokens_seen": 205324288
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577933801404213,
+ "loss": 1.6629,
+ "theoretical_loss": 4.369341643111902,
+ "tokens_seen": 205389824
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577773319959879,
+ "loss": 1.7171,
+ "theoretical_loss": 4.369169070064563,
+ "tokens_seen": 205455360
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577612838515547,
+ "loss": 1.6443,
+ "theoretical_loss": 4.368996567463292,
+ "tokens_seen": 205520896
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577452357071213,
+ "loss": 1.6624,
+ "theoretical_loss": 4.3688241352568795,
+ "tokens_seen": 205586432
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577291875626881,
+ "loss": 1.8175,
+ "theoretical_loss": 4.368651773394173,
+ "tokens_seen": 205651968
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007577131394182547,
+ "loss": 1.5977,
+ "theoretical_loss": 4.368479481824075,
+ "tokens_seen": 205717504
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576970912738215,
+ "loss": 1.6515,
+ "theoretical_loss": 4.368307260495536,
+ "tokens_seen": 205783040
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576810431293881,
+ "loss": 1.6509,
+ "theoretical_loss": 4.368135109357564,
+ "tokens_seen": 205848576
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576649949849549,
+ "loss": 1.7278,
+ "theoretical_loss": 4.36796302835922,
+ "tokens_seen": 205914112
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576489468405215,
+ "loss": 1.5491,
+ "theoretical_loss": 4.367791017449618,
+ "tokens_seen": 205979648
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576328986960883,
+ "loss": 1.555,
+ "theoretical_loss": 4.367619076577923,
+ "tokens_seen": 206045184
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000757616850551655,
+ "loss": 1.6893,
+ "theoretical_loss": 4.367447205693354,
+ "tokens_seen": 206110720
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007576008024072217,
+ "loss": 1.7705,
+ "theoretical_loss": 4.367275404745186,
+ "tokens_seen": 206176256
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575847542627884,
+ "loss": 1.7363,
+ "theoretical_loss": 4.367103673682742,
+ "tokens_seen": 206241792
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575687061183551,
+ "loss": 1.7842,
+ "theoretical_loss": 4.3669320124554,
+ "tokens_seen": 206307328
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575526579739218,
+ "loss": 1.6683,
+ "theoretical_loss": 4.366760421012592,
+ "tokens_seen": 206372864
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 75241,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.8373074531555176,
+ "objective/train/theoretical_loss": 4.366588899303801,
+ "objective/train/tokens_used": 226898400,
+ "theoretical_loss": 4.366588899303801,
+ "tokens_seen": 206438400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575366098294885,
+ "loss": 1.7905,
+ "theoretical_loss": 4.366588899303801,
+ "tokens_seen": 206438400
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575205616850552,
+ "loss": 1.7056,
+ "theoretical_loss": 4.366417447278565,
+ "tokens_seen": 206503936
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007575045135406219,
+ "loss": 1.6869,
+ "theoretical_loss": 4.366246064886468,
+ "tokens_seen": 206569472
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007574884653961886,
+ "loss": 1.7188,
+ "theoretical_loss": 4.3660747520771555,
+ "tokens_seen": 206635008
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007574724172517553,
+ "loss": 1.6279,
+ "theoretical_loss": 4.36590350880032,
+ "tokens_seen": 206700544
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000757456369107322,
+ "loss": 1.6978,
+ "theoretical_loss": 4.365732335005706,
+ "tokens_seen": 206766080
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007574403209628887,
+ "loss": 1.647,
+ "theoretical_loss": 4.365561230643112,
+ "tokens_seen": 206831616
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007574242728184554,
+ "loss": 1.691,
+ "theoretical_loss": 4.365390195662389,
+ "tokens_seen": 206897152
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007574082246740221,
+ "loss": 1.7532,
+ "theoretical_loss": 4.365219230013438,
+ "tokens_seen": 206962688
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573921765295888,
+ "loss": 1.7129,
+ "theoretical_loss": 4.3650483336462145,
+ "tokens_seen": 207028224
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573761283851555,
+ "loss": 1.7066,
+ "theoretical_loss": 4.364877506510723,
+ "tokens_seen": 207093760
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573600802407222,
+ "loss": 1.6376,
+ "theoretical_loss": 4.3647067485570235,
+ "tokens_seen": 207159296
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573440320962889,
+ "loss": 1.5943,
+ "theoretical_loss": 4.364536059735224,
+ "tokens_seen": 207224832
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573279839518556,
+ "loss": 1.7438,
+ "theoretical_loss": 4.364365439995488,
+ "tokens_seen": 207290368
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007573119358074223,
+ "loss": 1.7198,
+ "theoretical_loss": 4.3641948892880285,
+ "tokens_seen": 207355904
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000757295887662989,
+ "loss": 1.7217,
+ "theoretical_loss": 4.364024407563109,
+ "tokens_seen": 207421440
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007572798395185557,
+ "loss": 1.6305,
+ "theoretical_loss": 4.363853994771048,
+ "tokens_seen": 207486976
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007572637913741224,
+ "loss": 1.713,
+ "theoretical_loss": 4.363683650862212,
+ "tokens_seen": 207552512
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007572477432296891,
+ "loss": 1.6457,
+ "theoretical_loss": 4.36351337578702,
+ "tokens_seen": 207618048
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007572316950852558,
+ "loss": 1.6328,
+ "theoretical_loss": 4.363343169495945,
+ "tokens_seen": 207683584
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007572156469408225,
+ "loss": 1.717,
+ "theoretical_loss": 4.363173031939506,
+ "tokens_seen": 207749120
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571995987963892,
+ "loss": 1.6263,
+ "theoretical_loss": 4.363002963068277,
+ "tokens_seen": 207814656
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571835506519559,
+ "loss": 1.7193,
+ "theoretical_loss": 4.362832962832883,
+ "tokens_seen": 207880192
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571675025075226,
+ "loss": 1.7143,
+ "theoretical_loss": 4.362663031183999,
+ "tokens_seen": 207945728
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571514543630893,
+ "loss": 1.6003,
+ "theoretical_loss": 4.36249316807235,
+ "tokens_seen": 208011264
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000757135406218656,
+ "loss": 1.7562,
+ "theoretical_loss": 4.3623233734487155,
+ "tokens_seen": 208076800
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571193580742227,
+ "loss": 1.7425,
+ "theoretical_loss": 4.362153647263921,
+ "tokens_seen": 208142336
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007571033099297894,
+ "loss": 1.6691,
+ "theoretical_loss": 4.361983989468847,
+ "tokens_seen": 208207872
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570872617853561,
+ "loss": 1.5959,
+ "theoretical_loss": 4.3618144000144214,
+ "tokens_seen": 208273408
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570712136409228,
+ "loss": 1.6334,
+ "theoretical_loss": 4.361644878851626,
+ "tokens_seen": 208338944
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570551654964895,
+ "loss": 1.832,
+ "theoretical_loss": 4.36147542593149,
+ "tokens_seen": 208404480
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570391173520562,
+ "loss": 1.7518,
+ "theoretical_loss": 4.361306041205095,
+ "tokens_seen": 208470016
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570230692076229,
+ "loss": 1.7394,
+ "theoretical_loss": 4.361136724623573,
+ "tokens_seen": 208535552
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007570070210631896,
+ "loss": 1.6212,
+ "theoretical_loss": 4.360967476138105,
+ "tokens_seen": 208601088
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007569909729187563,
+ "loss": 1.732,
+ "theoretical_loss": 4.360798295699925,
+ "tokens_seen": 208666624
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756974924774323,
+ "loss": 1.7454,
+ "theoretical_loss": 4.360629183260313,
+ "tokens_seen": 208732160
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007569588766298897,
+ "loss": 1.6911,
+ "theoretical_loss": 4.360460138770604,
+ "tokens_seen": 208797696
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007569428284854564,
+ "loss": 1.5386,
+ "theoretical_loss": 4.360291162182179,
+ "tokens_seen": 208863232
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007569267803410231,
+ "loss": 1.6819,
+ "theoretical_loss": 4.360122253446472,
+ "tokens_seen": 208928768
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007569107321965898,
+ "loss": 1.7132,
+ "theoretical_loss": 4.359953412514965,
+ "tokens_seen": 208994304
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007568946840521565,
+ "loss": 1.6954,
+ "theoretical_loss": 4.359784639339191,
+ "tokens_seen": 209059840
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007568786359077232,
+ "loss": 1.5804,
+ "theoretical_loss": 4.359615933870732,
+ "tokens_seen": 209125376
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007568625877632899,
+ "loss": 1.826,
+ "theoretical_loss": 4.359447296061219,
+ "tokens_seen": 209190912
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007568465396188566,
+ "loss": 1.6614,
+ "theoretical_loss": 4.359278725862336,
+ "tokens_seen": 209256448
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007568304914744233,
+ "loss": 1.7834,
+ "theoretical_loss": 4.359110223225814,
+ "tokens_seen": 209321984
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.00075681444332999,
+ "loss": 1.7298,
+ "theoretical_loss": 4.358941788103433,
+ "tokens_seen": 209387520
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567983951855567,
+ "loss": 1.6115,
+ "theoretical_loss": 4.358773420447024,
+ "tokens_seen": 209453056
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567823470411234,
+ "loss": 1.7582,
+ "theoretical_loss": 4.358605120208469,
+ "tokens_seen": 209518592
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567662988966901,
+ "loss": 1.6202,
+ "theoretical_loss": 4.358436887339694,
+ "tokens_seen": 209584128
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567502507522569,
+ "loss": 1.6705,
+ "theoretical_loss": 4.35826872179268,
+ "tokens_seen": 209649664
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 76142,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.782550573348999,
+ "objective/train/theoretical_loss": 4.358100623519453,
+ "objective/train/tokens_used": 230175200,
+ "theoretical_loss": 4.358100623519453,
+ "tokens_seen": 209715200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567342026078235,
+ "loss": 1.6927,
+ "theoretical_loss": 4.358100623519453,
+ "tokens_seen": 209715200
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567181544633903,
+ "loss": 1.6178,
+ "theoretical_loss": 4.357932592472092,
+ "tokens_seen": 209780736
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007567021063189569,
+ "loss": 1.6931,
+ "theoretical_loss": 4.357764628602721,
+ "tokens_seen": 209846272
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566860581745237,
+ "loss": 1.6771,
+ "theoretical_loss": 4.357596731863517,
+ "tokens_seen": 209911808
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566700100300903,
+ "loss": 1.7497,
+ "theoretical_loss": 4.357428902206704,
+ "tokens_seen": 209977344
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566539618856571,
+ "loss": 1.792,
+ "theoretical_loss": 4.357261139584553,
+ "tokens_seen": 210042880
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566379137412237,
+ "loss": 1.7182,
+ "theoretical_loss": 4.357093443949388,
+ "tokens_seen": 210108416
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566218655967905,
+ "loss": 1.7378,
+ "theoretical_loss": 4.356925815253579,
+ "tokens_seen": 210173952
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007566058174523571,
+ "loss": 1.623,
+ "theoretical_loss": 4.356758253449544,
+ "tokens_seen": 210239488
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007565897693079239,
+ "loss": 1.6864,
+ "theoretical_loss": 4.356590758489753,
+ "tokens_seen": 210305024
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007565737211634905,
+ "loss": 1.6433,
+ "theoretical_loss": 4.356423330326722,
+ "tokens_seen": 210370560
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007565576730190572,
+ "loss": 1.6874,
+ "theoretical_loss": 4.356255968913015,
+ "tokens_seen": 210436096
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756541624874624,
+ "loss": 1.6989,
+ "theoretical_loss": 4.356088674201246,
+ "tokens_seen": 210501632
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007565255767301906,
+ "loss": 1.6619,
+ "theoretical_loss": 4.355921446144077,
+ "tokens_seen": 210567168
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007565095285857573,
+ "loss": 1.6612,
+ "theoretical_loss": 4.355754284694218,
+ "tokens_seen": 210632704
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756493480441324,
+ "loss": 1.8118,
+ "theoretical_loss": 4.355587189804427,
+ "tokens_seen": 210698240
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007564774322968907,
+ "loss": 1.6183,
+ "theoretical_loss": 4.355420161427512,
+ "tokens_seen": 210763776
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007564613841524574,
+ "loss": 1.6552,
+ "theoretical_loss": 4.355253199516326,
+ "tokens_seen": 210829312
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007564453360080242,
+ "loss": 1.6956,
+ "theoretical_loss": 4.355086304023773,
+ "tokens_seen": 210894848
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007564292878635908,
+ "loss": 1.6164,
+ "theoretical_loss": 4.354919474902804,
+ "tokens_seen": 210960384
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007564132397191574,
+ "loss": 1.6407,
+ "theoretical_loss": 4.354752712106417,
+ "tokens_seen": 211025920
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563971915747242,
+ "loss": 1.6663,
+ "theoretical_loss": 4.354586015587659,
+ "tokens_seen": 211091456
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563811434302908,
+ "loss": 1.7245,
+ "theoretical_loss": 4.354419385299623,
+ "tokens_seen": 211156992
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563650952858576,
+ "loss": 1.6848,
+ "theoretical_loss": 4.354252821195453,
+ "tokens_seen": 211222528
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563490471414242,
+ "loss": 1.7278,
+ "theoretical_loss": 4.354086323228338,
+ "tokens_seen": 211288064
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756332998996991,
+ "loss": 1.7467,
+ "theoretical_loss": 4.353919891351516,
+ "tokens_seen": 211353600
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563169508525576,
+ "loss": 1.6958,
+ "theoretical_loss": 4.35375352551827,
+ "tokens_seen": 211419136
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007563009027081244,
+ "loss": 1.7661,
+ "theoretical_loss": 4.353587225681935,
+ "tokens_seen": 211484672
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756284854563691,
+ "loss": 1.7008,
+ "theoretical_loss": 4.35342099179589,
+ "tokens_seen": 211550208
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007562688064192578,
+ "loss": 1.6198,
+ "theoretical_loss": 4.353254823813561,
+ "tokens_seen": 211615744
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007562527582748245,
+ "loss": 1.6464,
+ "theoretical_loss": 4.353088721688424,
+ "tokens_seen": 211681280
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007562367101303912,
+ "loss": 1.6186,
+ "theoretical_loss": 4.352922685374001,
+ "tokens_seen": 211746816
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007562206619859579,
+ "loss": 1.7021,
+ "theoretical_loss": 4.352756714823859,
+ "tokens_seen": 211812352
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007562046138415246,
+ "loss": 1.6447,
+ "theoretical_loss": 4.352590809991616,
+ "tokens_seen": 211877888
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007561885656970913,
+ "loss": 1.7092,
+ "theoretical_loss": 4.352424970830935,
+ "tokens_seen": 211943424
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756172517552658,
+ "loss": 1.6352,
+ "theoretical_loss": 4.352259197295525,
+ "tokens_seen": 212008960
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007561564694082247,
+ "loss": 1.6442,
+ "theoretical_loss": 4.352093489339143,
+ "tokens_seen": 212074496
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007561404212637914,
+ "loss": 1.6247,
+ "theoretical_loss": 4.351927846915595,
+ "tokens_seen": 212140032
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007561243731193581,
+ "loss": 1.7138,
+ "theoretical_loss": 4.35176226997873,
+ "tokens_seen": 212205568
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007561083249749248,
+ "loss": 1.7305,
+ "theoretical_loss": 4.351596758482445,
+ "tokens_seen": 212271104
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007560922768304915,
+ "loss": 1.6121,
+ "theoretical_loss": 4.351431312380684,
+ "tokens_seen": 212336640
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007560762286860582,
+ "loss": 1.6804,
+ "theoretical_loss": 4.351265931627439,
+ "tokens_seen": 212402176
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007560601805416249,
+ "loss": 1.664,
+ "theoretical_loss": 4.351100616176747,
+ "tokens_seen": 212467712
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007560441323971916,
+ "loss": 1.6836,
+ "theoretical_loss": 4.350935365982691,
+ "tokens_seen": 212533248
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007560280842527583,
+ "loss": 1.7276,
+ "theoretical_loss": 4.350770180999402,
+ "tokens_seen": 212598784
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000756012036108325,
+ "loss": 1.6969,
+ "theoretical_loss": 4.3506050611810565,
+ "tokens_seen": 212664320
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559959879638917,
+ "loss": 1.5556,
+ "theoretical_loss": 4.350440006481877,
+ "tokens_seen": 212729856
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559799398194584,
+ "loss": 1.7229,
+ "theoretical_loss": 4.350275016856132,
+ "tokens_seen": 212795392
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559638916750251,
+ "loss": 1.6752,
+ "theoretical_loss": 4.350110092258138,
+ "tokens_seen": 212860928
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559478435305918,
+ "loss": 1.6807,
+ "theoretical_loss": 4.349945232642256,
+ "tokens_seen": 212926464
+ },
+ {
+ "epoch": 0.06,
+ "objective/train/docs_used": 77362,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6674554347991943,
+ "objective/train/theoretical_loss": 4.3497804379628935,
+ "objective/train/tokens_used": 233452000,
+ "theoretical_loss": 4.3497804379628935,
+ "tokens_seen": 212992000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559317953861585,
+ "loss": 1.7286,
+ "theoretical_loss": 4.3497804379628935,
+ "tokens_seen": 212992000
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007559157472417252,
+ "loss": 1.6309,
+ "theoretical_loss": 4.349615708174504,
+ "tokens_seen": 213057536
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558996990972919,
+ "loss": 1.589,
+ "theoretical_loss": 4.3494510432315865,
+ "tokens_seen": 213123072
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558836509528586,
+ "loss": 1.6323,
+ "theoretical_loss": 4.349286443088687,
+ "tokens_seen": 213188608
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558676028084253,
+ "loss": 1.6095,
+ "theoretical_loss": 4.349121907700396,
+ "tokens_seen": 213254144
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000755851554663992,
+ "loss": 1.7698,
+ "theoretical_loss": 4.348957437021351,
+ "tokens_seen": 213319680
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558355065195587,
+ "loss": 1.7637,
+ "theoretical_loss": 4.348793031006235,
+ "tokens_seen": 213385216
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558194583751254,
+ "loss": 1.636,
+ "theoretical_loss": 4.348628689609775,
+ "tokens_seen": 213450752
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007558034102306921,
+ "loss": 1.6531,
+ "theoretical_loss": 4.348464412786746,
+ "tokens_seen": 213516288
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557873620862588,
+ "loss": 1.7114,
+ "theoretical_loss": 4.348300200491966,
+ "tokens_seen": 213581824
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557713139418255,
+ "loss": 1.6954,
+ "theoretical_loss": 4.348136052680301,
+ "tokens_seen": 213647360
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557552657973922,
+ "loss": 1.7568,
+ "theoretical_loss": 4.347971969306662,
+ "tokens_seen": 213712896
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557392176529589,
+ "loss": 1.5466,
+ "theoretical_loss": 4.347807950326002,
+ "tokens_seen": 213778432
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557231695085256,
+ "loss": 1.7092,
+ "theoretical_loss": 4.347643995693325,
+ "tokens_seen": 213843968
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007557071213640923,
+ "loss": 1.6906,
+ "theoretical_loss": 4.347480105363674,
+ "tokens_seen": 213909504
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.000755691073219659,
+ "loss": 1.767,
+ "theoretical_loss": 4.3473162792921425,
+ "tokens_seen": 213975040
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007556750250752257,
+ "loss": 1.5911,
+ "theoretical_loss": 4.347152517433866,
+ "tokens_seen": 214040576
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007556589769307924,
+ "loss": 1.5156,
+ "theoretical_loss": 4.346988819744026,
+ "tokens_seen": 214106112
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007556429287863591,
+ "loss": 1.6893,
+ "theoretical_loss": 4.346825186177848,
+ "tokens_seen": 214171648
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007556268806419258,
+ "loss": 1.6864,
+ "theoretical_loss": 4.346661616690605,
+ "tokens_seen": 214237184
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007556108324974925,
+ "loss": 1.7354,
+ "theoretical_loss": 4.346498111237612,
+ "tokens_seen": 214302720
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007555947843530592,
+ "loss": 1.6528,
+ "theoretical_loss": 4.346334669774231,
+ "tokens_seen": 214368256
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007555787362086259,
+ "loss": 1.5685,
+ "theoretical_loss": 4.346171292255867,
+ "tokens_seen": 214433792
+ },
+ {
+ "epoch": 0.06,
+ "learning_rate": 0.0007555626880641926,
+ "loss": 1.6147,
+ "theoretical_loss": 4.34600797863797,
+ "tokens_seen": 214499328
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007555466399197594,
+ "loss": 1.6934,
+ "theoretical_loss": 4.345844728876036,
+ "tokens_seen": 214564864
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000755530591775326,
+ "loss": 1.7624,
+ "theoretical_loss": 4.345681542925604,
+ "tokens_seen": 214630400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007555145436308928,
+ "loss": 1.5814,
+ "theoretical_loss": 4.345518420742259,
+ "tokens_seen": 214695936
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554984954864594,
+ "loss": 1.6765,
+ "theoretical_loss": 4.345355362281628,
+ "tokens_seen": 214761472
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554824473420262,
+ "loss": 1.6003,
+ "theoretical_loss": 4.345192367499386,
+ "tokens_seen": 214827008
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554663991975928,
+ "loss": 1.7416,
+ "theoretical_loss": 4.3450294363512505,
+ "tokens_seen": 214892544
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554503510531596,
+ "loss": 1.7229,
+ "theoretical_loss": 4.34486656879298,
+ "tokens_seen": 214958080
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554343029087262,
+ "loss": 1.557,
+ "theoretical_loss": 4.344703764780386,
+ "tokens_seen": 215023616
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000755418254764293,
+ "loss": 1.7031,
+ "theoretical_loss": 4.344541024269312,
+ "tokens_seen": 215089152
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007554022066198596,
+ "loss": 1.6627,
+ "theoretical_loss": 4.344378347215657,
+ "tokens_seen": 215154688
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007553861584754264,
+ "loss": 1.6637,
+ "theoretical_loss": 4.344215733575357,
+ "tokens_seen": 215220224
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000755370110330993,
+ "loss": 1.5809,
+ "theoretical_loss": 4.344053183304396,
+ "tokens_seen": 215285760
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007553540621865598,
+ "loss": 1.7869,
+ "theoretical_loss": 4.343890696358798,
+ "tokens_seen": 215351296
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007553380140421264,
+ "loss": 1.4857,
+ "theoretical_loss": 4.343728272694635,
+ "tokens_seen": 215416832
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007553219658976932,
+ "loss": 1.4274,
+ "theoretical_loss": 4.34356591226802,
+ "tokens_seen": 215482368
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007553059177532598,
+ "loss": 1.7362,
+ "theoretical_loss": 4.343403615035111,
+ "tokens_seen": 215547904
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007552898696088266,
+ "loss": 1.6923,
+ "theoretical_loss": 4.343241380952109,
+ "tokens_seen": 215613440
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007552738214643932,
+ "loss": 1.7153,
+ "theoretical_loss": 4.34307920997526,
+ "tokens_seen": 215678976
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075525777331996,
+ "loss": 1.7254,
+ "theoretical_loss": 4.34291710206085,
+ "tokens_seen": 215744512
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007552417251755266,
+ "loss": 1.769,
+ "theoretical_loss": 4.342755057165214,
+ "tokens_seen": 215810048
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007552256770310934,
+ "loss": 1.6395,
+ "theoretical_loss": 4.342593075244728,
+ "tokens_seen": 215875584
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075520962888666,
+ "loss": 1.6217,
+ "theoretical_loss": 4.342431156255809,
+ "tokens_seen": 215941120
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551935807422268,
+ "loss": 1.5778,
+ "theoretical_loss": 4.3422693001549195,
+ "tokens_seen": 216006656
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551775325977934,
+ "loss": 1.654,
+ "theoretical_loss": 4.342107506898567,
+ "tokens_seen": 216072192
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551614844533601,
+ "loss": 1.6215,
+ "theoretical_loss": 4.3419457764433,
+ "tokens_seen": 216137728
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551454363089268,
+ "loss": 1.5418,
+ "theoretical_loss": 4.34178410874571,
+ "tokens_seen": 216203264
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 78526,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7171001434326172,
+ "objective/train/theoretical_loss": 4.341622503762434,
+ "objective/train/tokens_used": 236728800,
+ "theoretical_loss": 4.341622503762434,
+ "tokens_seen": 216268800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551293881644935,
+ "loss": 1.6569,
+ "theoretical_loss": 4.341622503762434,
+ "tokens_seen": 216268800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007551133400200602,
+ "loss": 1.6461,
+ "theoretical_loss": 4.341460961450148,
+ "tokens_seen": 216334336
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550972918756269,
+ "loss": 1.5453,
+ "theoretical_loss": 4.341299481765575,
+ "tokens_seen": 216399872
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550812437311937,
+ "loss": 1.6849,
+ "theoretical_loss": 4.34113806466548,
+ "tokens_seen": 216465408
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550651955867603,
+ "loss": 1.7779,
+ "theoretical_loss": 4.340976710106671,
+ "tokens_seen": 216530944
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000755049147442327,
+ "loss": 1.6228,
+ "theoretical_loss": 4.340815418045995,
+ "tokens_seen": 216596480
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550330992978937,
+ "loss": 1.7109,
+ "theoretical_loss": 4.340654188440349,
+ "tokens_seen": 216662016
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550170511534603,
+ "loss": 1.6088,
+ "theoretical_loss": 4.340493021246668,
+ "tokens_seen": 216727552
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007550010030090271,
+ "loss": 1.63,
+ "theoretical_loss": 4.3403319164219285,
+ "tokens_seen": 216793088
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549849548645937,
+ "loss": 1.7283,
+ "theoretical_loss": 4.340170873923155,
+ "tokens_seen": 216858624
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549689067201605,
+ "loss": 1.6471,
+ "theoretical_loss": 4.340009893707408,
+ "tokens_seen": 216924160
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549528585757271,
+ "loss": 1.6254,
+ "theoretical_loss": 4.339848975731796,
+ "tokens_seen": 216989696
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549368104312939,
+ "loss": 1.6024,
+ "theoretical_loss": 4.3396881199534665,
+ "tokens_seen": 217055232
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549207622868605,
+ "loss": 1.6612,
+ "theoretical_loss": 4.339527326329613,
+ "tokens_seen": 217120768
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007549047141424273,
+ "loss": 1.6202,
+ "theoretical_loss": 4.3393665948174664,
+ "tokens_seen": 217186304
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000754888665997994,
+ "loss": 1.6312,
+ "theoretical_loss": 4.339205925374305,
+ "tokens_seen": 217251840
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007548726178535607,
+ "loss": 1.5936,
+ "theoretical_loss": 4.339045317957446,
+ "tokens_seen": 217317376
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007548565697091274,
+ "loss": 1.6884,
+ "theoretical_loss": 4.33888477252425,
+ "tokens_seen": 217382912
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007548405215646941,
+ "loss": 1.6815,
+ "theoretical_loss": 4.33872428903212,
+ "tokens_seen": 217448448
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007548244734202608,
+ "loss": 1.6695,
+ "theoretical_loss": 4.338563867438499,
+ "tokens_seen": 217513984
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007548084252758275,
+ "loss": 1.6838,
+ "theoretical_loss": 4.338403507700877,
+ "tokens_seen": 217579520
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007547923771313942,
+ "loss": 1.4434,
+ "theoretical_loss": 4.33824320977678,
+ "tokens_seen": 217645056
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007547763289869609,
+ "loss": 1.6037,
+ "theoretical_loss": 4.33808297362378,
+ "tokens_seen": 217710592
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007547602808425276,
+ "loss": 1.6591,
+ "theoretical_loss": 4.337922799199489,
+ "tokens_seen": 217776128
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007547442326980943,
+ "loss": 1.6701,
+ "theoretical_loss": 4.337762686461561,
+ "tokens_seen": 217841664
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000754728184553661,
+ "loss": 1.6801,
+ "theoretical_loss": 4.337602635367694,
+ "tokens_seen": 217907200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007547121364092277,
+ "loss": 1.642,
+ "theoretical_loss": 4.337442645875624,
+ "tokens_seen": 217972736
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546960882647944,
+ "loss": 1.695,
+ "theoretical_loss": 4.337282717943132,
+ "tokens_seen": 218038272
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546800401203611,
+ "loss": 1.7297,
+ "theoretical_loss": 4.337122851528037,
+ "tokens_seen": 218103808
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546639919759278,
+ "loss": 1.5796,
+ "theoretical_loss": 4.336963046588204,
+ "tokens_seen": 218169344
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546479438314945,
+ "loss": 1.6636,
+ "theoretical_loss": 4.336803303081538,
+ "tokens_seen": 218234880
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546318956870612,
+ "loss": 1.608,
+ "theoretical_loss": 4.336643620965982,
+ "tokens_seen": 218300416
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007546158475426279,
+ "loss": 1.6229,
+ "theoretical_loss": 4.336484000199524,
+ "tokens_seen": 218365952
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545997993981946,
+ "loss": 1.5969,
+ "theoretical_loss": 4.336324440740195,
+ "tokens_seen": 218431488
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545837512537613,
+ "loss": 1.6826,
+ "theoretical_loss": 4.336164942546061,
+ "tokens_seen": 218497024
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000754567703109328,
+ "loss": 1.5707,
+ "theoretical_loss": 4.336005505575235,
+ "tokens_seen": 218562560
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545516549648947,
+ "loss": 1.7505,
+ "theoretical_loss": 4.335846129785869,
+ "tokens_seen": 218628096
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545356068204614,
+ "loss": 1.554,
+ "theoretical_loss": 4.335686815136157,
+ "tokens_seen": 218693632
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545195586760281,
+ "loss": 1.6179,
+ "theoretical_loss": 4.335527561584333,
+ "tokens_seen": 218759168
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007545035105315948,
+ "loss": 1.6906,
+ "theoretical_loss": 4.3353683690886715,
+ "tokens_seen": 218824704
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007544874623871615,
+ "loss": 1.7096,
+ "theoretical_loss": 4.33520923760749,
+ "tokens_seen": 218890240
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007544714142427282,
+ "loss": 1.6438,
+ "theoretical_loss": 4.335050167099145,
+ "tokens_seen": 218955776
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007544553660982949,
+ "loss": 1.7231,
+ "theoretical_loss": 4.334891157522035,
+ "tokens_seen": 219021312
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007544393179538616,
+ "loss": 1.6563,
+ "theoretical_loss": 4.3347322088346,
+ "tokens_seen": 219086848
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007544232698094283,
+ "loss": 1.5714,
+ "theoretical_loss": 4.33457332099532,
+ "tokens_seen": 219152384
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000754407221664995,
+ "loss": 1.5984,
+ "theoretical_loss": 4.334414493962714,
+ "tokens_seen": 219217920
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543911735205617,
+ "loss": 1.6528,
+ "theoretical_loss": 4.334255727695344,
+ "tokens_seen": 219283456
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543751253761284,
+ "loss": 1.6906,
+ "theoretical_loss": 4.334097022151812,
+ "tokens_seen": 219348992
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543590772316951,
+ "loss": 1.71,
+ "theoretical_loss": 4.33393837729076,
+ "tokens_seen": 219414528
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543430290872619,
+ "loss": 1.7512,
+ "theoretical_loss": 4.333779793070871,
+ "tokens_seen": 219480064
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 79445,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6968114376068115,
+ "objective/train/theoretical_loss": 4.33362126945087,
+ "objective/train/tokens_used": 240005600,
+ "theoretical_loss": 4.33362126945087,
+ "tokens_seen": 219545600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543269809428285,
+ "loss": 1.7406,
+ "theoretical_loss": 4.33362126945087,
+ "tokens_seen": 219545600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007543109327983953,
+ "loss": 1.6159,
+ "theoretical_loss": 4.333462806389517,
+ "tokens_seen": 219611136
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542948846539619,
+ "loss": 1.6682,
+ "theoretical_loss": 4.33330440384562,
+ "tokens_seen": 219676672
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542788365095287,
+ "loss": 1.6814,
+ "theoretical_loss": 4.333146061778021,
+ "tokens_seen": 219742208
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542627883650953,
+ "loss": 1.6632,
+ "theoretical_loss": 4.332987780145606,
+ "tokens_seen": 219807744
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542467402206621,
+ "loss": 1.5992,
+ "theoretical_loss": 4.3328295589073,
+ "tokens_seen": 219873280
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542306920762287,
+ "loss": 1.6456,
+ "theoretical_loss": 4.332671398022067,
+ "tokens_seen": 219938816
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007542146439317955,
+ "loss": 1.632,
+ "theoretical_loss": 4.332513297448912,
+ "tokens_seen": 220004352
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541985957873621,
+ "loss": 1.7385,
+ "theoretical_loss": 4.332355257146881,
+ "tokens_seen": 220069888
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541825476429289,
+ "loss": 1.5406,
+ "theoretical_loss": 4.3321972770750605,
+ "tokens_seen": 220135424
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541664994984955,
+ "loss": 1.6117,
+ "theoretical_loss": 4.332039357192572,
+ "tokens_seen": 220200960
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541504513540623,
+ "loss": 1.6715,
+ "theoretical_loss": 4.331881497458584,
+ "tokens_seen": 220266496
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541344032096289,
+ "loss": 1.6171,
+ "theoretical_loss": 4.3317236978323,
+ "tokens_seen": 220332032
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541183550651957,
+ "loss": 1.6819,
+ "theoretical_loss": 4.3315659582729635,
+ "tokens_seen": 220397568
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007541023069207623,
+ "loss": 1.5839,
+ "theoretical_loss": 4.331408278739861,
+ "tokens_seen": 220463104
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540862587763291,
+ "loss": 1.6511,
+ "theoretical_loss": 4.331250659192316,
+ "tokens_seen": 220528640
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540702106318957,
+ "loss": 1.7196,
+ "theoretical_loss": 4.33109309958969,
+ "tokens_seen": 220594176
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540541624874625,
+ "loss": 1.6633,
+ "theoretical_loss": 4.330935599891389,
+ "tokens_seen": 220659712
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540381143430291,
+ "loss": 1.6971,
+ "theoretical_loss": 4.330778160056855,
+ "tokens_seen": 220725248
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540220661985959,
+ "loss": 1.7276,
+ "theoretical_loss": 4.330620780045571,
+ "tokens_seen": 220790784
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007540060180541625,
+ "loss": 1.7717,
+ "theoretical_loss": 4.330463459817057,
+ "tokens_seen": 220856320
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539899699097293,
+ "loss": 1.6917,
+ "theoretical_loss": 4.3303061993308765,
+ "tokens_seen": 220921856
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539739217652959,
+ "loss": 1.6614,
+ "theoretical_loss": 4.330148998546628,
+ "tokens_seen": 220987392
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539578736208627,
+ "loss": 1.6307,
+ "theoretical_loss": 4.329991857423953,
+ "tokens_seen": 221052928
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539418254764293,
+ "loss": 1.5909,
+ "theoretical_loss": 4.329834775922529,
+ "tokens_seen": 221118464
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539257773319961,
+ "loss": 1.7151,
+ "theoretical_loss": 4.329677754002074,
+ "tokens_seen": 221184000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007539097291875627,
+ "loss": 1.6996,
+ "theoretical_loss": 4.329520791622348,
+ "tokens_seen": 221249536
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007538936810431295,
+ "loss": 1.6523,
+ "theoretical_loss": 4.329363888743145,
+ "tokens_seen": 221315072
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007538776328986961,
+ "loss": 1.6329,
+ "theoretical_loss": 4.329207045324301,
+ "tokens_seen": 221380608
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007538615847542629,
+ "loss": 1.6521,
+ "theoretical_loss": 4.3290502613256905,
+ "tokens_seen": 221446144
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007538455366098295,
+ "loss": 1.8236,
+ "theoretical_loss": 4.328893536707228,
+ "tokens_seen": 221511680
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007538294884653963,
+ "loss": 1.7811,
+ "theoretical_loss": 4.328736871428864,
+ "tokens_seen": 221577216
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753813440320963,
+ "loss": 1.6121,
+ "theoretical_loss": 4.328580265450592,
+ "tokens_seen": 221642752
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537973921765297,
+ "loss": 1.6411,
+ "theoretical_loss": 4.32842371873244,
+ "tokens_seen": 221708288
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537813440320963,
+ "loss": 1.7128,
+ "theoretical_loss": 4.328267231234476,
+ "tokens_seen": 221773824
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753765295887663,
+ "loss": 1.7062,
+ "theoretical_loss": 4.328110802916811,
+ "tokens_seen": 221839360
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537492477432297,
+ "loss": 1.8148,
+ "theoretical_loss": 4.327954433739588,
+ "tokens_seen": 221904896
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537331995987964,
+ "loss": 1.6588,
+ "theoretical_loss": 4.327798123662992,
+ "tokens_seen": 221970432
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537171514543632,
+ "loss": 1.5768,
+ "theoretical_loss": 4.327641872647248,
+ "tokens_seen": 222035968
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007537011033099298,
+ "loss": 1.7152,
+ "theoretical_loss": 4.327485680652615,
+ "tokens_seen": 222101504
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007536850551654966,
+ "loss": 1.6536,
+ "theoretical_loss": 4.327329547639396,
+ "tokens_seen": 222167040
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007536690070210632,
+ "loss": 1.6875,
+ "theoretical_loss": 4.327173473567927,
+ "tokens_seen": 222232576
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075365295887663,
+ "loss": 1.7102,
+ "theoretical_loss": 4.327017458398587,
+ "tokens_seen": 222298112
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007536369107321966,
+ "loss": 1.6412,
+ "theoretical_loss": 4.32686150209179,
+ "tokens_seen": 222363648
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007536208625877632,
+ "loss": 1.6459,
+ "theoretical_loss": 4.32670560460799,
+ "tokens_seen": 222429184
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075360481444333,
+ "loss": 1.5749,
+ "theoretical_loss": 4.326549765907679,
+ "tokens_seen": 222494720
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007535887662988966,
+ "loss": 1.7364,
+ "theoretical_loss": 4.326393985951386,
+ "tokens_seen": 222560256
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007535727181544634,
+ "loss": 1.5982,
+ "theoretical_loss": 4.326238264699679,
+ "tokens_seen": 222625792
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075355667001003,
+ "loss": 1.7705,
+ "theoretical_loss": 4.326082602113164,
+ "tokens_seen": 222691328
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007535406218655968,
+ "loss": 1.6681,
+ "theoretical_loss": 4.325926998152487,
+ "tokens_seen": 222756864
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 80592,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5793734788894653,
+ "objective/train/theoretical_loss": 4.325771452778328,
+ "objective/train/tokens_used": 243282400,
+ "theoretical_loss": 4.325771452778328,
+ "tokens_seen": 222822400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007535245737211634,
+ "loss": 1.6079,
+ "theoretical_loss": 4.325771452778328,
+ "tokens_seen": 222822400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007535085255767302,
+ "loss": 1.731,
+ "theoretical_loss": 4.325615965951408,
+ "tokens_seen": 222887936
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007534924774322969,
+ "loss": 1.6734,
+ "theoretical_loss": 4.325460537632483,
+ "tokens_seen": 222953472
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007534764292878636,
+ "loss": 1.6681,
+ "theoretical_loss": 4.325305167782352,
+ "tokens_seen": 223019008
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007534603811434303,
+ "loss": 1.6635,
+ "theoretical_loss": 4.325149856361845,
+ "tokens_seen": 223084544
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753444332998997,
+ "loss": 1.6284,
+ "theoretical_loss": 4.3249946033318345,
+ "tokens_seen": 223150080
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007534282848545637,
+ "loss": 1.7733,
+ "theoretical_loss": 4.32483940865323,
+ "tokens_seen": 223215616
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007534122367101304,
+ "loss": 1.6484,
+ "theoretical_loss": 4.324684272286978,
+ "tokens_seen": 223281152
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533961885656971,
+ "loss": 1.6395,
+ "theoretical_loss": 4.324529194194062,
+ "tokens_seen": 223346688
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533801404212638,
+ "loss": 1.6507,
+ "theoretical_loss": 4.324374174335503,
+ "tokens_seen": 223412224
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533640922768305,
+ "loss": 1.7281,
+ "theoretical_loss": 4.324219212672363,
+ "tokens_seen": 223477760
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533480441323972,
+ "loss": 1.7564,
+ "theoretical_loss": 4.324064309165734,
+ "tokens_seen": 223543296
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533319959879639,
+ "loss": 1.5957,
+ "theoretical_loss": 4.3239094637767534,
+ "tokens_seen": 223608832
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007533159478435306,
+ "loss": 1.7318,
+ "theoretical_loss": 4.323754676466592,
+ "tokens_seen": 223674368
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532998996990973,
+ "loss": 1.7811,
+ "theoretical_loss": 4.323599947196458,
+ "tokens_seen": 223739904
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753283851554664,
+ "loss": 1.7292,
+ "theoretical_loss": 4.323445275927597,
+ "tokens_seen": 223805440
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532678034102307,
+ "loss": 1.712,
+ "theoretical_loss": 4.323290662621292,
+ "tokens_seen": 223870976
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532517552657974,
+ "loss": 1.7663,
+ "theoretical_loss": 4.323136107238865,
+ "tokens_seen": 223936512
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532357071213641,
+ "loss": 1.7012,
+ "theoretical_loss": 4.322981609741671,
+ "tokens_seen": 224002048
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532196589769308,
+ "loss": 1.6102,
+ "theoretical_loss": 4.322827170091106,
+ "tokens_seen": 224067584
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007532036108324975,
+ "loss": 1.6167,
+ "theoretical_loss": 4.322672788248601,
+ "tokens_seen": 224133120
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007531875626880642,
+ "loss": 1.5478,
+ "theoretical_loss": 4.3225184641756265,
+ "tokens_seen": 224198656
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007531715145436309,
+ "loss": 1.6447,
+ "theoretical_loss": 4.322364197833684,
+ "tokens_seen": 224264192
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007531554663991976,
+ "loss": 1.618,
+ "theoretical_loss": 4.322209989184319,
+ "tokens_seen": 224329728
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007531394182547644,
+ "loss": 1.6174,
+ "theoretical_loss": 4.32205583818911,
+ "tokens_seen": 224395264
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753123370110331,
+ "loss": 1.6718,
+ "theoretical_loss": 4.321901744809672,
+ "tokens_seen": 224460800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007531073219658978,
+ "loss": 1.6626,
+ "theoretical_loss": 4.321747709007658,
+ "tokens_seen": 224526336
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007530912738214644,
+ "loss": 1.6943,
+ "theoretical_loss": 4.321593730744759,
+ "tokens_seen": 224591872
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007530752256770312,
+ "loss": 1.7384,
+ "theoretical_loss": 4.3214398099826985,
+ "tokens_seen": 224657408
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007530591775325978,
+ "loss": 1.6295,
+ "theoretical_loss": 4.321285946683242,
+ "tokens_seen": 224722944
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007530431293881646,
+ "loss": 1.6245,
+ "theoretical_loss": 4.321132140808186,
+ "tokens_seen": 224788480
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007530270812437312,
+ "loss": 1.6358,
+ "theoretical_loss": 4.320978392319368,
+ "tokens_seen": 224854016
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000753011033099298,
+ "loss": 1.6467,
+ "theoretical_loss": 4.320824701178661,
+ "tokens_seen": 224919552
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007529949849548646,
+ "loss": 1.6154,
+ "theoretical_loss": 4.320671067347972,
+ "tokens_seen": 224985088
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007529789368104314,
+ "loss": 1.7091,
+ "theoretical_loss": 4.320517490789246,
+ "tokens_seen": 225050624
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752962888665998,
+ "loss": 1.5441,
+ "theoretical_loss": 4.320363971464466,
+ "tokens_seen": 225116160
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007529468405215648,
+ "loss": 1.913,
+ "theoretical_loss": 4.320210509335649,
+ "tokens_seen": 225181696
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007529307923771314,
+ "loss": 1.7173,
+ "theoretical_loss": 4.320057104364848,
+ "tokens_seen": 225247232
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007529147442326982,
+ "loss": 1.5435,
+ "theoretical_loss": 4.3199037565141545,
+ "tokens_seen": 225312768
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007528986960882648,
+ "loss": 1.5532,
+ "theoretical_loss": 4.319750465745694,
+ "tokens_seen": 225378304
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007528826479438316,
+ "loss": 1.5809,
+ "theoretical_loss": 4.31959723202163,
+ "tokens_seen": 225443840
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007528665997993982,
+ "loss": 1.6615,
+ "theoretical_loss": 4.319444055304161,
+ "tokens_seen": 225509376
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752850551654965,
+ "loss": 1.6242,
+ "theoretical_loss": 4.319290935555521,
+ "tokens_seen": 225574912
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007528345035105316,
+ "loss": 1.6218,
+ "theoretical_loss": 4.319137872737979,
+ "tokens_seen": 225640448
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007528184553660984,
+ "loss": 1.7047,
+ "theoretical_loss": 4.318984866813844,
+ "tokens_seen": 225705984
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752802407221665,
+ "loss": 1.678,
+ "theoretical_loss": 4.318831917745458,
+ "tokens_seen": 225771520
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527863590772318,
+ "loss": 1.6995,
+ "theoretical_loss": 4.318679025495197,
+ "tokens_seen": 225837056
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527703109327984,
+ "loss": 1.6975,
+ "theoretical_loss": 4.318526190025478,
+ "tokens_seen": 225902592
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527542627883652,
+ "loss": 1.7384,
+ "theoretical_loss": 4.318373411298749,
+ "tokens_seen": 225968128
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527382146439318,
+ "loss": 1.6878,
+ "theoretical_loss": 4.318220689277496,
+ "tokens_seen": 226033664
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 81462,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.65253746509552,
+ "objective/train/theoretical_loss": 4.318068023924241,
+ "objective/train/tokens_used": 246559200,
+ "theoretical_loss": 4.318068023924241,
+ "tokens_seen": 226099200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527221664994986,
+ "loss": 1.7266,
+ "theoretical_loss": 4.318068023924241,
+ "tokens_seen": 226099200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007527061183550652,
+ "loss": 1.536,
+ "theoretical_loss": 4.317915415201538,
+ "tokens_seen": 226164736
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752690070210632,
+ "loss": 1.512,
+ "theoretical_loss": 4.317762863071982,
+ "tokens_seen": 226230272
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007526740220661986,
+ "loss": 1.5732,
+ "theoretical_loss": 4.317610367498201,
+ "tokens_seen": 226295808
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007526579739217654,
+ "loss": 1.7787,
+ "theoretical_loss": 4.317457928442856,
+ "tokens_seen": 226361344
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752641925777332,
+ "loss": 1.6781,
+ "theoretical_loss": 4.317305545868648,
+ "tokens_seen": 226426880
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007526258776328988,
+ "loss": 1.7043,
+ "theoretical_loss": 4.317153219738311,
+ "tokens_seen": 226492416
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007526098294884654,
+ "loss": 1.6454,
+ "theoretical_loss": 4.317000950014615,
+ "tokens_seen": 226557952
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007525937813440322,
+ "loss": 1.5971,
+ "theoretical_loss": 4.316848736660363,
+ "tokens_seen": 226623488
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007525777331995988,
+ "loss": 1.6237,
+ "theoretical_loss": 4.316696579638396,
+ "tokens_seen": 226689024
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007525616850551656,
+ "loss": 1.6166,
+ "theoretical_loss": 4.316544478911592,
+ "tokens_seen": 226754560
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007525456369107322,
+ "loss": 1.7207,
+ "theoretical_loss": 4.316392434442858,
+ "tokens_seen": 226820096
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752529588766299,
+ "loss": 1.6948,
+ "theoretical_loss": 4.316240446195142,
+ "tokens_seen": 226885632
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007525135406218656,
+ "loss": 1.6371,
+ "theoretical_loss": 4.3160885141314225,
+ "tokens_seen": 226951168
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524974924774324,
+ "loss": 1.6582,
+ "theoretical_loss": 4.315936638214718,
+ "tokens_seen": 227016704
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752481444332999,
+ "loss": 1.6293,
+ "theoretical_loss": 4.315784818408079,
+ "tokens_seen": 227082240
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524653961885658,
+ "loss": 1.6876,
+ "theoretical_loss": 4.31563305467459,
+ "tokens_seen": 227147776
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524493480441324,
+ "loss": 1.6117,
+ "theoretical_loss": 4.315481346977374,
+ "tokens_seen": 227213312
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524332998996992,
+ "loss": 1.5356,
+ "theoretical_loss": 4.315329695279584,
+ "tokens_seen": 227278848
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524172517552658,
+ "loss": 1.7156,
+ "theoretical_loss": 4.315178099544413,
+ "tokens_seen": 227344384
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007524012036108326,
+ "loss": 1.5758,
+ "theoretical_loss": 4.315026559735085,
+ "tokens_seen": 227409920
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007523851554663992,
+ "loss": 1.7364,
+ "theoretical_loss": 4.314875075814861,
+ "tokens_seen": 227475456
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007523691073219659,
+ "loss": 1.716,
+ "theoretical_loss": 4.314723647747035,
+ "tokens_seen": 227540992
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007523530591775327,
+ "loss": 1.5991,
+ "theoretical_loss": 4.314572275494937,
+ "tokens_seen": 227606528
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007523370110330993,
+ "loss": 1.4546,
+ "theoretical_loss": 4.3144209590219305,
+ "tokens_seen": 227672064
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752320962888666,
+ "loss": 1.6459,
+ "theoretical_loss": 4.314269698291415,
+ "tokens_seen": 227737600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007523049147442327,
+ "loss": 1.7276,
+ "theoretical_loss": 4.3141184932668235,
+ "tokens_seen": 227803136
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522888665997995,
+ "loss": 1.6812,
+ "theoretical_loss": 4.3139673439116235,
+ "tokens_seen": 227868672
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522728184553661,
+ "loss": 1.7196,
+ "theoretical_loss": 4.313816250189317,
+ "tokens_seen": 227934208
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522567703109329,
+ "loss": 1.6224,
+ "theoretical_loss": 4.313665212063441,
+ "tokens_seen": 227999744
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522407221664995,
+ "loss": 1.6276,
+ "theoretical_loss": 4.313514229497566,
+ "tokens_seen": 228065280
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522246740220661,
+ "loss": 1.5987,
+ "theoretical_loss": 4.313363302455299,
+ "tokens_seen": 228130816
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007522086258776329,
+ "loss": 1.7328,
+ "theoretical_loss": 4.313212430900277,
+ "tokens_seen": 228196352
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007521925777331995,
+ "loss": 1.5894,
+ "theoretical_loss": 4.313061614796176,
+ "tokens_seen": 228261888
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007521765295887663,
+ "loss": 1.766,
+ "theoretical_loss": 4.312910854106702,
+ "tokens_seen": 228327424
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752160481444333,
+ "loss": 1.6051,
+ "theoretical_loss": 4.3127601487956,
+ "tokens_seen": 228392960
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007521444332998997,
+ "loss": 1.7288,
+ "theoretical_loss": 4.312609498826644,
+ "tokens_seen": 228458496
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007521283851554664,
+ "loss": 1.5024,
+ "theoretical_loss": 4.312458904163645,
+ "tokens_seen": 228524032
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007521123370110331,
+ "loss": 1.7327,
+ "theoretical_loss": 4.312308364770448,
+ "tokens_seen": 228589568
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520962888665998,
+ "loss": 1.5368,
+ "theoretical_loss": 4.312157880610931,
+ "tokens_seen": 228655104
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520802407221665,
+ "loss": 1.5935,
+ "theoretical_loss": 4.312007451649006,
+ "tokens_seen": 228720640
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520641925777332,
+ "loss": 1.6157,
+ "theoretical_loss": 4.31185707784862,
+ "tokens_seen": 228786176
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520481444332999,
+ "loss": 1.5835,
+ "theoretical_loss": 4.311706759173753,
+ "tokens_seen": 228851712
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520320962888666,
+ "loss": 1.5946,
+ "theoretical_loss": 4.311556495588419,
+ "tokens_seen": 228917248
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007520160481444333,
+ "loss": 1.6795,
+ "theoretical_loss": 4.3114062870566645,
+ "tokens_seen": 228982784
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000752,
+ "loss": 1.5991,
+ "theoretical_loss": 4.311256133542573,
+ "tokens_seen": 229048320
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519839518555667,
+ "loss": 1.6723,
+ "theoretical_loss": 4.311106035010259,
+ "tokens_seen": 229113856
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519679037111334,
+ "loss": 1.5913,
+ "theoretical_loss": 4.310955991423871,
+ "tokens_seen": 229179392
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519518555667001,
+ "loss": 1.6719,
+ "theoretical_loss": 4.310806002747592,
+ "tokens_seen": 229244928
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519358074222669,
+ "loss": 1.5921,
+ "theoretical_loss": 4.310656068945637,
+ "tokens_seen": 229310464
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 82681,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5853383541107178,
+ "objective/train/theoretical_loss": 4.310506189982257,
+ "objective/train/tokens_used": 249836000,
+ "theoretical_loss": 4.310506189982257,
+ "tokens_seen": 229376000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519197592778335,
+ "loss": 1.601,
+ "theoretical_loss": 4.310506189982257,
+ "tokens_seen": 229376000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007519037111334003,
+ "loss": 1.7332,
+ "theoretical_loss": 4.310356365821734,
+ "tokens_seen": 229441536
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518876629889669,
+ "loss": 1.7276,
+ "theoretical_loss": 4.310206596428387,
+ "tokens_seen": 229507072
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518716148445337,
+ "loss": 1.6907,
+ "theoretical_loss": 4.310056881766561,
+ "tokens_seen": 229572608
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518555667001003,
+ "loss": 1.6095,
+ "theoretical_loss": 4.309907221800645,
+ "tokens_seen": 229638144
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518395185556671,
+ "loss": 1.7901,
+ "theoretical_loss": 4.309757616495053,
+ "tokens_seen": 229703680
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518234704112337,
+ "loss": 1.6816,
+ "theoretical_loss": 4.309608065814234,
+ "tokens_seen": 229769216
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007518074222668005,
+ "loss": 1.5572,
+ "theoretical_loss": 4.309458569722673,
+ "tokens_seen": 229834752
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517913741223671,
+ "loss": 1.7167,
+ "theoretical_loss": 4.309309128184886,
+ "tokens_seen": 229900288
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517753259779339,
+ "loss": 1.6913,
+ "theoretical_loss": 4.309159741165422,
+ "tokens_seen": 229965824
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517592778335005,
+ "loss": 1.6231,
+ "theoretical_loss": 4.309010408628865,
+ "tokens_seen": 230031360
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517432296890673,
+ "loss": 1.5985,
+ "theoretical_loss": 4.308861130539829,
+ "tokens_seen": 230096896
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517271815446339,
+ "loss": 1.6563,
+ "theoretical_loss": 4.308711906862965,
+ "tokens_seen": 230162432
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007517111334002007,
+ "loss": 1.6262,
+ "theoretical_loss": 4.308562737562953,
+ "tokens_seen": 230227968
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516950852557673,
+ "loss": 1.7089,
+ "theoretical_loss": 4.30841362260451,
+ "tokens_seen": 230293504
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516790371113341,
+ "loss": 1.612,
+ "theoretical_loss": 4.308264561952381,
+ "tokens_seen": 230359040
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516629889669007,
+ "loss": 1.6092,
+ "theoretical_loss": 4.30811555557135,
+ "tokens_seen": 230424576
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516469408224675,
+ "loss": 1.6265,
+ "theoretical_loss": 4.307966603426227,
+ "tokens_seen": 230490112
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516308926780341,
+ "loss": 1.7801,
+ "theoretical_loss": 4.307817705481861,
+ "tokens_seen": 230555648
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007516148445336009,
+ "loss": 1.7435,
+ "theoretical_loss": 4.307668861703131,
+ "tokens_seen": 230621184
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515987963891675,
+ "loss": 1.6081,
+ "theoretical_loss": 4.307520072054947,
+ "tokens_seen": 230686720
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515827482447343,
+ "loss": 1.5812,
+ "theoretical_loss": 4.307371336502254,
+ "tokens_seen": 230752256
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515667001003009,
+ "loss": 1.6561,
+ "theoretical_loss": 4.307222655010031,
+ "tokens_seen": 230817792
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515506519558677,
+ "loss": 1.6483,
+ "theoretical_loss": 4.307074027543287,
+ "tokens_seen": 230883328
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515346038114343,
+ "loss": 1.7229,
+ "theoretical_loss": 4.3069254540670645,
+ "tokens_seen": 230948864
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515185556670011,
+ "loss": 1.7828,
+ "theoretical_loss": 4.306776934546436,
+ "tokens_seen": 231014400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007515025075225677,
+ "loss": 1.5098,
+ "theoretical_loss": 4.306628468946512,
+ "tokens_seen": 231079936
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514864593781345,
+ "loss": 1.55,
+ "theoretical_loss": 4.306480057232431,
+ "tokens_seen": 231145472
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514704112337011,
+ "loss": 1.6725,
+ "theoretical_loss": 4.306331699369366,
+ "tokens_seen": 231211008
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514543630892679,
+ "loss": 1.6515,
+ "theoretical_loss": 4.30618339532252,
+ "tokens_seen": 231276544
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514383149448345,
+ "loss": 1.7387,
+ "theoretical_loss": 4.306035145057132,
+ "tokens_seen": 231342080
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514222668004013,
+ "loss": 1.6858,
+ "theoretical_loss": 4.30588694853847,
+ "tokens_seen": 231407616
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007514062186559679,
+ "loss": 1.6754,
+ "theoretical_loss": 4.305738805731836,
+ "tokens_seen": 231473152
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513901705115347,
+ "loss": 1.6655,
+ "theoretical_loss": 4.305590716602563,
+ "tokens_seen": 231538688
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513741223671013,
+ "loss": 1.6448,
+ "theoretical_loss": 4.305442681116018,
+ "tokens_seen": 231604224
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513580742226681,
+ "loss": 1.6165,
+ "theoretical_loss": 4.305294699237598,
+ "tokens_seen": 231669760
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513420260782347,
+ "loss": 1.5662,
+ "theoretical_loss": 4.305146770932734,
+ "tokens_seen": 231735296
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513259779338015,
+ "loss": 1.6911,
+ "theoretical_loss": 4.304998896166888,
+ "tokens_seen": 231800832
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007513099297893681,
+ "loss": 1.709,
+ "theoretical_loss": 4.304851074905554,
+ "tokens_seen": 231866368
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512938816449349,
+ "loss": 1.6598,
+ "theoretical_loss": 4.304703307114258,
+ "tokens_seen": 231931904
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512778335005015,
+ "loss": 1.6776,
+ "theoretical_loss": 4.304555592758559,
+ "tokens_seen": 231997440
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512617853560683,
+ "loss": 1.5082,
+ "theoretical_loss": 4.304407931804047,
+ "tokens_seen": 232062976
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512457372116349,
+ "loss": 1.6469,
+ "theoretical_loss": 4.304260324216344,
+ "tokens_seen": 232128512
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512296890672017,
+ "loss": 1.6334,
+ "theoretical_loss": 4.304112769961102,
+ "tokens_seen": 232194048
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007512136409227683,
+ "loss": 1.532,
+ "theoretical_loss": 4.303965269004008,
+ "tokens_seen": 232259584
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511975927783351,
+ "loss": 1.7242,
+ "theoretical_loss": 4.30381782131078,
+ "tokens_seen": 232325120
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511815446339017,
+ "loss": 1.6597,
+ "theoretical_loss": 4.303670426847166,
+ "tokens_seen": 232390656
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511654964894685,
+ "loss": 1.5863,
+ "theoretical_loss": 4.303523085578946,
+ "tokens_seen": 232456192
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511494483450351,
+ "loss": 1.6315,
+ "theoretical_loss": 4.303375797471935,
+ "tokens_seen": 232521728
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511334002006019,
+ "loss": 1.6744,
+ "theoretical_loss": 4.303228562491974,
+ "tokens_seen": 232587264
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 83903,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6346615552902222,
+ "objective/train/theoretical_loss": 4.303081380604939,
+ "objective/train/tokens_used": 253112800,
+ "theoretical_loss": 4.303081380604939,
+ "tokens_seen": 232652800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511173520561685,
+ "loss": 1.6698,
+ "theoretical_loss": 4.303081380604939,
+ "tokens_seen": 232652800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007511013039117353,
+ "loss": 1.5696,
+ "theoretical_loss": 4.302934251776739,
+ "tokens_seen": 232718336
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000751085255767302,
+ "loss": 1.7224,
+ "theoretical_loss": 4.302787175973311,
+ "tokens_seen": 232783872
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007510692076228687,
+ "loss": 1.7526,
+ "theoretical_loss": 4.302640153160625,
+ "tokens_seen": 232849408
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007510531594784353,
+ "loss": 1.6315,
+ "theoretical_loss": 4.302493183304683,
+ "tokens_seen": 232914944
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007510371113340021,
+ "loss": 1.6435,
+ "theoretical_loss": 4.302346266371517,
+ "tokens_seen": 232980480
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007510210631895687,
+ "loss": 1.6499,
+ "theoretical_loss": 4.302199402327191,
+ "tokens_seen": 233046016
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007510050150451355,
+ "loss": 1.6708,
+ "theoretical_loss": 4.302052591137802,
+ "tokens_seen": 233111552
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007509889669007021,
+ "loss": 1.6101,
+ "theoretical_loss": 4.301905832769474,
+ "tokens_seen": 233177088
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007509729187562688,
+ "loss": 1.6334,
+ "theoretical_loss": 4.301759127188367,
+ "tokens_seen": 233242624
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007509568706118356,
+ "loss": 1.7197,
+ "theoretical_loss": 4.301612474360669,
+ "tokens_seen": 233308160
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007509408224674022,
+ "loss": 1.7176,
+ "theoretical_loss": 4.301465874252601,
+ "tokens_seen": 233373696
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750924774322969,
+ "loss": 1.7187,
+ "theoretical_loss": 4.301319326830413,
+ "tokens_seen": 233439232
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007509087261785356,
+ "loss": 1.7268,
+ "theoretical_loss": 4.301172832060389,
+ "tokens_seen": 233504768
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007508926780341024,
+ "loss": 1.6295,
+ "theoretical_loss": 4.3010263899088415,
+ "tokens_seen": 233570304
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750876629889669,
+ "loss": 1.6408,
+ "theoretical_loss": 4.300880000342114,
+ "tokens_seen": 233635840
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007508605817452358,
+ "loss": 1.6239,
+ "theoretical_loss": 4.300733663326583,
+ "tokens_seen": 233701376
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007508445336008024,
+ "loss": 1.5342,
+ "theoretical_loss": 4.300587378828655,
+ "tokens_seen": 233766912
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750828485456369,
+ "loss": 1.7235,
+ "theoretical_loss": 4.300441146814766,
+ "tokens_seen": 233832448
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007508124373119358,
+ "loss": 1.619,
+ "theoretical_loss": 4.300294967251386,
+ "tokens_seen": 233897984
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507963891675024,
+ "loss": 1.6433,
+ "theoretical_loss": 4.300148840105011,
+ "tokens_seen": 233963520
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507803410230692,
+ "loss": 1.5639,
+ "theoretical_loss": 4.300002765342173,
+ "tokens_seen": 234029056
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507642928786359,
+ "loss": 1.672,
+ "theoretical_loss": 4.299856742929431,
+ "tokens_seen": 234094592
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507482447342026,
+ "loss": 1.6359,
+ "theoretical_loss": 4.299710772833377,
+ "tokens_seen": 234160128
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507321965897694,
+ "loss": 1.7174,
+ "theoretical_loss": 4.299564855020631,
+ "tokens_seen": 234225664
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750716148445336,
+ "loss": 1.5812,
+ "theoretical_loss": 4.299418989457848,
+ "tokens_seen": 234291200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007507001003009028,
+ "loss": 1.6371,
+ "theoretical_loss": 4.299273176111709,
+ "tokens_seen": 234356736
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007506840521564694,
+ "loss": 1.6428,
+ "theoretical_loss": 4.299127414948927,
+ "tokens_seen": 234422272
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007506680040120362,
+ "loss": 1.654,
+ "theoretical_loss": 4.298981705936248,
+ "tokens_seen": 234487808
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007506519558676028,
+ "loss": 1.7671,
+ "theoretical_loss": 4.298836049040444,
+ "tokens_seen": 234553344
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007506359077231696,
+ "loss": 1.7702,
+ "theoretical_loss": 4.298690444228322,
+ "tokens_seen": 234618880
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007506198595787362,
+ "loss": 1.5735,
+ "theoretical_loss": 4.298544891466717,
+ "tokens_seen": 234684416
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750603811434303,
+ "loss": 1.6077,
+ "theoretical_loss": 4.298399390722493,
+ "tokens_seen": 234749952
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007505877632898696,
+ "loss": 1.725,
+ "theoretical_loss": 4.298253941962548,
+ "tokens_seen": 234815488
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007505717151454364,
+ "loss": 1.5374,
+ "theoretical_loss": 4.2981085451538075,
+ "tokens_seen": 234881024
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750555667001003,
+ "loss": 1.6508,
+ "theoretical_loss": 4.297963200263228,
+ "tokens_seen": 234946560
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007505396188565698,
+ "loss": 1.7077,
+ "theoretical_loss": 4.297817907257797,
+ "tokens_seen": 235012096
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007505235707121364,
+ "loss": 1.6419,
+ "theoretical_loss": 4.29767266610453,
+ "tokens_seen": 235077632
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007505075225677032,
+ "loss": 1.6664,
+ "theoretical_loss": 4.297527476770476,
+ "tokens_seen": 235143168
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007504914744232698,
+ "loss": 1.6296,
+ "theoretical_loss": 4.297382339222711,
+ "tokens_seen": 235208704
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007504754262788366,
+ "loss": 1.693,
+ "theoretical_loss": 4.297237253428342,
+ "tokens_seen": 235274240
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007504593781344032,
+ "loss": 1.8014,
+ "theoretical_loss": 4.297092219354509,
+ "tokens_seen": 235339776
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075044332998997,
+ "loss": 1.7015,
+ "theoretical_loss": 4.296947236968376,
+ "tokens_seen": 235405312
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007504272818455366,
+ "loss": 1.6181,
+ "theoretical_loss": 4.296802306237143,
+ "tokens_seen": 235470848
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007504112337011034,
+ "loss": 1.7072,
+ "theoretical_loss": 4.296657427128036,
+ "tokens_seen": 235536384
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.00075039518555667,
+ "loss": 1.629,
+ "theoretical_loss": 4.296512599608311,
+ "tokens_seen": 235601920
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007503791374122368,
+ "loss": 1.6502,
+ "theoretical_loss": 4.296367823645259,
+ "tokens_seen": 235667456
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007503630892678034,
+ "loss": 1.671,
+ "theoretical_loss": 4.296223099206193,
+ "tokens_seen": 235732992
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007503470411233702,
+ "loss": 1.7477,
+ "theoretical_loss": 4.296078426258462,
+ "tokens_seen": 235798528
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007503309929789368,
+ "loss": 1.521,
+ "theoretical_loss": 4.295933804769442,
+ "tokens_seen": 235864064
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 84743,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6421600580215454,
+ "objective/train/theoretical_loss": 4.295789234706538,
+ "objective/train/tokens_used": 256389600,
+ "theoretical_loss": 4.295789234706538,
+ "tokens_seen": 235929600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007503149448345036,
+ "loss": 1.7047,
+ "theoretical_loss": 4.295789234706538,
+ "tokens_seen": 235929600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007502988966900702,
+ "loss": 1.7024,
+ "theoretical_loss": 4.295644716037188,
+ "tokens_seen": 235995136
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750282848545637,
+ "loss": 1.7471,
+ "theoretical_loss": 4.295500248728856,
+ "tokens_seen": 236060672
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007502668004012036,
+ "loss": 1.7045,
+ "theoretical_loss": 4.295355832749038,
+ "tokens_seen": 236126208
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007502507522567704,
+ "loss": 1.7213,
+ "theoretical_loss": 4.295211468065258,
+ "tokens_seen": 236191744
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750234704112337,
+ "loss": 1.715,
+ "theoretical_loss": 4.295067154645071,
+ "tokens_seen": 236257280
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007502186559679038,
+ "loss": 1.6261,
+ "theoretical_loss": 4.294922892456061,
+ "tokens_seen": 236322816
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007502026078234704,
+ "loss": 1.601,
+ "theoretical_loss": 4.294778681465841,
+ "tokens_seen": 236388352
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007501865596790372,
+ "loss": 1.4909,
+ "theoretical_loss": 4.294634521642053,
+ "tokens_seen": 236453888
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007501705115346038,
+ "loss": 1.7144,
+ "theoretical_loss": 4.294490412952371,
+ "tokens_seen": 236519424
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007501544633901706,
+ "loss": 1.7742,
+ "theoretical_loss": 4.294346355364494,
+ "tokens_seen": 236584960
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007501384152457372,
+ "loss": 1.678,
+ "theoretical_loss": 4.294202348846156,
+ "tokens_seen": 236650496
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750122367101304,
+ "loss": 1.6133,
+ "theoretical_loss": 4.294058393365114,
+ "tokens_seen": 236716032
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007501063189568706,
+ "loss": 1.685,
+ "theoretical_loss": 4.293914488889161,
+ "tokens_seen": 236781568
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007500902708124374,
+ "loss": 1.6133,
+ "theoretical_loss": 4.293770635386112,
+ "tokens_seen": 236847104
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000750074222668004,
+ "loss": 1.716,
+ "theoretical_loss": 4.293626832823818,
+ "tokens_seen": 236912640
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007500581745235708,
+ "loss": 1.6319,
+ "theoretical_loss": 4.293483081170153,
+ "tokens_seen": 236978176
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007500421263791374,
+ "loss": 1.5912,
+ "theoretical_loss": 4.293339380393027,
+ "tokens_seen": 237043712
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007500260782347042,
+ "loss": 1.6264,
+ "theoretical_loss": 4.293195730460372,
+ "tokens_seen": 237109248
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007500100300902708,
+ "loss": 1.5781,
+ "theoretical_loss": 4.293052131340154,
+ "tokens_seen": 237174784
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007499939819458376,
+ "loss": 1.7007,
+ "theoretical_loss": 4.292908583000365,
+ "tokens_seen": 237240320
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007499779338014042,
+ "loss": 1.663,
+ "theoretical_loss": 4.29276508540903,
+ "tokens_seen": 237305856
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749961885656971,
+ "loss": 1.5671,
+ "theoretical_loss": 4.292621638534198,
+ "tokens_seen": 237371392
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007499458375125376,
+ "loss": 1.7502,
+ "theoretical_loss": 4.29247824234395,
+ "tokens_seen": 237436928
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007499297893681044,
+ "loss": 1.715,
+ "theoretical_loss": 4.292334896806397,
+ "tokens_seen": 237502464
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749913741223671,
+ "loss": 1.6047,
+ "theoretical_loss": 4.292191601889673,
+ "tokens_seen": 237568000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498976930792378,
+ "loss": 1.6171,
+ "theoretical_loss": 4.2920483575619475,
+ "tokens_seen": 237633536
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498816449348044,
+ "loss": 1.7442,
+ "theoretical_loss": 4.291905163791417,
+ "tokens_seen": 237699072
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498655967903712,
+ "loss": 1.6901,
+ "theoretical_loss": 4.291762020546304,
+ "tokens_seen": 237764608
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498495486459378,
+ "loss": 1.6801,
+ "theoretical_loss": 4.291618927794862,
+ "tokens_seen": 237830144
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498335005015046,
+ "loss": 1.669,
+ "theoretical_loss": 4.291475885505374,
+ "tokens_seen": 237895680
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007498174523570712,
+ "loss": 1.6468,
+ "theoretical_loss": 4.291332893646149,
+ "tokens_seen": 237961216
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749801404212638,
+ "loss": 1.5978,
+ "theoretical_loss": 4.291189952185527,
+ "tokens_seen": 238026752
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007497853560682046,
+ "loss": 1.5644,
+ "theoretical_loss": 4.291047061091875,
+ "tokens_seen": 238092288
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007497693079237714,
+ "loss": 1.5605,
+ "theoretical_loss": 4.290904220333591,
+ "tokens_seen": 238157824
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749753259779338,
+ "loss": 1.6864,
+ "theoretical_loss": 4.2907614298790975,
+ "tokens_seen": 238223360
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007497372116349048,
+ "loss": 1.685,
+ "theoretical_loss": 4.290618689696848,
+ "tokens_seen": 238288896
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007497211634904714,
+ "loss": 1.6605,
+ "theoretical_loss": 4.290475999755326,
+ "tokens_seen": 238354432
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007497051153460382,
+ "loss": 1.7451,
+ "theoretical_loss": 4.29033336002304,
+ "tokens_seen": 238419968
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007496890672016048,
+ "loss": 1.5776,
+ "theoretical_loss": 4.290190770468529,
+ "tokens_seen": 238485504
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007496730190571716,
+ "loss": 1.6556,
+ "theoretical_loss": 4.290048231060361,
+ "tokens_seen": 238551040
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007496569709127382,
+ "loss": 1.5922,
+ "theoretical_loss": 4.289905741767129,
+ "tokens_seen": 238616576
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749640922768305,
+ "loss": 1.6411,
+ "theoretical_loss": 4.289763302557457,
+ "tokens_seen": 238682112
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007496248746238716,
+ "loss": 1.7339,
+ "theoretical_loss": 4.289620913399998,
+ "tokens_seen": 238747648
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007496088264794384,
+ "loss": 1.61,
+ "theoretical_loss": 4.2894785742634305,
+ "tokens_seen": 238813184
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000749592778335005,
+ "loss": 1.6489,
+ "theoretical_loss": 4.289336285116463,
+ "tokens_seen": 238878720
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007495767301905718,
+ "loss": 1.6366,
+ "theoretical_loss": 4.289194045927832,
+ "tokens_seen": 238944256
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007495606820461385,
+ "loss": 1.57,
+ "theoretical_loss": 4.289051856666303,
+ "tokens_seen": 239009792
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007495446339017051,
+ "loss": 1.659,
+ "theoretical_loss": 4.288909717300667,
+ "tokens_seen": 239075328
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007495285857572719,
+ "loss": 1.6939,
+ "theoretical_loss": 4.288767627799745,
+ "tokens_seen": 239140864
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 85669,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6066745519638062,
+ "objective/train/theoretical_loss": 4.288625588132385,
+ "objective/train/tokens_used": 259666400,
+ "theoretical_loss": 4.288625588132385,
+ "tokens_seen": 239206400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007495125376128385,
+ "loss": 1.6276,
+ "theoretical_loss": 4.288625588132385,
+ "tokens_seen": 239206400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494964894684053,
+ "loss": 1.715,
+ "theoretical_loss": 4.288483598267465,
+ "tokens_seen": 239271936
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494804413239719,
+ "loss": 1.5889,
+ "theoretical_loss": 4.288341658173888,
+ "tokens_seen": 239337472
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494643931795387,
+ "loss": 1.7141,
+ "theoretical_loss": 4.288199767820587,
+ "tokens_seen": 239403008
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494483450351053,
+ "loss": 1.682,
+ "theoretical_loss": 4.2880579271765225,
+ "tokens_seen": 239468544
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494322968906721,
+ "loss": 1.5611,
+ "theoretical_loss": 4.287916136210682,
+ "tokens_seen": 239534080
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494162487462387,
+ "loss": 1.6503,
+ "theoretical_loss": 4.287774394892082,
+ "tokens_seen": 239599616
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007494002006018055,
+ "loss": 1.6604,
+ "theoretical_loss": 4.287632703189766,
+ "tokens_seen": 239665152
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493841524573721,
+ "loss": 1.5737,
+ "theoretical_loss": 4.2874910610728065,
+ "tokens_seen": 239730688
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493681043129389,
+ "loss": 1.6437,
+ "theoretical_loss": 4.287349468510302,
+ "tokens_seen": 239796224
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493520561685055,
+ "loss": 1.4903,
+ "theoretical_loss": 4.28720792547138,
+ "tokens_seen": 239861760
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493360080240723,
+ "loss": 1.6124,
+ "theoretical_loss": 4.287066431925194,
+ "tokens_seen": 239927296
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493199598796389,
+ "loss": 1.5895,
+ "theoretical_loss": 4.286924987840926,
+ "tokens_seen": 239992832
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007493039117352057,
+ "loss": 1.7164,
+ "theoretical_loss": 4.2867835931877885,
+ "tokens_seen": 240058368
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492878635907723,
+ "loss": 1.6419,
+ "theoretical_loss": 4.286642247935017,
+ "tokens_seen": 240123904
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492718154463391,
+ "loss": 1.6765,
+ "theoretical_loss": 4.286500952051876,
+ "tokens_seen": 240189440
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492557673019057,
+ "loss": 1.5405,
+ "theoretical_loss": 4.286359705507659,
+ "tokens_seen": 240254976
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492397191574725,
+ "loss": 1.558,
+ "theoretical_loss": 4.286218508271686,
+ "tokens_seen": 240320512
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492236710130391,
+ "loss": 1.5783,
+ "theoretical_loss": 4.286077360313303,
+ "tokens_seen": 240386048
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007492076228686059,
+ "loss": 1.5213,
+ "theoretical_loss": 4.285936261601887,
+ "tokens_seen": 240451584
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491915747241725,
+ "loss": 1.5629,
+ "theoretical_loss": 4.285795212106839,
+ "tokens_seen": 240517120
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491755265797393,
+ "loss": 1.7373,
+ "theoretical_loss": 4.2856542117975875,
+ "tokens_seen": 240582656
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491594784353059,
+ "loss": 1.7176,
+ "theoretical_loss": 4.285513260643591,
+ "tokens_seen": 240648192
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491434302908727,
+ "loss": 1.6888,
+ "theoretical_loss": 4.285372358614332,
+ "tokens_seen": 240713728
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491273821464393,
+ "loss": 1.6917,
+ "theoretical_loss": 4.285231505679324,
+ "tokens_seen": 240779264
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007491113340020061,
+ "loss": 1.6539,
+ "theoretical_loss": 4.285090701808103,
+ "tokens_seen": 240844800
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490952858575727,
+ "loss": 1.612,
+ "theoretical_loss": 4.284949946970236,
+ "tokens_seen": 240910336
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490792377131395,
+ "loss": 1.7414,
+ "theoretical_loss": 4.284809241135315,
+ "tokens_seen": 240975872
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490631895687061,
+ "loss": 1.6637,
+ "theoretical_loss": 4.284668584272962,
+ "tokens_seen": 241041408
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490471414242729,
+ "loss": 1.6183,
+ "theoretical_loss": 4.284527976352823,
+ "tokens_seen": 241106944
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490310932798395,
+ "loss": 1.6879,
+ "theoretical_loss": 4.284387417344572,
+ "tokens_seen": 241172480
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007490150451354063,
+ "loss": 1.6791,
+ "theoretical_loss": 4.28424690721791,
+ "tokens_seen": 241238016
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489989969909729,
+ "loss": 1.7164,
+ "theoretical_loss": 4.284106445942565,
+ "tokens_seen": 241303552
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489829488465397,
+ "loss": 1.6794,
+ "theoretical_loss": 4.2839660334882925,
+ "tokens_seen": 241369088
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489669007021063,
+ "loss": 1.4591,
+ "theoretical_loss": 4.283825669824875,
+ "tokens_seen": 241434624
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489508525576731,
+ "loss": 1.6535,
+ "theoretical_loss": 4.28368535492212,
+ "tokens_seen": 241500160
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489348044132397,
+ "loss": 1.6211,
+ "theoretical_loss": 4.283545088749865,
+ "tokens_seen": 241565696
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489187562688065,
+ "loss": 1.6274,
+ "theoretical_loss": 4.2834048712779715,
+ "tokens_seen": 241631232
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007489027081243731,
+ "loss": 1.5905,
+ "theoretical_loss": 4.28326470247633,
+ "tokens_seen": 241696768
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488866599799399,
+ "loss": 1.6188,
+ "theoretical_loss": 4.283124582314856,
+ "tokens_seen": 241762304
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488706118355065,
+ "loss": 1.5419,
+ "theoretical_loss": 4.282984510763493,
+ "tokens_seen": 241827840
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488545636910733,
+ "loss": 1.6195,
+ "theoretical_loss": 4.282844487792209,
+ "tokens_seen": 241893376
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488385155466399,
+ "loss": 1.6174,
+ "theoretical_loss": 4.282704513371002,
+ "tokens_seen": 241958912
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488224674022067,
+ "loss": 1.5593,
+ "theoretical_loss": 4.282564587469896,
+ "tokens_seen": 242024448
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007488064192577733,
+ "loss": 1.5889,
+ "theoretical_loss": 4.282424710058938,
+ "tokens_seen": 242089984
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487903711133401,
+ "loss": 1.5381,
+ "theoretical_loss": 4.282284881108206,
+ "tokens_seen": 242155520
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487743229689067,
+ "loss": 1.6208,
+ "theoretical_loss": 4.282145100587802,
+ "tokens_seen": 242221056
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487582748244735,
+ "loss": 1.6344,
+ "theoretical_loss": 4.282005368467856,
+ "tokens_seen": 242286592
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487422266800401,
+ "loss": 1.6803,
+ "theoretical_loss": 4.2818656847185235,
+ "tokens_seen": 242352128
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487261785356069,
+ "loss": 1.523,
+ "theoretical_loss": 4.281726049309987,
+ "tokens_seen": 242417664
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 86947,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.533555507659912,
+ "objective/train/theoretical_loss": 4.281586462212455,
+ "objective/train/tokens_used": 262943200,
+ "theoretical_loss": 4.281586462212455,
+ "tokens_seen": 242483200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007487101303911735,
+ "loss": 1.5863,
+ "theoretical_loss": 4.281586462212455,
+ "tokens_seen": 242483200
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486940822467403,
+ "loss": 1.6372,
+ "theoretical_loss": 4.281446923396162,
+ "tokens_seen": 242548736
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486780341023069,
+ "loss": 1.6006,
+ "theoretical_loss": 4.281307432831371,
+ "tokens_seen": 242614272
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486619859578737,
+ "loss": 1.5637,
+ "theoretical_loss": 4.281167990488369,
+ "tokens_seen": 242679808
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486459378134403,
+ "loss": 1.6963,
+ "theoretical_loss": 4.281028596337469,
+ "tokens_seen": 242745344
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486298896690071,
+ "loss": 1.5761,
+ "theoretical_loss": 4.280889250349014,
+ "tokens_seen": 242810880
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007486138415245737,
+ "loss": 1.5397,
+ "theoretical_loss": 4.280749952493368,
+ "tokens_seen": 242876416
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485977933801405,
+ "loss": 1.6975,
+ "theoretical_loss": 4.280610702740926,
+ "tokens_seen": 242941952
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485817452357071,
+ "loss": 1.5826,
+ "theoretical_loss": 4.280471501062106,
+ "tokens_seen": 243007488
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485656970912739,
+ "loss": 1.7064,
+ "theoretical_loss": 4.280332347427353,
+ "tokens_seen": 243073024
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485496489468405,
+ "loss": 1.6329,
+ "theoretical_loss": 4.280193241807139,
+ "tokens_seen": 243138560
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485336008024073,
+ "loss": 1.5267,
+ "theoretical_loss": 4.280054184171963,
+ "tokens_seen": 243204096
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485175526579739,
+ "loss": 1.6792,
+ "theoretical_loss": 4.279915174492345,
+ "tokens_seen": 243269632
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007485015045135407,
+ "loss": 1.638,
+ "theoretical_loss": 4.279776212738838,
+ "tokens_seen": 243335168
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484854563691073,
+ "loss": 1.6731,
+ "theoretical_loss": 4.279637298882016,
+ "tokens_seen": 243400704
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484694082246741,
+ "loss": 1.5625,
+ "theoretical_loss": 4.27949843289248,
+ "tokens_seen": 243466240
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484533600802407,
+ "loss": 1.6964,
+ "theoretical_loss": 4.2793596147408595,
+ "tokens_seen": 243531776
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484373119358075,
+ "loss": 1.6091,
+ "theoretical_loss": 4.279220844397806,
+ "tokens_seen": 243597312
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484212637913741,
+ "loss": 1.6271,
+ "theoretical_loss": 4.279082121834001,
+ "tokens_seen": 243662848
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007484052156469409,
+ "loss": 1.6291,
+ "theoretical_loss": 4.278943447020148,
+ "tokens_seen": 243728384
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483891675025075,
+ "loss": 1.7249,
+ "theoretical_loss": 4.278804819926978,
+ "tokens_seen": 243793920
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483731193580743,
+ "loss": 1.656,
+ "theoretical_loss": 4.278666240525249,
+ "tokens_seen": 243859456
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483570712136409,
+ "loss": 1.7174,
+ "theoretical_loss": 4.278527708785743,
+ "tokens_seen": 243924992
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483410230692077,
+ "loss": 1.6751,
+ "theoretical_loss": 4.278389224679268,
+ "tokens_seen": 243990528
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483249749247745,
+ "loss": 1.4917,
+ "theoretical_loss": 4.27825078817666,
+ "tokens_seen": 244056064
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007483089267803411,
+ "loss": 1.5895,
+ "theoretical_loss": 4.2781123992487755,
+ "tokens_seen": 244121600
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482928786359077,
+ "loss": 1.6703,
+ "theoretical_loss": 4.277974057866503,
+ "tokens_seen": 244187136
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482768304914745,
+ "loss": 1.5676,
+ "theoretical_loss": 4.277835764000752,
+ "tokens_seen": 244252672
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482607823470411,
+ "loss": 1.6488,
+ "theoretical_loss": 4.277697517622459,
+ "tokens_seen": 244318208
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482447342026079,
+ "loss": 1.6331,
+ "theoretical_loss": 4.277559318702588,
+ "tokens_seen": 244383744
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482286860581746,
+ "loss": 1.5206,
+ "theoretical_loss": 4.277421167212125,
+ "tokens_seen": 244449280
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007482126379137413,
+ "loss": 1.6105,
+ "theoretical_loss": 4.277283063122083,
+ "tokens_seen": 244514816
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000748196589769308,
+ "loss": 1.6427,
+ "theoretical_loss": 4.277145006403503,
+ "tokens_seen": 244580352
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007481805416248747,
+ "loss": 1.7534,
+ "theoretical_loss": 4.2770069970274465,
+ "tokens_seen": 244645888
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007481644934804414,
+ "loss": 1.6926,
+ "theoretical_loss": 4.276869034965005,
+ "tokens_seen": 244711424
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000748148445336008,
+ "loss": 1.7342,
+ "theoretical_loss": 4.276731120187293,
+ "tokens_seen": 244776960
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007481323971915748,
+ "loss": 1.6409,
+ "theoretical_loss": 4.2765932526654495,
+ "tokens_seen": 244842496
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007481163490471414,
+ "loss": 1.6483,
+ "theoretical_loss": 4.276455432370643,
+ "tokens_seen": 244908032
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007481003009027082,
+ "loss": 1.5419,
+ "theoretical_loss": 4.276317659274062,
+ "tokens_seen": 244973568
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007480842527582748,
+ "loss": 1.5604,
+ "theoretical_loss": 4.276179933346924,
+ "tokens_seen": 245039104
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007480682046138416,
+ "loss": 1.4418,
+ "theoretical_loss": 4.27604225456047,
+ "tokens_seen": 245104640
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007480521564694082,
+ "loss": 1.6223,
+ "theoretical_loss": 4.275904622885967,
+ "tokens_seen": 245170176
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000748036108324975,
+ "loss": 1.6681,
+ "theoretical_loss": 4.275767038294708,
+ "tokens_seen": 245235712
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007480200601805416,
+ "loss": 1.4692,
+ "theoretical_loss": 4.275629500758008,
+ "tokens_seen": 245301248
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007480040120361084,
+ "loss": 1.5969,
+ "theoretical_loss": 4.27549201024721,
+ "tokens_seen": 245366784
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747987963891675,
+ "loss": 1.6522,
+ "theoretical_loss": 4.275354566733682,
+ "tokens_seen": 245432320
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007479719157472418,
+ "loss": 1.6379,
+ "theoretical_loss": 4.2752171701888155,
+ "tokens_seen": 245497856
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007479558676028084,
+ "loss": 1.5753,
+ "theoretical_loss": 4.275079820584029,
+ "tokens_seen": 245563392
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007479398194583752,
+ "loss": 1.6753,
+ "theoretical_loss": 4.274942517890764,
+ "tokens_seen": 245628928
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007479237713139418,
+ "loss": 1.6301,
+ "theoretical_loss": 4.2748052620804895,
+ "tokens_seen": 245694464
+ },
+ {
+ "epoch": 0.07,
+ "objective/train/docs_used": 87859,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.655349850654602,
+ "objective/train/theoretical_loss": 4.274668053124696,
+ "objective/train/tokens_used": 266220000,
+ "theoretical_loss": 4.274668053124696,
+ "tokens_seen": 245760000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007479077231695086,
+ "loss": 1.6396,
+ "theoretical_loss": 4.274668053124696,
+ "tokens_seen": 245760000
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007478916750250752,
+ "loss": 1.5806,
+ "theoretical_loss": 4.2745308909949005,
+ "tokens_seen": 245825536
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747875626880642,
+ "loss": 1.7082,
+ "theoretical_loss": 4.274393775662648,
+ "tokens_seen": 245891072
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007478595787362086,
+ "loss": 1.6241,
+ "theoretical_loss": 4.274256707099502,
+ "tokens_seen": 245956608
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007478435305917754,
+ "loss": 1.5844,
+ "theoretical_loss": 4.274119685277058,
+ "tokens_seen": 246022144
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747827482447342,
+ "loss": 1.5921,
+ "theoretical_loss": 4.273982710166931,
+ "tokens_seen": 246087680
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007478114343029088,
+ "loss": 1.6682,
+ "theoretical_loss": 4.273845781740762,
+ "tokens_seen": 246153216
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007477953861584754,
+ "loss": 1.5675,
+ "theoretical_loss": 4.273708899970218,
+ "tokens_seen": 246218752
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007477793380140422,
+ "loss": 1.7248,
+ "theoretical_loss": 4.273572064826991,
+ "tokens_seen": 246284288
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007477632898696088,
+ "loss": 1.6493,
+ "theoretical_loss": 4.273435276282795,
+ "tokens_seen": 246349824
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007477472417251756,
+ "loss": 1.6431,
+ "theoretical_loss": 4.2732985343093715,
+ "tokens_seen": 246415360
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007477311935807422,
+ "loss": 1.5834,
+ "theoretical_loss": 4.273161838878485,
+ "tokens_seen": 246480896
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747715145436309,
+ "loss": 1.649,
+ "theoretical_loss": 4.273025189961926,
+ "tokens_seen": 246546432
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476990972918756,
+ "loss": 1.5257,
+ "theoretical_loss": 4.272888587531508,
+ "tokens_seen": 246611968
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476830491474424,
+ "loss": 1.7432,
+ "theoretical_loss": 4.272752031559071,
+ "tokens_seen": 246677504
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747667001003009,
+ "loss": 1.5807,
+ "theoretical_loss": 4.272615522016476,
+ "tokens_seen": 246743040
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476509528585758,
+ "loss": 1.5903,
+ "theoretical_loss": 4.272479058875614,
+ "tokens_seen": 246808576
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476349047141424,
+ "loss": 1.5591,
+ "theoretical_loss": 4.272342642108396,
+ "tokens_seen": 246874112
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476188565697092,
+ "loss": 1.5049,
+ "theoretical_loss": 4.272206271686758,
+ "tokens_seen": 246939648
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007476028084252758,
+ "loss": 1.6674,
+ "theoretical_loss": 4.272069947582662,
+ "tokens_seen": 247005184
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007475867602808426,
+ "loss": 1.5916,
+ "theoretical_loss": 4.271933669768094,
+ "tokens_seen": 247070720
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007475707121364092,
+ "loss": 1.6944,
+ "theoretical_loss": 4.271797438215064,
+ "tokens_seen": 247136256
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747554663991976,
+ "loss": 1.5855,
+ "theoretical_loss": 4.271661252895605,
+ "tokens_seen": 247201792
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007475386158475426,
+ "loss": 1.7893,
+ "theoretical_loss": 4.271525113781777,
+ "tokens_seen": 247267328
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007475225677031094,
+ "loss": 1.6591,
+ "theoretical_loss": 4.271389020845662,
+ "tokens_seen": 247332864
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.000747506519558676,
+ "loss": 1.6706,
+ "theoretical_loss": 4.27125297405937,
+ "tokens_seen": 247398400
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 0.0007474904714142428,
+ "loss": 1.6419,
+ "theoretical_loss": 4.271116973395028,
+ "tokens_seen": 247463936
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007474744232698094,
+ "loss": 1.7433,
+ "theoretical_loss": 4.270981018824795,
+ "tokens_seen": 247529472
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007474583751253762,
+ "loss": 1.6313,
+ "theoretical_loss": 4.270845110320851,
+ "tokens_seen": 247595008
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007474423269809428,
+ "loss": 1.6649,
+ "theoretical_loss": 4.270709247855397,
+ "tokens_seen": 247660544
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007474262788365096,
+ "loss": 1.5666,
+ "theoretical_loss": 4.270573431400663,
+ "tokens_seen": 247726080
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007474102306920762,
+ "loss": 1.729,
+ "theoretical_loss": 4.270437660928902,
+ "tokens_seen": 247791616
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000747394182547643,
+ "loss": 1.6638,
+ "theoretical_loss": 4.270301936412389,
+ "tokens_seen": 247857152
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007473781344032096,
+ "loss": 1.6416,
+ "theoretical_loss": 4.2701662578234245,
+ "tokens_seen": 247922688
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007473620862587764,
+ "loss": 1.5788,
+ "theoretical_loss": 4.270030625134334,
+ "tokens_seen": 247988224
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000747346038114343,
+ "loss": 1.5882,
+ "theoretical_loss": 4.269895038317464,
+ "tokens_seen": 248053760
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007473299899699098,
+ "loss": 1.5104,
+ "theoretical_loss": 4.269759497345188,
+ "tokens_seen": 248119296
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007473139418254764,
+ "loss": 1.6717,
+ "theoretical_loss": 4.269624002189901,
+ "tokens_seen": 248184832
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472978936810432,
+ "loss": 1.7941,
+ "theoretical_loss": 4.269488552824024,
+ "tokens_seen": 248250368
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472818455366098,
+ "loss": 1.7017,
+ "theoretical_loss": 4.269353149220001,
+ "tokens_seen": 248315904
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472657973921766,
+ "loss": 1.5691,
+ "theoretical_loss": 4.2692177913503,
+ "tokens_seen": 248381440
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472497492477432,
+ "loss": 1.5977,
+ "theoretical_loss": 4.2690824791874125,
+ "tokens_seen": 248446976
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074723370110331,
+ "loss": 1.6137,
+ "theoretical_loss": 4.2689472127038535,
+ "tokens_seen": 248512512
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472176529588766,
+ "loss": 1.5904,
+ "theoretical_loss": 4.2688119918721625,
+ "tokens_seen": 248578048
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007472016048144434,
+ "loss": 1.6154,
+ "theoretical_loss": 4.268676816664902,
+ "tokens_seen": 248643584
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074718555667001,
+ "loss": 1.6636,
+ "theoretical_loss": 4.26854168705466,
+ "tokens_seen": 248709120
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007471695085255768,
+ "loss": 1.6526,
+ "theoretical_loss": 4.2684066030140455,
+ "tokens_seen": 248774656
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007471534603811434,
+ "loss": 1.5885,
+ "theoretical_loss": 4.268271564515693,
+ "tokens_seen": 248840192
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007471374122367102,
+ "loss": 1.6694,
+ "theoretical_loss": 4.26813657153226,
+ "tokens_seen": 248905728
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000747121364092277,
+ "loss": 1.6456,
+ "theoretical_loss": 4.268001624036429,
+ "tokens_seen": 248971264
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 89113,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5617129802703857,
+ "objective/train/theoretical_loss": 4.267866722000902,
+ "objective/train/tokens_used": 269496800,
+ "theoretical_loss": 4.267866722000902,
+ "tokens_seen": 249036800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007471053159478436,
+ "loss": 1.6179,
+ "theoretical_loss": 4.267866722000902,
+ "tokens_seen": 249036800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007470892678034103,
+ "loss": 1.8281,
+ "theoretical_loss": 4.26773186539841,
+ "tokens_seen": 249102336
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000747073219658977,
+ "loss": 1.6578,
+ "theoretical_loss": 4.267597054201705,
+ "tokens_seen": 249167872
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007470571715145438,
+ "loss": 1.7296,
+ "theoretical_loss": 4.267462288383561,
+ "tokens_seen": 249233408
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007470411233701104,
+ "loss": 1.5899,
+ "theoretical_loss": 4.267327567916777,
+ "tokens_seen": 249298944
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007470250752256772,
+ "loss": 1.6601,
+ "theoretical_loss": 4.267192892774176,
+ "tokens_seen": 249364480
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007470090270812438,
+ "loss": 1.5475,
+ "theoretical_loss": 4.267058262928604,
+ "tokens_seen": 249430016
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007469929789368106,
+ "loss": 1.6116,
+ "theoretical_loss": 4.26692367835293,
+ "tokens_seen": 249495552
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007469769307923772,
+ "loss": 1.4896,
+ "theoretical_loss": 4.266789139020045,
+ "tokens_seen": 249561088
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000746960882647944,
+ "loss": 1.734,
+ "theoretical_loss": 4.266654644902868,
+ "tokens_seen": 249626624
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007469448345035106,
+ "loss": 1.6688,
+ "theoretical_loss": 4.266520195974336,
+ "tokens_seen": 249692160
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007469287863590774,
+ "loss": 1.6633,
+ "theoretical_loss": 4.266385792207412,
+ "tokens_seen": 249757696
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000746912738214644,
+ "loss": 1.7313,
+ "theoretical_loss": 4.266251433575082,
+ "tokens_seen": 249823232
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468966900702106,
+ "loss": 1.6623,
+ "theoretical_loss": 4.266117120050355,
+ "tokens_seen": 249888768
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468806419257774,
+ "loss": 1.659,
+ "theoretical_loss": 4.265982851606263,
+ "tokens_seen": 249954304
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000746864593781344,
+ "loss": 1.7441,
+ "theoretical_loss": 4.265848628215862,
+ "tokens_seen": 250019840
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468485456369108,
+ "loss": 1.7166,
+ "theoretical_loss": 4.265714449852229,
+ "tokens_seen": 250085376
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468324974924775,
+ "loss": 1.7004,
+ "theoretical_loss": 4.265580316488467,
+ "tokens_seen": 250150912
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468164493480442,
+ "loss": 1.6103,
+ "theoretical_loss": 4.2654462280977015,
+ "tokens_seen": 250216448
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007468004012036109,
+ "loss": 1.5756,
+ "theoretical_loss": 4.265312184653078,
+ "tokens_seen": 250281984
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467843530591776,
+ "loss": 1.6858,
+ "theoretical_loss": 4.2651781861277716,
+ "tokens_seen": 250347520
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467683049147443,
+ "loss": 1.7365,
+ "theoretical_loss": 4.265044232494972,
+ "tokens_seen": 250413056
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467522567703109,
+ "loss": 1.7981,
+ "theoretical_loss": 4.264910323727898,
+ "tokens_seen": 250478592
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467362086258777,
+ "loss": 1.6383,
+ "theoretical_loss": 4.264776459799791,
+ "tokens_seen": 250544128
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467201604814443,
+ "loss": 1.6194,
+ "theoretical_loss": 4.264642640683911,
+ "tokens_seen": 250609664
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007467041123370111,
+ "loss": 1.6848,
+ "theoretical_loss": 4.264508866353546,
+ "tokens_seen": 250675200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466880641925777,
+ "loss": 1.7286,
+ "theoretical_loss": 4.264375136782004,
+ "tokens_seen": 250740736
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466720160481445,
+ "loss": 1.7089,
+ "theoretical_loss": 4.264241451942617,
+ "tokens_seen": 250806272
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466559679037111,
+ "loss": 1.5735,
+ "theoretical_loss": 4.264107811808739,
+ "tokens_seen": 250871808
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466399197592779,
+ "loss": 1.6118,
+ "theoretical_loss": 4.263974216353749,
+ "tokens_seen": 250937344
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466238716148445,
+ "loss": 1.5047,
+ "theoretical_loss": 4.2638406655510455,
+ "tokens_seen": 251002880
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007466078234704113,
+ "loss": 1.6449,
+ "theoretical_loss": 4.263707159374051,
+ "tokens_seen": 251068416
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465917753259779,
+ "loss": 1.5712,
+ "theoretical_loss": 4.263573697796213,
+ "tokens_seen": 251133952
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465757271815447,
+ "loss": 1.6243,
+ "theoretical_loss": 4.263440280790999,
+ "tokens_seen": 251199488
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465596790371113,
+ "loss": 1.6167,
+ "theoretical_loss": 4.263306908331899,
+ "tokens_seen": 251265024
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465436308926781,
+ "loss": 1.6755,
+ "theoretical_loss": 4.2631735803924276,
+ "tokens_seen": 251330560
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465275827482447,
+ "loss": 1.7118,
+ "theoretical_loss": 4.263040296946122,
+ "tokens_seen": 251396096
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007465115346038115,
+ "loss": 1.7401,
+ "theoretical_loss": 4.26290705796654,
+ "tokens_seen": 251461632
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464954864593781,
+ "loss": 1.6834,
+ "theoretical_loss": 4.262773863427265,
+ "tokens_seen": 251527168
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464794383149449,
+ "loss": 1.7491,
+ "theoretical_loss": 4.262640713301899,
+ "tokens_seen": 251592704
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464633901705115,
+ "loss": 1.5683,
+ "theoretical_loss": 4.26250760756407,
+ "tokens_seen": 251658240
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464473420260783,
+ "loss": 1.6373,
+ "theoretical_loss": 4.262374546187428,
+ "tokens_seen": 251723776
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464312938816449,
+ "loss": 1.4961,
+ "theoretical_loss": 4.262241529145643,
+ "tokens_seen": 251789312
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007464152457372117,
+ "loss": 1.6324,
+ "theoretical_loss": 4.262108556412411,
+ "tokens_seen": 251854848
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463991975927783,
+ "loss": 1.8033,
+ "theoretical_loss": 4.261975627961448,
+ "tokens_seen": 251920384
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463831494483451,
+ "loss": 1.7477,
+ "theoretical_loss": 4.2618427437664925,
+ "tokens_seen": 251985920
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463671013039117,
+ "loss": 1.6968,
+ "theoretical_loss": 4.261709903801307,
+ "tokens_seen": 252051456
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463510531594785,
+ "loss": 1.6687,
+ "theoretical_loss": 4.261577108039674,
+ "tokens_seen": 252116992
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463350050150451,
+ "loss": 1.7375,
+ "theoretical_loss": 4.261444356455402,
+ "tokens_seen": 252182528
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463189568706119,
+ "loss": 1.6986,
+ "theoretical_loss": 4.261311649022318,
+ "tokens_seen": 252248064
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 90341,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.549142599105835,
+ "objective/train/theoretical_loss": 4.261178985714273,
+ "objective/train/tokens_used": 272773600,
+ "theoretical_loss": 4.261178985714273,
+ "tokens_seen": 252313600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007463029087261785,
+ "loss": 1.6768,
+ "theoretical_loss": 4.261178985714273,
+ "tokens_seen": 252313600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462868605817453,
+ "loss": 1.5756,
+ "theoretical_loss": 4.26104636650514,
+ "tokens_seen": 252379136
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462708124373119,
+ "loss": 1.4787,
+ "theoretical_loss": 4.260913791368815,
+ "tokens_seen": 252444672
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462547642928787,
+ "loss": 1.5706,
+ "theoretical_loss": 4.260781260279216,
+ "tokens_seen": 252510208
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462387161484453,
+ "loss": 1.5784,
+ "theoretical_loss": 4.260648773210281,
+ "tokens_seen": 252575744
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462226680040121,
+ "loss": 1.6752,
+ "theoretical_loss": 4.2605163301359745,
+ "tokens_seen": 252641280
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007462066198595787,
+ "loss": 1.5707,
+ "theoretical_loss": 4.260383931030279,
+ "tokens_seen": 252706816
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461905717151455,
+ "loss": 1.6033,
+ "theoretical_loss": 4.2602515758672,
+ "tokens_seen": 252772352
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461745235707121,
+ "loss": 1.7105,
+ "theoretical_loss": 4.260119264620768,
+ "tokens_seen": 252837888
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461584754262789,
+ "loss": 1.7694,
+ "theoretical_loss": 4.259986997265033,
+ "tokens_seen": 252903424
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461424272818455,
+ "loss": 1.5492,
+ "theoretical_loss": 4.259854773774066,
+ "tokens_seen": 252968960
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461263791374123,
+ "loss": 1.5911,
+ "theoretical_loss": 4.259722594121963,
+ "tokens_seen": 253034496
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007461103309929789,
+ "loss": 1.6801,
+ "theoretical_loss": 4.259590458282839,
+ "tokens_seen": 253100032
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460942828485457,
+ "loss": 1.6821,
+ "theoretical_loss": 4.259458366230835,
+ "tokens_seen": 253165568
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460782347041123,
+ "loss": 1.6812,
+ "theoretical_loss": 4.2593263179401095,
+ "tokens_seen": 253231104
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460621865596791,
+ "loss": 1.6885,
+ "theoretical_loss": 4.259194313384846,
+ "tokens_seen": 253296640
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460461384152457,
+ "loss": 1.6347,
+ "theoretical_loss": 4.259062352539247,
+ "tokens_seen": 253362176
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460300902708125,
+ "loss": 1.6389,
+ "theoretical_loss": 4.258930435377541,
+ "tokens_seen": 253427712
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007460140421263791,
+ "loss": 1.6788,
+ "theoretical_loss": 4.258798561873974,
+ "tokens_seen": 253493248
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459979939819459,
+ "loss": 1.6139,
+ "theoretical_loss": 4.258666732002817,
+ "tokens_seen": 253558784
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459819458375125,
+ "loss": 1.6973,
+ "theoretical_loss": 4.258534945738362,
+ "tokens_seen": 253624320
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459658976930793,
+ "loss": 1.6564,
+ "theoretical_loss": 4.258403203054923,
+ "tokens_seen": 253689856
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459498495486459,
+ "loss": 1.6421,
+ "theoretical_loss": 4.258271503926833,
+ "tokens_seen": 253755392
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459338014042127,
+ "loss": 1.6334,
+ "theoretical_loss": 4.258139848328451,
+ "tokens_seen": 253820928
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459177532597794,
+ "loss": 1.6213,
+ "theoretical_loss": 4.258008236234154,
+ "tokens_seen": 253886464
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007459017051153461,
+ "loss": 1.6186,
+ "theoretical_loss": 4.257876667618344,
+ "tokens_seen": 253952000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458856569709128,
+ "loss": 1.6503,
+ "theoretical_loss": 4.257745142455442,
+ "tokens_seen": 254017536
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458696088264795,
+ "loss": 1.7115,
+ "theoretical_loss": 4.257613660719892,
+ "tokens_seen": 254083072
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458535606820462,
+ "loss": 1.6601,
+ "theoretical_loss": 4.25748222238616,
+ "tokens_seen": 254148608
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458375125376129,
+ "loss": 1.5442,
+ "theoretical_loss": 4.2573508274287315,
+ "tokens_seen": 254214144
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458214643931796,
+ "loss": 1.4927,
+ "theoretical_loss": 4.257219475822116,
+ "tokens_seen": 254279680
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007458054162487463,
+ "loss": 1.5835,
+ "theoretical_loss": 4.257088167540843,
+ "tokens_seen": 254345216
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745789368104313,
+ "loss": 1.5372,
+ "theoretical_loss": 4.256956902559465,
+ "tokens_seen": 254410752
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007457733199598797,
+ "loss": 1.6738,
+ "theoretical_loss": 4.256825680852554,
+ "tokens_seen": 254476288
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007457572718154464,
+ "loss": 1.646,
+ "theoretical_loss": 4.256694502394704,
+ "tokens_seen": 254541824
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007457412236710131,
+ "loss": 1.6983,
+ "theoretical_loss": 4.256563367160533,
+ "tokens_seen": 254607360
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007457251755265798,
+ "loss": 1.6854,
+ "theoretical_loss": 4.256432275124676,
+ "tokens_seen": 254672896
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007457091273821465,
+ "loss": 1.6195,
+ "theoretical_loss": 4.256301226261794,
+ "tokens_seen": 254738432
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007456930792377133,
+ "loss": 1.5589,
+ "theoretical_loss": 4.256170220546565,
+ "tokens_seen": 254803968
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007456770310932799,
+ "loss": 1.4944,
+ "theoretical_loss": 4.2560392579536925,
+ "tokens_seen": 254869504
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007456609829488467,
+ "loss": 1.5076,
+ "theoretical_loss": 4.255908338457898,
+ "tokens_seen": 254935040
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007456449348044133,
+ "loss": 1.5108,
+ "theoretical_loss": 4.255777462033926,
+ "tokens_seen": 255000576
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074562888665998,
+ "loss": 1.5947,
+ "theoretical_loss": 4.255646628656542,
+ "tokens_seen": 255066112
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007456128385155467,
+ "loss": 1.5601,
+ "theoretical_loss": 4.255515838300534,
+ "tokens_seen": 255131648
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455967903711135,
+ "loss": 1.6591,
+ "theoretical_loss": 4.255385090940708,
+ "tokens_seen": 255197184
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455807422266801,
+ "loss": 1.6293,
+ "theoretical_loss": 4.255254386551894,
+ "tokens_seen": 255262720
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455646940822469,
+ "loss": 1.5927,
+ "theoretical_loss": 4.255123725108943,
+ "tokens_seen": 255328256
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455486459378135,
+ "loss": 1.584,
+ "theoretical_loss": 4.254993106586725,
+ "tokens_seen": 255393792
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455325977933803,
+ "loss": 1.6033,
+ "theoretical_loss": 4.254862530960134,
+ "tokens_seen": 255459328
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455165496489469,
+ "loss": 1.628,
+ "theoretical_loss": 4.254731998204083,
+ "tokens_seen": 255524864
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 91319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5541565418243408,
+ "objective/train/theoretical_loss": 4.254601508293508,
+ "objective/train/tokens_used": 276050400,
+ "theoretical_loss": 4.254601508293508,
+ "tokens_seen": 255590400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007455005015045136,
+ "loss": 1.5715,
+ "theoretical_loss": 4.254601508293508,
+ "tokens_seen": 255590400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007454844533600803,
+ "loss": 1.6553,
+ "theoretical_loss": 4.254471061203364,
+ "tokens_seen": 255655936
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745468405215647,
+ "loss": 1.7168,
+ "theoretical_loss": 4.254340656908628,
+ "tokens_seen": 255721472
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007454523570712137,
+ "loss": 1.636,
+ "theoretical_loss": 4.254210295384298,
+ "tokens_seen": 255787008
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007454363089267804,
+ "loss": 1.6813,
+ "theoretical_loss": 4.254079976605394,
+ "tokens_seen": 255852544
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007454202607823471,
+ "loss": 1.5497,
+ "theoretical_loss": 4.253949700546956,
+ "tokens_seen": 255918080
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007454042126379138,
+ "loss": 1.5719,
+ "theoretical_loss": 4.2538194671840435,
+ "tokens_seen": 255983616
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007453881644934805,
+ "loss": 1.7041,
+ "theoretical_loss": 4.25368927649174,
+ "tokens_seen": 256049152
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007453721163490472,
+ "loss": 1.5223,
+ "theoretical_loss": 4.253559128445148,
+ "tokens_seen": 256114688
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007453560682046138,
+ "loss": 1.6966,
+ "theoretical_loss": 4.253429023019391,
+ "tokens_seen": 256180224
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007453400200601806,
+ "loss": 1.6931,
+ "theoretical_loss": 4.253298960189614,
+ "tokens_seen": 256245760
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007453239719157472,
+ "loss": 1.5582,
+ "theoretical_loss": 4.253168939930982,
+ "tokens_seen": 256311296
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745307923771314,
+ "loss": 1.5669,
+ "theoretical_loss": 4.253038962218682,
+ "tokens_seen": 256376832
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007452918756268806,
+ "loss": 1.7125,
+ "theoretical_loss": 4.25290902702792,
+ "tokens_seen": 256442368
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007452758274824474,
+ "loss": 1.6,
+ "theoretical_loss": 4.252779134333926,
+ "tokens_seen": 256507904
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745259779338014,
+ "loss": 1.7343,
+ "theoretical_loss": 4.252649284111946,
+ "tokens_seen": 256573440
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007452437311935808,
+ "loss": 1.6937,
+ "theoretical_loss": 4.252519476337251,
+ "tokens_seen": 256638976
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007452276830491474,
+ "loss": 1.5897,
+ "theoretical_loss": 4.252389710985131,
+ "tokens_seen": 256704512
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007452116349047142,
+ "loss": 1.7181,
+ "theoretical_loss": 4.252259988030896,
+ "tokens_seen": 256770048
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007451955867602808,
+ "loss": 1.6677,
+ "theoretical_loss": 4.252130307449878,
+ "tokens_seen": 256835584
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007451795386158476,
+ "loss": 1.6119,
+ "theoretical_loss": 4.252000669217429,
+ "tokens_seen": 256901120
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007451634904714142,
+ "loss": 1.722,
+ "theoretical_loss": 4.251871073308923,
+ "tokens_seen": 256966656
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745147442326981,
+ "loss": 1.6881,
+ "theoretical_loss": 4.251741519699751,
+ "tokens_seen": 257032192
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007451313941825476,
+ "loss": 1.6454,
+ "theoretical_loss": 4.251612008365328,
+ "tokens_seen": 257097728
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007451153460381144,
+ "loss": 1.527,
+ "theoretical_loss": 4.2514825392810875,
+ "tokens_seen": 257163264
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000745099297893681,
+ "loss": 1.6586,
+ "theoretical_loss": 4.251353112422486,
+ "tokens_seen": 257228800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450832497492478,
+ "loss": 1.6275,
+ "theoretical_loss": 4.251223727764999,
+ "tokens_seen": 257294336
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450672016048144,
+ "loss": 1.716,
+ "theoretical_loss": 4.251094385284121,
+ "tokens_seen": 257359872
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450511534603812,
+ "loss": 1.6393,
+ "theoretical_loss": 4.25096508495537,
+ "tokens_seen": 257425408
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450351053159478,
+ "loss": 1.6811,
+ "theoretical_loss": 4.250835826754281,
+ "tokens_seen": 257490944
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450190571715146,
+ "loss": 1.6071,
+ "theoretical_loss": 4.2507066106564135,
+ "tokens_seen": 257556480
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007450030090270812,
+ "loss": 1.7105,
+ "theoretical_loss": 4.250577436637344,
+ "tokens_seen": 257622016
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744986960882648,
+ "loss": 1.6548,
+ "theoretical_loss": 4.25044830467267,
+ "tokens_seen": 257687552
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007449709127382146,
+ "loss": 1.7264,
+ "theoretical_loss": 4.250319214738011,
+ "tokens_seen": 257753088
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007449548645937814,
+ "loss": 1.6103,
+ "theoretical_loss": 4.250190166809005,
+ "tokens_seen": 257818624
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744938816449348,
+ "loss": 1.6217,
+ "theoretical_loss": 4.250061160861311,
+ "tokens_seen": 257884160
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007449227683049148,
+ "loss": 1.5565,
+ "theoretical_loss": 4.249932196870608,
+ "tokens_seen": 257949696
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007449067201604814,
+ "loss": 1.6584,
+ "theoretical_loss": 4.249803274812598,
+ "tokens_seen": 258015232
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007448906720160482,
+ "loss": 1.7406,
+ "theoretical_loss": 4.249674394662998,
+ "tokens_seen": 258080768
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007448746238716148,
+ "loss": 1.5355,
+ "theoretical_loss": 4.249545556397549,
+ "tokens_seen": 258146304
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007448585757271816,
+ "loss": 1.6406,
+ "theoretical_loss": 4.249416759992011,
+ "tokens_seen": 258211840
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007448425275827482,
+ "loss": 1.5055,
+ "theoretical_loss": 4.249288005422166,
+ "tokens_seen": 258277376
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744826479438315,
+ "loss": 1.5312,
+ "theoretical_loss": 4.249159292663812,
+ "tokens_seen": 258342912
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007448104312938816,
+ "loss": 1.6441,
+ "theoretical_loss": 4.249030621692772,
+ "tokens_seen": 258408448
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007447943831494484,
+ "loss": 1.6091,
+ "theoretical_loss": 4.248901992484885,
+ "tokens_seen": 258473984
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744778335005015,
+ "loss": 1.6485,
+ "theoretical_loss": 4.248773405016014,
+ "tokens_seen": 258539520
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007447622868605818,
+ "loss": 1.6405,
+ "theoretical_loss": 4.248644859262039,
+ "tokens_seen": 258605056
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007447462387161484,
+ "loss": 1.5547,
+ "theoretical_loss": 4.248516355198861,
+ "tokens_seen": 258670592
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007447301905717152,
+ "loss": 1.6182,
+ "theoretical_loss": 4.2483878928024,
+ "tokens_seen": 258736128
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007447141424272819,
+ "loss": 1.592,
+ "theoretical_loss": 4.248259472048598,
+ "tokens_seen": 258801664
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 92500,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5000085830688477,
+ "objective/train/theoretical_loss": 4.248131092913416,
+ "objective/train/tokens_used": 279327200,
+ "theoretical_loss": 4.248131092913416,
+ "tokens_seen": 258867200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446980942828486,
+ "loss": 1.5452,
+ "theoretical_loss": 4.248131092913416,
+ "tokens_seen": 258867200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446820461384153,
+ "loss": 1.542,
+ "theoretical_loss": 4.248002755372836,
+ "tokens_seen": 258932736
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744665997993982,
+ "loss": 1.6977,
+ "theoretical_loss": 4.247874459402857,
+ "tokens_seen": 258998272
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446499498495487,
+ "loss": 1.5523,
+ "theoretical_loss": 4.2477462049795,
+ "tokens_seen": 259063808
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446339017051154,
+ "loss": 1.6554,
+ "theoretical_loss": 4.2476179920788075,
+ "tokens_seen": 259129344
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446178535606821,
+ "loss": 1.557,
+ "theoretical_loss": 4.247489820676839,
+ "tokens_seen": 259194880
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007446018054162488,
+ "loss": 1.6801,
+ "theoretical_loss": 4.247361690749676,
+ "tokens_seen": 259260416
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007445857572718155,
+ "loss": 1.604,
+ "theoretical_loss": 4.2472336022734165,
+ "tokens_seen": 259325952
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007445697091273822,
+ "loss": 1.6637,
+ "theoretical_loss": 4.2471055552241825,
+ "tokens_seen": 259391488
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007445536609829489,
+ "loss": 1.6168,
+ "theoretical_loss": 4.246977549578113,
+ "tokens_seen": 259457024
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007445376128385156,
+ "loss": 1.6913,
+ "theoretical_loss": 4.246849585311368,
+ "tokens_seen": 259522560
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007445215646940823,
+ "loss": 1.6747,
+ "theoretical_loss": 4.246721662400128,
+ "tokens_seen": 259588096
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744505516549649,
+ "loss": 1.5241,
+ "theoretical_loss": 4.246593780820589,
+ "tokens_seen": 259653632
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444894684052157,
+ "loss": 1.5544,
+ "theoretical_loss": 4.2464659405489735,
+ "tokens_seen": 259719168
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444734202607824,
+ "loss": 1.4963,
+ "theoretical_loss": 4.246338141561518,
+ "tokens_seen": 259784704
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444573721163491,
+ "loss": 1.5923,
+ "theoretical_loss": 4.24621038383448,
+ "tokens_seen": 259850240
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444413239719158,
+ "loss": 1.643,
+ "theoretical_loss": 4.246082667344139,
+ "tokens_seen": 259915776
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444252758274825,
+ "loss": 1.5979,
+ "theoretical_loss": 4.245954992066792,
+ "tokens_seen": 259981312
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007444092276830492,
+ "loss": 1.7024,
+ "theoretical_loss": 4.245827357978754,
+ "tokens_seen": 260046848
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744393179538616,
+ "loss": 1.5572,
+ "theoretical_loss": 4.245699765056365,
+ "tokens_seen": 260112384
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007443771313941826,
+ "loss": 1.5896,
+ "theoretical_loss": 4.245572213275978,
+ "tokens_seen": 260177920
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007443610832497493,
+ "loss": 1.8514,
+ "theoretical_loss": 4.245444702613971,
+ "tokens_seen": 260243456
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744345035105316,
+ "loss": 1.586,
+ "theoretical_loss": 4.245317233046738,
+ "tokens_seen": 260308992
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007443289869608828,
+ "loss": 1.7146,
+ "theoretical_loss": 4.2451898045506935,
+ "tokens_seen": 260374528
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007443129388164494,
+ "loss": 1.6225,
+ "theoretical_loss": 4.245062417102272,
+ "tokens_seen": 260440064
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442968906720162,
+ "loss": 1.592,
+ "theoretical_loss": 4.244935070677927,
+ "tokens_seen": 260505600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442808425275828,
+ "loss": 1.6766,
+ "theoretical_loss": 4.244807765254132,
+ "tokens_seen": 260571136
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442647943831496,
+ "loss": 1.6643,
+ "theoretical_loss": 4.244680500807378,
+ "tokens_seen": 260636672
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442487462387162,
+ "loss": 1.5448,
+ "theoretical_loss": 4.244553277314178,
+ "tokens_seen": 260702208
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744232698094283,
+ "loss": 1.6036,
+ "theoretical_loss": 4.244426094751063,
+ "tokens_seen": 260767744
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442166499498496,
+ "loss": 1.6473,
+ "theoretical_loss": 4.244298953094583,
+ "tokens_seen": 260833280
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007442006018054164,
+ "loss": 1.6384,
+ "theoretical_loss": 4.244171852321308,
+ "tokens_seen": 260898816
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000744184553660983,
+ "loss": 1.6358,
+ "theoretical_loss": 4.244044792407828,
+ "tokens_seen": 260964352
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007441685055165498,
+ "loss": 1.6318,
+ "theoretical_loss": 4.24391777333075,
+ "tokens_seen": 261029888
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007441524573721164,
+ "loss": 1.6431,
+ "theoretical_loss": 4.243790795066703,
+ "tokens_seen": 261095424
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007441364092276832,
+ "loss": 1.5781,
+ "theoretical_loss": 4.243663857592332,
+ "tokens_seen": 261160960
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007441203610832498,
+ "loss": 1.5393,
+ "theoretical_loss": 4.243536960884305,
+ "tokens_seen": 261226496
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007441043129388166,
+ "loss": 1.6555,
+ "theoretical_loss": 4.243410104919307,
+ "tokens_seen": 261292032
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007440882647943832,
+ "loss": 1.6852,
+ "theoretical_loss": 4.243283289674042,
+ "tokens_seen": 261357568
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007440722166499499,
+ "loss": 1.4922,
+ "theoretical_loss": 4.243156515125234,
+ "tokens_seen": 261423104
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007440561685055166,
+ "loss": 1.6475,
+ "theoretical_loss": 4.243029781249625,
+ "tokens_seen": 261488640
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007440401203610833,
+ "loss": 1.5878,
+ "theoretical_loss": 4.2429030880239775,
+ "tokens_seen": 261554176
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074402407221665,
+ "loss": 1.6059,
+ "theoretical_loss": 4.242776435425072,
+ "tokens_seen": 261619712
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007440080240722167,
+ "loss": 1.5873,
+ "theoretical_loss": 4.24264982342971,
+ "tokens_seen": 261685248
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439919759277834,
+ "loss": 1.5429,
+ "theoretical_loss": 4.242523252014709,
+ "tokens_seen": 261750784
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439759277833501,
+ "loss": 1.6651,
+ "theoretical_loss": 4.242396721156908,
+ "tokens_seen": 261816320
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439598796389168,
+ "loss": 1.7161,
+ "theoretical_loss": 4.242270230833164,
+ "tokens_seen": 261881856
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439438314944835,
+ "loss": 1.5095,
+ "theoretical_loss": 4.242143781020353,
+ "tokens_seen": 261947392
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439277833500501,
+ "loss": 1.6434,
+ "theoretical_loss": 4.24201737169537,
+ "tokens_seen": 262012928
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007439117352056169,
+ "loss": 1.6189,
+ "theoretical_loss": 4.24189100283513,
+ "tokens_seen": 262078464
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 93423,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6364566087722778,
+ "objective/train/theoretical_loss": 4.241764674416565,
+ "objective/train/tokens_used": 282604000,
+ "theoretical_loss": 4.241764674416565,
+ "tokens_seen": 262144000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438956870611835,
+ "loss": 1.6476,
+ "theoretical_loss": 4.241764674416565,
+ "tokens_seen": 262144000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438796389167503,
+ "loss": 1.5735,
+ "theoretical_loss": 4.241638386416628,
+ "tokens_seen": 262209536
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438635907723169,
+ "loss": 1.6634,
+ "theoretical_loss": 4.241512138812288,
+ "tokens_seen": 262275072
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438475426278837,
+ "loss": 1.5158,
+ "theoretical_loss": 4.2413859315805365,
+ "tokens_seen": 262340608
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438314944834503,
+ "loss": 1.5613,
+ "theoretical_loss": 4.241259764698381,
+ "tokens_seen": 262406144
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007438154463390171,
+ "loss": 1.5698,
+ "theoretical_loss": 4.24113363814285,
+ "tokens_seen": 262471680
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437993981945837,
+ "loss": 1.7,
+ "theoretical_loss": 4.241007551890988,
+ "tokens_seen": 262537216
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437833500501505,
+ "loss": 1.5795,
+ "theoretical_loss": 4.2408815059198615,
+ "tokens_seen": 262602752
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437673019057171,
+ "loss": 1.6302,
+ "theoretical_loss": 4.240755500206554,
+ "tokens_seen": 262668288
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437512537612839,
+ "loss": 1.6311,
+ "theoretical_loss": 4.240629534728167,
+ "tokens_seen": 262733824
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437352056168505,
+ "loss": 1.6255,
+ "theoretical_loss": 4.240503609461822,
+ "tokens_seen": 262799360
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437191574724173,
+ "loss": 1.8019,
+ "theoretical_loss": 4.24037772438466,
+ "tokens_seen": 262864896
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007437031093279839,
+ "loss": 1.5956,
+ "theoretical_loss": 4.240251879473839,
+ "tokens_seen": 262930432
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436870611835507,
+ "loss": 1.6039,
+ "theoretical_loss": 4.240126074706536,
+ "tokens_seen": 262995968
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436710130391173,
+ "loss": 1.6342,
+ "theoretical_loss": 4.240000310059948,
+ "tokens_seen": 263061504
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436549648946841,
+ "loss": 1.5717,
+ "theoretical_loss": 4.239874585511288,
+ "tokens_seen": 263127040
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436389167502507,
+ "loss": 1.7075,
+ "theoretical_loss": 4.239748901037791,
+ "tokens_seen": 263192576
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436228686058175,
+ "loss": 1.5863,
+ "theoretical_loss": 4.2396232566167065,
+ "tokens_seen": 263258112
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007436068204613841,
+ "loss": 1.7253,
+ "theoretical_loss": 4.239497652225307,
+ "tokens_seen": 263323648
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007435907723169509,
+ "loss": 1.5759,
+ "theoretical_loss": 4.23937208784088,
+ "tokens_seen": 263389184
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007435747241725175,
+ "loss": 1.5407,
+ "theoretical_loss": 4.239246563440734,
+ "tokens_seen": 263454720
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007435586760280843,
+ "loss": 1.512,
+ "theoretical_loss": 4.239121079002194,
+ "tokens_seen": 263520256
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000743542627883651,
+ "loss": 1.5218,
+ "theoretical_loss": 4.238995634502606,
+ "tokens_seen": 263585792
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007435265797392177,
+ "loss": 1.6162,
+ "theoretical_loss": 4.238870229919331,
+ "tokens_seen": 263651328
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007435105315947844,
+ "loss": 1.5465,
+ "theoretical_loss": 4.2387448652297515,
+ "tokens_seen": 263716864
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434944834503511,
+ "loss": 1.6953,
+ "theoretical_loss": 4.238619540411268,
+ "tokens_seen": 263782400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434784353059178,
+ "loss": 1.6432,
+ "theoretical_loss": 4.238494255441296,
+ "tokens_seen": 263847936
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434623871614845,
+ "loss": 1.6252,
+ "theoretical_loss": 4.238369010297275,
+ "tokens_seen": 263913472
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434463390170512,
+ "loss": 1.6152,
+ "theoretical_loss": 4.238243804956658,
+ "tokens_seen": 263979008
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434302908726179,
+ "loss": 1.6624,
+ "theoretical_loss": 4.23811863939692,
+ "tokens_seen": 264044544
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007434142427281846,
+ "loss": 1.605,
+ "theoretical_loss": 4.237993513595552,
+ "tokens_seen": 264110080
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433981945837513,
+ "loss": 1.5313,
+ "theoretical_loss": 4.2378684275300635,
+ "tokens_seen": 264175616
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000743382146439318,
+ "loss": 1.5915,
+ "theoretical_loss": 4.237743381177983,
+ "tokens_seen": 264241152
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433660982948847,
+ "loss": 1.6108,
+ "theoretical_loss": 4.237618374516858,
+ "tokens_seen": 264306688
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433500501504514,
+ "loss": 1.6528,
+ "theoretical_loss": 4.237493407524253,
+ "tokens_seen": 264372224
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433340020060181,
+ "loss": 1.5117,
+ "theoretical_loss": 4.23736848017775,
+ "tokens_seen": 264437760
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433179538615848,
+ "loss": 1.6246,
+ "theoretical_loss": 4.237243592454951,
+ "tokens_seen": 264503296
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007433019057171515,
+ "loss": 1.6768,
+ "theoretical_loss": 4.2371187443334755,
+ "tokens_seen": 264568832
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007432858575727182,
+ "loss": 1.7698,
+ "theoretical_loss": 4.236993935790962,
+ "tokens_seen": 264634368
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007432698094282849,
+ "loss": 1.5859,
+ "theoretical_loss": 4.236869166805064,
+ "tokens_seen": 264699904
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007432537612838516,
+ "loss": 1.4231,
+ "theoretical_loss": 4.236744437353458,
+ "tokens_seen": 264765440
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007432377131394183,
+ "loss": 1.7275,
+ "theoretical_loss": 4.236619747413836,
+ "tokens_seen": 264830976
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000743221664994985,
+ "loss": 1.5382,
+ "theoretical_loss": 4.236495096963906,
+ "tokens_seen": 264896512
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007432056168505517,
+ "loss": 1.6893,
+ "theoretical_loss": 4.236370485981398,
+ "tokens_seen": 264962048
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431895687061184,
+ "loss": 1.676,
+ "theoretical_loss": 4.236245914444058,
+ "tokens_seen": 265027584
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431735205616851,
+ "loss": 1.5495,
+ "theoretical_loss": 4.2361213823296495,
+ "tokens_seen": 265093120
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431574724172518,
+ "loss": 1.6253,
+ "theoretical_loss": 4.235996889615957,
+ "tokens_seen": 265158656
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431414242728185,
+ "loss": 1.646,
+ "theoretical_loss": 4.235872436280779,
+ "tokens_seen": 265224192
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431253761283852,
+ "loss": 1.7312,
+ "theoretical_loss": 4.235748022301935,
+ "tokens_seen": 265289728
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007431093279839519,
+ "loss": 1.5934,
+ "theoretical_loss": 4.235623647657261,
+ "tokens_seen": 265355264
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 94744,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6758512258529663,
+ "objective/train/theoretical_loss": 4.235499312324611,
+ "objective/train/tokens_used": 285880800,
+ "theoretical_loss": 4.235499312324611,
+ "tokens_seen": 265420800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007430932798395186,
+ "loss": 1.6044,
+ "theoretical_loss": 4.235499312324611,
+ "tokens_seen": 265420800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007430772316950853,
+ "loss": 1.6172,
+ "theoretical_loss": 4.235375016281858,
+ "tokens_seen": 265486336
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000743061183550652,
+ "loss": 1.5535,
+ "theoretical_loss": 4.235250759506892,
+ "tokens_seen": 265551872
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007430451354062187,
+ "loss": 1.4944,
+ "theoretical_loss": 4.235126541977621,
+ "tokens_seen": 265617408
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007430290872617854,
+ "loss": 1.6137,
+ "theoretical_loss": 4.2350023636719705,
+ "tokens_seen": 265682944
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007430130391173521,
+ "loss": 1.4967,
+ "theoretical_loss": 4.234878224567885,
+ "tokens_seen": 265748480
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429969909729188,
+ "loss": 1.5846,
+ "theoretical_loss": 4.2347541246433265,
+ "tokens_seen": 265814016
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429809428284855,
+ "loss": 1.6215,
+ "theoretical_loss": 4.234630063876273,
+ "tokens_seen": 265879552
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429648946840523,
+ "loss": 1.5888,
+ "theoretical_loss": 4.234506042244723,
+ "tokens_seen": 265945088
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429488465396189,
+ "loss": 1.6198,
+ "theoretical_loss": 4.234382059726691,
+ "tokens_seen": 266010624
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429327983951857,
+ "loss": 1.635,
+ "theoretical_loss": 4.234258116300211,
+ "tokens_seen": 266076160
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007429167502507523,
+ "loss": 1.5904,
+ "theoretical_loss": 4.234134211943332,
+ "tokens_seen": 266141696
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000742900702106319,
+ "loss": 1.6317,
+ "theoretical_loss": 4.234010346634123,
+ "tokens_seen": 266207232
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428846539618857,
+ "loss": 1.6059,
+ "theoretical_loss": 4.23388652035067,
+ "tokens_seen": 266272768
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428686058174525,
+ "loss": 1.5262,
+ "theoretical_loss": 4.233762733071076,
+ "tokens_seen": 266338304
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428525576730191,
+ "loss": 1.6322,
+ "theoretical_loss": 4.233638984773464,
+ "tokens_seen": 266403840
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428365095285859,
+ "loss": 1.7282,
+ "theoretical_loss": 4.2335152754359715,
+ "tokens_seen": 266469376
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428204613841525,
+ "loss": 1.5823,
+ "theoretical_loss": 4.233391605036756,
+ "tokens_seen": 266534912
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007428044132397193,
+ "loss": 1.555,
+ "theoretical_loss": 4.233267973553991,
+ "tokens_seen": 266600448
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427883650952859,
+ "loss": 1.6832,
+ "theoretical_loss": 4.233144380965869,
+ "tokens_seen": 266665984
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427723169508527,
+ "loss": 1.5752,
+ "theoretical_loss": 4.2330208272505985,
+ "tokens_seen": 266731520
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427562688064193,
+ "loss": 1.6642,
+ "theoretical_loss": 4.232897312386408,
+ "tokens_seen": 266797056
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427402206619861,
+ "loss": 1.6123,
+ "theoretical_loss": 4.232773836351541,
+ "tokens_seen": 266862592
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427241725175527,
+ "loss": 1.6768,
+ "theoretical_loss": 4.2326503991242586,
+ "tokens_seen": 266928128
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007427081243731195,
+ "loss": 1.5605,
+ "theoretical_loss": 4.232527000682843,
+ "tokens_seen": 266993664
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426920762286861,
+ "loss": 1.6535,
+ "theoretical_loss": 4.232403641005589,
+ "tokens_seen": 267059200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426760280842528,
+ "loss": 1.6531,
+ "theoretical_loss": 4.232280320070812,
+ "tokens_seen": 267124736
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426599799398195,
+ "loss": 1.6343,
+ "theoretical_loss": 4.232157037856844,
+ "tokens_seen": 267190272
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426439317953862,
+ "loss": 1.6088,
+ "theoretical_loss": 4.232033794342033,
+ "tokens_seen": 267255808
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426278836509529,
+ "loss": 1.5571,
+ "theoretical_loss": 4.231910589504748,
+ "tokens_seen": 267321344
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007426118355065196,
+ "loss": 1.626,
+ "theoretical_loss": 4.2317874233233725,
+ "tokens_seen": 267386880
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007425957873620863,
+ "loss": 1.561,
+ "theoretical_loss": 4.231664295776307,
+ "tokens_seen": 267452416
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000742579739217653,
+ "loss": 1.5373,
+ "theoretical_loss": 4.231541206841971,
+ "tokens_seen": 267517952
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007425636910732197,
+ "loss": 1.5562,
+ "theoretical_loss": 4.2314181564988,
+ "tokens_seen": 267583488
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007425476429287864,
+ "loss": 1.614,
+ "theoretical_loss": 4.23129514472525,
+ "tokens_seen": 267649024
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000742531594784353,
+ "loss": 1.5684,
+ "theoretical_loss": 4.231172171499789,
+ "tokens_seen": 267714560
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007425155466399198,
+ "loss": 1.6038,
+ "theoretical_loss": 4.231049236800907,
+ "tokens_seen": 267780096
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424994984954864,
+ "loss": 1.5656,
+ "theoretical_loss": 4.230926340607109,
+ "tokens_seen": 267845632
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424834503510532,
+ "loss": 1.6124,
+ "theoretical_loss": 4.230803482896918,
+ "tokens_seen": 267911168
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424674022066198,
+ "loss": 1.6741,
+ "theoretical_loss": 4.230680663648872,
+ "tokens_seen": 267976704
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424513540621866,
+ "loss": 1.6197,
+ "theoretical_loss": 4.230557882841531,
+ "tokens_seen": 268042240
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424353059177532,
+ "loss": 1.7328,
+ "theoretical_loss": 4.2304351404534675,
+ "tokens_seen": 268107776
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074241925777332,
+ "loss": 1.6919,
+ "theoretical_loss": 4.230312436463272,
+ "tokens_seen": 268173312
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007424032096288866,
+ "loss": 1.6116,
+ "theoretical_loss": 4.230189770849556,
+ "tokens_seen": 268238848
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007423871614844534,
+ "loss": 1.6098,
+ "theoretical_loss": 4.230067143590943,
+ "tokens_seen": 268304384
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074237111334002,
+ "loss": 1.5206,
+ "theoretical_loss": 4.229944554666076,
+ "tokens_seen": 268369920
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007423550651955868,
+ "loss": 1.6026,
+ "theoretical_loss": 4.229822004053617,
+ "tokens_seen": 268435456
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007423390170511535,
+ "loss": 1.6134,
+ "theoretical_loss": 4.229699491732242,
+ "tokens_seen": 268500992
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007423229689067202,
+ "loss": 1.5862,
+ "theoretical_loss": 4.2295770176806435,
+ "tokens_seen": 268566528
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007423069207622869,
+ "loss": 1.6335,
+ "theoretical_loss": 4.229454581877535,
+ "tokens_seen": 268632064
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 95678,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.629549503326416,
+ "objective/train/theoretical_loss": 4.229332184301644,
+ "objective/train/tokens_used": 289157600,
+ "theoretical_loss": 4.229332184301644,
+ "tokens_seen": 268697600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007422908726178536,
+ "loss": 1.6398,
+ "theoretical_loss": 4.229332184301644,
+ "tokens_seen": 268697600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007422748244734203,
+ "loss": 1.4887,
+ "theoretical_loss": 4.229209824931714,
+ "tokens_seen": 268763136
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000742258776328987,
+ "loss": 1.6316,
+ "theoretical_loss": 4.22908750374651,
+ "tokens_seen": 268828672
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007422427281845537,
+ "loss": 1.5414,
+ "theoretical_loss": 4.22896522072481,
+ "tokens_seen": 268894208
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007422266800401204,
+ "loss": 1.6666,
+ "theoretical_loss": 4.22884297584541,
+ "tokens_seen": 268959744
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007422106318956871,
+ "loss": 1.6226,
+ "theoretical_loss": 4.228720769087124,
+ "tokens_seen": 269025280
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421945837512538,
+ "loss": 1.4824,
+ "theoretical_loss": 4.22859860042878,
+ "tokens_seen": 269090816
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421785356068205,
+ "loss": 1.574,
+ "theoretical_loss": 4.2284764698492285,
+ "tokens_seen": 269156352
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421624874623872,
+ "loss": 1.6074,
+ "theoretical_loss": 4.22835437732733,
+ "tokens_seen": 269221888
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421464393179539,
+ "loss": 1.6837,
+ "theoretical_loss": 4.228232322841967,
+ "tokens_seen": 269287424
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421303911735206,
+ "loss": 1.5557,
+ "theoretical_loss": 4.228110306372036,
+ "tokens_seen": 269352960
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007421143430290873,
+ "loss": 1.5514,
+ "theoretical_loss": 4.227988327896453,
+ "tokens_seen": 269418496
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000742098294884654,
+ "loss": 1.6616,
+ "theoretical_loss": 4.2278663873941476,
+ "tokens_seen": 269484032
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420822467402207,
+ "loss": 1.5448,
+ "theoretical_loss": 4.227744484844068,
+ "tokens_seen": 269549568
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420661985957874,
+ "loss": 1.5115,
+ "theoretical_loss": 4.227622620225182,
+ "tokens_seen": 269615104
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420501504513541,
+ "loss": 1.6058,
+ "theoretical_loss": 4.227500793516466,
+ "tokens_seen": 269680640
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420341023069208,
+ "loss": 1.6111,
+ "theoretical_loss": 4.227379004696923,
+ "tokens_seen": 269746176
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420180541624875,
+ "loss": 1.5459,
+ "theoretical_loss": 4.227257253745566,
+ "tokens_seen": 269811712
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007420020060180542,
+ "loss": 1.6436,
+ "theoretical_loss": 4.227135540641426,
+ "tokens_seen": 269877248
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007419859578736209,
+ "loss": 1.6299,
+ "theoretical_loss": 4.227013865363553,
+ "tokens_seen": 269942784
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007419699097291876,
+ "loss": 1.5194,
+ "theoretical_loss": 4.226892227891012,
+ "tokens_seen": 270008320
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007419538615847543,
+ "loss": 1.5954,
+ "theoretical_loss": 4.2267706282028845,
+ "tokens_seen": 270073856
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741937813440321,
+ "loss": 1.596,
+ "theoretical_loss": 4.226649066278269,
+ "tokens_seen": 270139392
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007419217652958877,
+ "loss": 1.7046,
+ "theoretical_loss": 4.226527542096282,
+ "tokens_seen": 270204928
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007419057171514544,
+ "loss": 1.6551,
+ "theoretical_loss": 4.226406055636053,
+ "tokens_seen": 270270464
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418896690070211,
+ "loss": 1.6659,
+ "theoretical_loss": 4.226284606876731,
+ "tokens_seen": 270336000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418736208625878,
+ "loss": 1.5405,
+ "theoretical_loss": 4.226163195797481,
+ "tokens_seen": 270401536
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418575727181545,
+ "loss": 1.6297,
+ "theoretical_loss": 4.226041822377486,
+ "tokens_seen": 270467072
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418415245737212,
+ "loss": 1.5975,
+ "theoretical_loss": 4.225920486595941,
+ "tokens_seen": 270532608
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418254764292879,
+ "loss": 1.6092,
+ "theoretical_loss": 4.225799188432063,
+ "tokens_seen": 270598144
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007418094282848546,
+ "loss": 1.665,
+ "theoretical_loss": 4.225677927865083,
+ "tokens_seen": 270663680
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007417933801404213,
+ "loss": 1.5838,
+ "theoretical_loss": 4.225556704874247,
+ "tokens_seen": 270729216
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741777331995988,
+ "loss": 1.6257,
+ "theoretical_loss": 4.22543551943882,
+ "tokens_seen": 270794752
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007417612838515547,
+ "loss": 1.5903,
+ "theoretical_loss": 4.225314371538083,
+ "tokens_seen": 270860288
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007417452357071214,
+ "loss": 1.5667,
+ "theoretical_loss": 4.225193261151332,
+ "tokens_seen": 270925824
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007417291875626881,
+ "loss": 1.5673,
+ "theoretical_loss": 4.22507218825788,
+ "tokens_seen": 270991360
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007417131394182548,
+ "loss": 1.5828,
+ "theoretical_loss": 4.224951152837058,
+ "tokens_seen": 271056896
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007416970912738215,
+ "loss": 1.6085,
+ "theoretical_loss": 4.2248301548682115,
+ "tokens_seen": 271122432
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007416810431293882,
+ "loss": 1.5875,
+ "theoretical_loss": 4.2247091943307025,
+ "tokens_seen": 271187968
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741664994984955,
+ "loss": 1.5357,
+ "theoretical_loss": 4.224588271203911,
+ "tokens_seen": 271253504
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007416489468405216,
+ "loss": 1.7111,
+ "theoretical_loss": 4.22446738546723,
+ "tokens_seen": 271319040
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007416328986960883,
+ "loss": 1.6451,
+ "theoretical_loss": 4.224346537100074,
+ "tokens_seen": 271384576
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741616850551655,
+ "loss": 1.4726,
+ "theoretical_loss": 4.22422572608187,
+ "tokens_seen": 271450112
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007416008024072218,
+ "loss": 1.5835,
+ "theoretical_loss": 4.224104952392061,
+ "tokens_seen": 271515648
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007415847542627884,
+ "loss": 1.6244,
+ "theoretical_loss": 4.223984216010107,
+ "tokens_seen": 271581184
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007415687061183552,
+ "loss": 1.6205,
+ "theoretical_loss": 4.223863516915486,
+ "tokens_seen": 271646720
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007415526579739218,
+ "loss": 1.5239,
+ "theoretical_loss": 4.223742855087691,
+ "tokens_seen": 271712256
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007415366098294886,
+ "loss": 1.5946,
+ "theoretical_loss": 4.223622230506231,
+ "tokens_seen": 271777792
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007415205616850552,
+ "loss": 1.7028,
+ "theoretical_loss": 4.22350164315063,
+ "tokens_seen": 271843328
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741504513540622,
+ "loss": 1.5819,
+ "theoretical_loss": 4.22338109300043,
+ "tokens_seen": 271908864
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 96867,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5667804479599,
+ "objective/train/theoretical_loss": 4.22326058003519,
+ "objective/train/tokens_used": 292434400,
+ "theoretical_loss": 4.22326058003519,
+ "tokens_seen": 271974400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007414884653961886,
+ "loss": 1.5744,
+ "theoretical_loss": 4.22326058003519,
+ "tokens_seen": 271974400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007414724172517554,
+ "loss": 1.6947,
+ "theoretical_loss": 4.223140104234482,
+ "tokens_seen": 272039936
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741456369107322,
+ "loss": 1.5442,
+ "theoretical_loss": 4.2230196655778975,
+ "tokens_seen": 272105472
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007414403209628888,
+ "loss": 1.707,
+ "theoretical_loss": 4.222899264045042,
+ "tokens_seen": 272171008
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007414242728184554,
+ "loss": 1.635,
+ "theoretical_loss": 4.2227788996155375,
+ "tokens_seen": 272236544
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007414082246740222,
+ "loss": 1.6033,
+ "theoretical_loss": 4.222658572269022,
+ "tokens_seen": 272302080
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007413921765295888,
+ "loss": 1.6143,
+ "theoretical_loss": 4.222538281985151,
+ "tokens_seen": 272367616
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007413761283851556,
+ "loss": 1.6715,
+ "theoretical_loss": 4.2224180287435935,
+ "tokens_seen": 272433152
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007413600802407222,
+ "loss": 1.5306,
+ "theoretical_loss": 4.222297812524037,
+ "tokens_seen": 272498688
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741344032096289,
+ "loss": 1.6365,
+ "theoretical_loss": 4.222177633306183,
+ "tokens_seen": 272564224
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007413279839518556,
+ "loss": 1.657,
+ "theoretical_loss": 4.222057491069751,
+ "tokens_seen": 272629760
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007413119358074224,
+ "loss": 1.5467,
+ "theoretical_loss": 4.221937385794474,
+ "tokens_seen": 272695296
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741295887662989,
+ "loss": 1.5622,
+ "theoretical_loss": 4.221817317460104,
+ "tokens_seen": 272760832
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007412798395185557,
+ "loss": 1.53,
+ "theoretical_loss": 4.221697286046407,
+ "tokens_seen": 272826368
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007412637913741224,
+ "loss": 1.4924,
+ "theoretical_loss": 4.221577291533165,
+ "tokens_seen": 272891904
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007412477432296891,
+ "loss": 1.583,
+ "theoretical_loss": 4.221457333900176,
+ "tokens_seen": 272957440
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007412316950852558,
+ "loss": 1.5246,
+ "theoretical_loss": 4.221337413127254,
+ "tokens_seen": 273022976
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007412156469408225,
+ "loss": 1.42,
+ "theoretical_loss": 4.221217529194231,
+ "tokens_seen": 273088512
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411995987963892,
+ "loss": 1.6641,
+ "theoretical_loss": 4.22109768208095,
+ "tokens_seen": 273154048
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411835506519559,
+ "loss": 1.5788,
+ "theoretical_loss": 4.220977871767275,
+ "tokens_seen": 273219584
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411675025075226,
+ "loss": 1.6302,
+ "theoretical_loss": 4.220858098233082,
+ "tokens_seen": 273285120
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411514543630893,
+ "loss": 1.5654,
+ "theoretical_loss": 4.220738361458266,
+ "tokens_seen": 273350656
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000741135406218656,
+ "loss": 1.5599,
+ "theoretical_loss": 4.220618661422735,
+ "tokens_seen": 273416192
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411193580742227,
+ "loss": 1.6897,
+ "theoretical_loss": 4.2204989981064145,
+ "tokens_seen": 273481728
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007411033099297894,
+ "loss": 1.5683,
+ "theoretical_loss": 4.220379371489246,
+ "tokens_seen": 273547264
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410872617853561,
+ "loss": 1.5457,
+ "theoretical_loss": 4.220259781551185,
+ "tokens_seen": 273612800
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410712136409228,
+ "loss": 1.6194,
+ "theoretical_loss": 4.2201402282722045,
+ "tokens_seen": 273678336
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410551654964895,
+ "loss": 1.6366,
+ "theoretical_loss": 4.2200207116322925,
+ "tokens_seen": 273743872
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410391173520562,
+ "loss": 1.6136,
+ "theoretical_loss": 4.219901231611452,
+ "tokens_seen": 273809408
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410230692076229,
+ "loss": 1.6047,
+ "theoretical_loss": 4.219781788189703,
+ "tokens_seen": 273874944
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007410070210631896,
+ "loss": 1.4999,
+ "theoretical_loss": 4.2196623813470815,
+ "tokens_seen": 273940480
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007409909729187563,
+ "loss": 1.6752,
+ "theoretical_loss": 4.219543011063637,
+ "tokens_seen": 274006016
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000740974924774323,
+ "loss": 1.7212,
+ "theoretical_loss": 4.219423677319437,
+ "tokens_seen": 274071552
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007409588766298897,
+ "loss": 1.6442,
+ "theoretical_loss": 4.219304380094562,
+ "tokens_seen": 274137088
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007409428284854564,
+ "loss": 1.6143,
+ "theoretical_loss": 4.2191851193691114,
+ "tokens_seen": 274202624
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007409267803410231,
+ "loss": 1.6529,
+ "theoretical_loss": 4.219065895123197,
+ "tokens_seen": 274268160
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007409107321965898,
+ "loss": 1.6902,
+ "theoretical_loss": 4.218946707336949,
+ "tokens_seen": 274333696
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007408946840521565,
+ "loss": 1.5987,
+ "theoretical_loss": 4.2188275559905115,
+ "tokens_seen": 274399232
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007408786359077232,
+ "loss": 1.527,
+ "theoretical_loss": 4.218708441064044,
+ "tokens_seen": 274464768
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007408625877632899,
+ "loss": 1.5728,
+ "theoretical_loss": 4.218589362537723,
+ "tokens_seen": 274530304
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007408465396188566,
+ "loss": 1.5688,
+ "theoretical_loss": 4.218470320391739,
+ "tokens_seen": 274595840
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007408304914744233,
+ "loss": 1.5903,
+ "theoretical_loss": 4.218351314606298,
+ "tokens_seen": 274661376
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.00074081444332999,
+ "loss": 1.6266,
+ "theoretical_loss": 4.218232345161622,
+ "tokens_seen": 274726912
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407983951855567,
+ "loss": 1.5244,
+ "theoretical_loss": 4.21811341203795,
+ "tokens_seen": 274792448
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407823470411234,
+ "loss": 1.6264,
+ "theoretical_loss": 4.217994515215534,
+ "tokens_seen": 274857984
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407662988966901,
+ "loss": 1.6983,
+ "theoretical_loss": 4.217875654674642,
+ "tokens_seen": 274923520
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407502507522568,
+ "loss": 1.5818,
+ "theoretical_loss": 4.217756830395559,
+ "tokens_seen": 274989056
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407342026078235,
+ "loss": 1.6596,
+ "theoretical_loss": 4.217638042358583,
+ "tokens_seen": 275054592
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407181544633902,
+ "loss": 1.6276,
+ "theoretical_loss": 4.21751929054403,
+ "tokens_seen": 275120128
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007407021063189569,
+ "loss": 1.6848,
+ "theoretical_loss": 4.2174005749322285,
+ "tokens_seen": 275185664
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 97725,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.526691198348999,
+ "objective/train/theoretical_loss": 4.2172818955035245,
+ "objective/train/tokens_used": 295711200,
+ "theoretical_loss": 4.2172818955035245,
+ "tokens_seen": 275251200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007406860581745236,
+ "loss": 1.5652,
+ "theoretical_loss": 4.2172818955035245,
+ "tokens_seen": 275251200
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007406700100300903,
+ "loss": 1.5369,
+ "theoretical_loss": 4.217163252238279,
+ "tokens_seen": 275316736
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000740653961885657,
+ "loss": 1.4076,
+ "theoretical_loss": 4.217044645116869,
+ "tokens_seen": 275382272
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007406379137412237,
+ "loss": 1.6474,
+ "theoretical_loss": 4.216926074119684,
+ "tokens_seen": 275447808
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007406218655967904,
+ "loss": 1.5036,
+ "theoretical_loss": 4.216807539227132,
+ "tokens_seen": 275513344
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007406058174523571,
+ "loss": 1.6137,
+ "theoretical_loss": 4.2166890404196335,
+ "tokens_seen": 275578880
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405897693079238,
+ "loss": 1.6188,
+ "theoretical_loss": 4.216570577677627,
+ "tokens_seen": 275644416
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405737211634905,
+ "loss": 1.6735,
+ "theoretical_loss": 4.216452150981566,
+ "tokens_seen": 275709952
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405576730190572,
+ "loss": 1.4854,
+ "theoretical_loss": 4.216333760311916,
+ "tokens_seen": 275775488
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405416248746239,
+ "loss": 1.6522,
+ "theoretical_loss": 4.216215405649161,
+ "tokens_seen": 275841024
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405255767301906,
+ "loss": 1.5212,
+ "theoretical_loss": 4.2160970869738,
+ "tokens_seen": 275906560
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007405095285857573,
+ "loss": 1.7428,
+ "theoretical_loss": 4.215978804266345,
+ "tokens_seen": 275972096
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000740493480441324,
+ "loss": 1.6548,
+ "theoretical_loss": 4.215860557507324,
+ "tokens_seen": 276037632
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007404774322968907,
+ "loss": 1.4961,
+ "theoretical_loss": 4.215742346677283,
+ "tokens_seen": 276103168
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007404613841524574,
+ "loss": 1.5168,
+ "theoretical_loss": 4.215624171756779,
+ "tokens_seen": 276168704
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007404453360080241,
+ "loss": 1.5582,
+ "theoretical_loss": 4.215506032726386,
+ "tokens_seen": 276234240
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007404292878635908,
+ "loss": 1.6843,
+ "theoretical_loss": 4.215387929566694,
+ "tokens_seen": 276299776
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007404132397191575,
+ "loss": 1.6185,
+ "theoretical_loss": 4.215269862258307,
+ "tokens_seen": 276365312
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403971915747242,
+ "loss": 1.4809,
+ "theoretical_loss": 4.215151830781843,
+ "tokens_seen": 276430848
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403811434302909,
+ "loss": 1.4883,
+ "theoretical_loss": 4.2150338351179375,
+ "tokens_seen": 276496384
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403650952858576,
+ "loss": 1.6069,
+ "theoretical_loss": 4.21491587524724,
+ "tokens_seen": 276561920
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403490471414243,
+ "loss": 1.7038,
+ "theoretical_loss": 4.214797951150415,
+ "tokens_seen": 276627456
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000740332998996991,
+ "loss": 1.581,
+ "theoretical_loss": 4.214680062808142,
+ "tokens_seen": 276692992
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403169508525577,
+ "loss": 1.542,
+ "theoretical_loss": 4.214562210201114,
+ "tokens_seen": 276758528
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007403009027081244,
+ "loss": 1.4822,
+ "theoretical_loss": 4.214444393310042,
+ "tokens_seen": 276824064
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402848545636911,
+ "loss": 1.5861,
+ "theoretical_loss": 4.2143266121156495,
+ "tokens_seen": 276889600
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402688064192578,
+ "loss": 1.6427,
+ "theoretical_loss": 4.2142088665986766,
+ "tokens_seen": 276955136
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402527582748245,
+ "loss": 1.535,
+ "theoretical_loss": 4.214091156739878,
+ "tokens_seen": 277020672
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402367101303912,
+ "loss": 1.6281,
+ "theoretical_loss": 4.213973482520021,
+ "tokens_seen": 277086208
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402206619859579,
+ "loss": 1.6517,
+ "theoretical_loss": 4.213855843919891,
+ "tokens_seen": 277151744
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007402046138415247,
+ "loss": 1.6089,
+ "theoretical_loss": 4.2137382409202875,
+ "tokens_seen": 277217280
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007401885656970913,
+ "loss": 1.6273,
+ "theoretical_loss": 4.2136206735020245,
+ "tokens_seen": 277282816
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000740172517552658,
+ "loss": 1.5309,
+ "theoretical_loss": 4.21350314164593,
+ "tokens_seen": 277348352
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007401564694082247,
+ "loss": 1.612,
+ "theoretical_loss": 4.213385645332848,
+ "tokens_seen": 277413888
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007401404212637915,
+ "loss": 1.7019,
+ "theoretical_loss": 4.213268184543638,
+ "tokens_seen": 277479424
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007401243731193581,
+ "loss": 1.5525,
+ "theoretical_loss": 4.213150759259172,
+ "tokens_seen": 277544960
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007401083249749249,
+ "loss": 1.601,
+ "theoretical_loss": 4.213033369460339,
+ "tokens_seen": 277610496
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400922768304915,
+ "loss": 1.55,
+ "theoretical_loss": 4.212916015128041,
+ "tokens_seen": 277676032
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400762286860583,
+ "loss": 1.6188,
+ "theoretical_loss": 4.212798696243197,
+ "tokens_seen": 277741568
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400601805416249,
+ "loss": 1.6865,
+ "theoretical_loss": 4.212681412786738,
+ "tokens_seen": 277807104
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400441323971917,
+ "loss": 1.6119,
+ "theoretical_loss": 4.212564164739613,
+ "tokens_seen": 277872640
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400280842527583,
+ "loss": 1.653,
+ "theoretical_loss": 4.212446952082783,
+ "tokens_seen": 277938176
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007400120361083251,
+ "loss": 1.5911,
+ "theoretical_loss": 4.212329774797225,
+ "tokens_seen": 278003712
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399959879638917,
+ "loss": 1.5933,
+ "theoretical_loss": 4.212212632863931,
+ "tokens_seen": 278069248
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399799398194585,
+ "loss": 1.5824,
+ "theoretical_loss": 4.212095526263905,
+ "tokens_seen": 278134784
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399638916750251,
+ "loss": 1.593,
+ "theoretical_loss": 4.211978454978171,
+ "tokens_seen": 278200320
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399478435305919,
+ "loss": 1.4985,
+ "theoretical_loss": 4.211861418987762,
+ "tokens_seen": 278265856
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399317953861585,
+ "loss": 1.7031,
+ "theoretical_loss": 4.211744418273728,
+ "tokens_seen": 278331392
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007399157472417253,
+ "loss": 1.4938,
+ "theoretical_loss": 4.211627452817136,
+ "tokens_seen": 278396928
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398996990972919,
+ "loss": 1.5638,
+ "theoretical_loss": 4.211510522599063,
+ "tokens_seen": 278462464
+ },
+ {
+ "epoch": 0.08,
+ "objective/train/docs_used": 98910,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5239521265029907,
+ "objective/train/theoretical_loss": 4.2113936276006045,
+ "objective/train/tokens_used": 298988000,
+ "theoretical_loss": 4.2113936276006045,
+ "tokens_seen": 278528000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398836509528586,
+ "loss": 1.5001,
+ "theoretical_loss": 4.2113936276006045,
+ "tokens_seen": 278528000
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398676028084253,
+ "loss": 1.4508,
+ "theoretical_loss": 4.211276767802868,
+ "tokens_seen": 278593536
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000739851554663992,
+ "loss": 1.5051,
+ "theoretical_loss": 4.211159943186978,
+ "tokens_seen": 278659072
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398355065195587,
+ "loss": 1.5914,
+ "theoretical_loss": 4.211043153734071,
+ "tokens_seen": 278724608
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398194583751254,
+ "loss": 1.5629,
+ "theoretical_loss": 4.2109263994253,
+ "tokens_seen": 278790144
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007398034102306921,
+ "loss": 1.5097,
+ "theoretical_loss": 4.21080968024183,
+ "tokens_seen": 278855680
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397873620862588,
+ "loss": 1.6827,
+ "theoretical_loss": 4.210692996164845,
+ "tokens_seen": 278921216
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397713139418255,
+ "loss": 1.539,
+ "theoretical_loss": 4.210576347175539,
+ "tokens_seen": 278986752
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397552657973922,
+ "loss": 1.6543,
+ "theoretical_loss": 4.210459733255123,
+ "tokens_seen": 279052288
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397392176529589,
+ "loss": 1.5957,
+ "theoretical_loss": 4.2103431543848195,
+ "tokens_seen": 279117824
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397231695085256,
+ "loss": 1.5742,
+ "theoretical_loss": 4.210226610545872,
+ "tokens_seen": 279183360
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007397071213640923,
+ "loss": 1.6426,
+ "theoretical_loss": 4.21011010171953,
+ "tokens_seen": 279248896
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000739691073219659,
+ "loss": 1.5525,
+ "theoretical_loss": 4.209993627887064,
+ "tokens_seen": 279314432
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007396750250752257,
+ "loss": 1.5869,
+ "theoretical_loss": 4.209877189029756,
+ "tokens_seen": 279379968
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007396589769307924,
+ "loss": 1.6383,
+ "theoretical_loss": 4.209760785128903,
+ "tokens_seen": 279445504
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007396429287863591,
+ "loss": 1.638,
+ "theoretical_loss": 4.2096444161658155,
+ "tokens_seen": 279511040
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007396268806419258,
+ "loss": 1.5023,
+ "theoretical_loss": 4.209528082121819,
+ "tokens_seen": 279576576
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007396108324974925,
+ "loss": 1.5516,
+ "theoretical_loss": 4.209411782978256,
+ "tokens_seen": 279642112
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007395947843530592,
+ "loss": 1.5677,
+ "theoretical_loss": 4.209295518716477,
+ "tokens_seen": 279707648
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007395787362086259,
+ "loss": 1.5189,
+ "theoretical_loss": 4.209179289317854,
+ "tokens_seen": 279773184
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007395626880641926,
+ "loss": 1.5775,
+ "theoretical_loss": 4.209063094763767,
+ "tokens_seen": 279838720
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007395466399197593,
+ "loss": 1.5707,
+ "theoretical_loss": 4.208946935035616,
+ "tokens_seen": 279904256
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.000739530591775326,
+ "loss": 1.5592,
+ "theoretical_loss": 4.2088308101148115,
+ "tokens_seen": 279969792
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007395145436308927,
+ "loss": 1.6149,
+ "theoretical_loss": 4.208714719982778,
+ "tokens_seen": 280035328
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394984954864594,
+ "loss": 1.5742,
+ "theoretical_loss": 4.208598664620958,
+ "tokens_seen": 280100864
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394824473420261,
+ "loss": 1.6079,
+ "theoretical_loss": 4.208482644010804,
+ "tokens_seen": 280166400
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394663991975928,
+ "loss": 1.5717,
+ "theoretical_loss": 4.2083666581337855,
+ "tokens_seen": 280231936
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394503510531595,
+ "loss": 1.5169,
+ "theoretical_loss": 4.208250706971385,
+ "tokens_seen": 280297472
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394343029087262,
+ "loss": 1.6327,
+ "theoretical_loss": 4.208134790505099,
+ "tokens_seen": 280363008
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394182547642929,
+ "loss": 1.6121,
+ "theoretical_loss": 4.20801890871644,
+ "tokens_seen": 280428544
+ },
+ {
+ "epoch": 0.08,
+ "learning_rate": 0.0007394022066198596,
+ "loss": 1.5979,
+ "theoretical_loss": 4.207903061586931,
+ "tokens_seen": 280494080
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007393861584754263,
+ "loss": 1.6794,
+ "theoretical_loss": 4.207787249098113,
+ "tokens_seen": 280559616
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000739370110330993,
+ "loss": 1.5258,
+ "theoretical_loss": 4.20767147123154,
+ "tokens_seen": 280625152
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007393540621865597,
+ "loss": 1.6724,
+ "theoretical_loss": 4.20755572796878,
+ "tokens_seen": 280690688
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007393380140421264,
+ "loss": 1.5395,
+ "theoretical_loss": 4.207440019291413,
+ "tokens_seen": 280756224
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007393219658976931,
+ "loss": 1.6244,
+ "theoretical_loss": 4.2073243451810365,
+ "tokens_seen": 280821760
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007393059177532598,
+ "loss": 1.6882,
+ "theoretical_loss": 4.20720870561926,
+ "tokens_seen": 280887296
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007392898696088265,
+ "loss": 1.4866,
+ "theoretical_loss": 4.2070931005877075,
+ "tokens_seen": 280952832
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007392738214643932,
+ "loss": 1.5088,
+ "theoretical_loss": 4.206977530068017,
+ "tokens_seen": 281018368
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007392577733199599,
+ "loss": 1.583,
+ "theoretical_loss": 4.206861994041842,
+ "tokens_seen": 281083904
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007392417251755266,
+ "loss": 1.5661,
+ "theoretical_loss": 4.206746492490847,
+ "tokens_seen": 281149440
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007392256770310933,
+ "loss": 1.6379,
+ "theoretical_loss": 4.206631025396714,
+ "tokens_seen": 281214976
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.00073920962888666,
+ "loss": 1.587,
+ "theoretical_loss": 4.206515592741135,
+ "tokens_seen": 281280512
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391935807422267,
+ "loss": 1.6539,
+ "theoretical_loss": 4.20640019450582,
+ "tokens_seen": 281346048
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391775325977934,
+ "loss": 1.6555,
+ "theoretical_loss": 4.206284830672491,
+ "tokens_seen": 281411584
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391614844533601,
+ "loss": 1.6364,
+ "theoretical_loss": 4.206169501222885,
+ "tokens_seen": 281477120
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391454363089268,
+ "loss": 1.5291,
+ "theoretical_loss": 4.20605420613875,
+ "tokens_seen": 281542656
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391293881644935,
+ "loss": 1.5918,
+ "theoretical_loss": 4.205938945401851,
+ "tokens_seen": 281608192
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007391133400200602,
+ "loss": 1.5979,
+ "theoretical_loss": 4.205823718993966,
+ "tokens_seen": 281673728
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390972918756269,
+ "loss": 1.6177,
+ "theoretical_loss": 4.2057085268968875,
+ "tokens_seen": 281739264
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 99932,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.517197608947754,
+ "objective/train/theoretical_loss": 4.205593369092421,
+ "objective/train/tokens_used": 302264800,
+ "theoretical_loss": 4.205593369092421,
+ "tokens_seen": 281804800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390812437311936,
+ "loss": 1.5729,
+ "theoretical_loss": 4.205593369092421,
+ "tokens_seen": 281804800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390651955867603,
+ "loss": 1.6341,
+ "theoretical_loss": 4.205478245562386,
+ "tokens_seen": 281870336
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000739049147442327,
+ "loss": 1.6788,
+ "theoretical_loss": 4.205363156288615,
+ "tokens_seen": 281935872
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390330992978937,
+ "loss": 1.621,
+ "theoretical_loss": 4.205248101252957,
+ "tokens_seen": 282001408
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390170511534604,
+ "loss": 1.544,
+ "theoretical_loss": 4.205133080437273,
+ "tokens_seen": 282066944
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007390010030090271,
+ "loss": 1.6125,
+ "theoretical_loss": 4.205018093823437,
+ "tokens_seen": 282132480
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007389849548645938,
+ "loss": 1.5787,
+ "theoretical_loss": 4.204903141393338,
+ "tokens_seen": 282198016
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007389689067201605,
+ "loss": 1.6345,
+ "theoretical_loss": 4.204788223128879,
+ "tokens_seen": 282263552
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007389528585757272,
+ "loss": 1.5378,
+ "theoretical_loss": 4.204673339011976,
+ "tokens_seen": 282329088
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738936810431294,
+ "loss": 1.6958,
+ "theoretical_loss": 4.204558489024561,
+ "tokens_seen": 282394624
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007389207622868606,
+ "loss": 1.5529,
+ "theoretical_loss": 4.204443673148575,
+ "tokens_seen": 282460160
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007389047141424273,
+ "loss": 1.4651,
+ "theoretical_loss": 4.204328891365977,
+ "tokens_seen": 282525696
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738888665997994,
+ "loss": 1.551,
+ "theoretical_loss": 4.2042141436587395,
+ "tokens_seen": 282591232
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007388726178535607,
+ "loss": 1.5604,
+ "theoretical_loss": 4.204099430008846,
+ "tokens_seen": 282656768
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007388565697091274,
+ "loss": 1.5569,
+ "theoretical_loss": 4.203984750398296,
+ "tokens_seen": 282722304
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007388405215646942,
+ "loss": 1.6475,
+ "theoretical_loss": 4.203870104809101,
+ "tokens_seen": 282787840
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007388244734202608,
+ "loss": 1.5684,
+ "theoretical_loss": 4.203755493223289,
+ "tokens_seen": 282853376
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007388084252758276,
+ "loss": 1.7048,
+ "theoretical_loss": 4.203640915622898,
+ "tokens_seen": 282918912
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007387923771313942,
+ "loss": 1.6332,
+ "theoretical_loss": 4.203526371989983,
+ "tokens_seen": 282984448
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738776328986961,
+ "loss": 1.6873,
+ "theoretical_loss": 4.203411862306609,
+ "tokens_seen": 283049984
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007387602808425276,
+ "loss": 1.6564,
+ "theoretical_loss": 4.203297386554859,
+ "tokens_seen": 283115520
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007387442326980944,
+ "loss": 1.6088,
+ "theoretical_loss": 4.203182944716826,
+ "tokens_seen": 283181056
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007387281845536611,
+ "loss": 1.6262,
+ "theoretical_loss": 4.203068536774619,
+ "tokens_seen": 283246592
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007387121364092278,
+ "loss": 1.5406,
+ "theoretical_loss": 4.202954162710357,
+ "tokens_seen": 283312128
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007386960882647945,
+ "loss": 1.731,
+ "theoretical_loss": 4.202839822506178,
+ "tokens_seen": 283377664
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007386800401203612,
+ "loss": 1.6566,
+ "theoretical_loss": 4.202725516144228,
+ "tokens_seen": 283443200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007386639919759279,
+ "loss": 1.6085,
+ "theoretical_loss": 4.202611243606671,
+ "tokens_seen": 283508736
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007386479438314946,
+ "loss": 1.7107,
+ "theoretical_loss": 4.202497004875681,
+ "tokens_seen": 283574272
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007386318956870613,
+ "loss": 1.6096,
+ "theoretical_loss": 4.202382799933449,
+ "tokens_seen": 283639808
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738615847542628,
+ "loss": 1.4388,
+ "theoretical_loss": 4.2022686287621775,
+ "tokens_seen": 283705344
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385997993981946,
+ "loss": 1.588,
+ "theoretical_loss": 4.20215449134408,
+ "tokens_seen": 283770880
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385837512537614,
+ "loss": 1.634,
+ "theoretical_loss": 4.202040387661389,
+ "tokens_seen": 283836416
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738567703109328,
+ "loss": 1.6546,
+ "theoretical_loss": 4.201926317696347,
+ "tokens_seen": 283901952
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385516549648948,
+ "loss": 1.656,
+ "theoretical_loss": 4.201812281431209,
+ "tokens_seen": 283967488
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385356068204614,
+ "loss": 1.6949,
+ "theoretical_loss": 4.201698278848246,
+ "tokens_seen": 284033024
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385195586760282,
+ "loss": 1.5907,
+ "theoretical_loss": 4.201584309929743,
+ "tokens_seen": 284098560
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007385035105315948,
+ "loss": 1.6153,
+ "theoretical_loss": 4.201470374657993,
+ "tokens_seen": 284164096
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007384874623871615,
+ "loss": 1.6676,
+ "theoretical_loss": 4.201356473015308,
+ "tokens_seen": 284229632
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007384714142427282,
+ "loss": 1.6801,
+ "theoretical_loss": 4.201242604984013,
+ "tokens_seen": 284295168
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007384553660982949,
+ "loss": 1.7536,
+ "theoretical_loss": 4.201128770546442,
+ "tokens_seen": 284360704
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007384393179538616,
+ "loss": 1.6957,
+ "theoretical_loss": 4.201014969684948,
+ "tokens_seen": 284426240
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007384232698094283,
+ "loss": 1.6393,
+ "theoretical_loss": 4.200901202381893,
+ "tokens_seen": 284491776
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738407221664995,
+ "loss": 1.6103,
+ "theoretical_loss": 4.200787468619653,
+ "tokens_seen": 284557312
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383911735205617,
+ "loss": 1.6311,
+ "theoretical_loss": 4.200673768380621,
+ "tokens_seen": 284622848
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383751253761284,
+ "loss": 1.6207,
+ "theoretical_loss": 4.200560101647198,
+ "tokens_seen": 284688384
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383590772316951,
+ "loss": 1.5322,
+ "theoretical_loss": 4.2004464684018,
+ "tokens_seen": 284753920
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383430290872618,
+ "loss": 1.5648,
+ "theoretical_loss": 4.200332868626861,
+ "tokens_seen": 284819456
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383269809428285,
+ "loss": 1.4811,
+ "theoretical_loss": 4.200219302304821,
+ "tokens_seen": 284884992
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007383109327983952,
+ "loss": 1.5884,
+ "theoretical_loss": 4.200105769418138,
+ "tokens_seen": 284950528
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007382948846539619,
+ "loss": 1.6323,
+ "theoretical_loss": 4.199992269949281,
+ "tokens_seen": 285016064
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 101193,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5234239101409912,
+ "objective/train/theoretical_loss": 4.199878803880733,
+ "objective/train/tokens_used": 305541600,
+ "theoretical_loss": 4.199878803880733,
+ "tokens_seen": 285081600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007382788365095286,
+ "loss": 1.4773,
+ "theoretical_loss": 4.199878803880733,
+ "tokens_seen": 285081600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007382627883650953,
+ "loss": 1.4435,
+ "theoretical_loss": 4.1997653711949905,
+ "tokens_seen": 285147136
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738246740220662,
+ "loss": 1.5471,
+ "theoretical_loss": 4.199651971874562,
+ "tokens_seen": 285212672
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007382306920762287,
+ "loss": 1.5007,
+ "theoretical_loss": 4.199538605901973,
+ "tokens_seen": 285278208
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007382146439317954,
+ "loss": 1.6037,
+ "theoretical_loss": 4.199425273259756,
+ "tokens_seen": 285343744
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381985957873621,
+ "loss": 1.5669,
+ "theoretical_loss": 4.199311973930461,
+ "tokens_seen": 285409280
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381825476429288,
+ "loss": 1.5726,
+ "theoretical_loss": 4.199198707896651,
+ "tokens_seen": 285474816
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381664994984955,
+ "loss": 1.5804,
+ "theoretical_loss": 4.199085475140899,
+ "tokens_seen": 285540352
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381504513540622,
+ "loss": 1.6743,
+ "theoretical_loss": 4.1989722756457954,
+ "tokens_seen": 285605888
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381344032096289,
+ "loss": 1.5639,
+ "theoretical_loss": 4.198859109393941,
+ "tokens_seen": 285671424
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381183550651956,
+ "loss": 1.5105,
+ "theoretical_loss": 4.198745976367949,
+ "tokens_seen": 285736960
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007381023069207623,
+ "loss": 1.5629,
+ "theoretical_loss": 4.198632876550449,
+ "tokens_seen": 285802496
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000738086258776329,
+ "loss": 1.5286,
+ "theoretical_loss": 4.19851980992408,
+ "tokens_seen": 285868032
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007380702106318957,
+ "loss": 1.6526,
+ "theoretical_loss": 4.198406776471497,
+ "tokens_seen": 285933568
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007380541624874624,
+ "loss": 1.7248,
+ "theoretical_loss": 4.198293776175365,
+ "tokens_seen": 285999104
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007380381143430291,
+ "loss": 1.4311,
+ "theoretical_loss": 4.198180809018366,
+ "tokens_seen": 286064640
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007380220661985958,
+ "loss": 1.5445,
+ "theoretical_loss": 4.198067874983192,
+ "tokens_seen": 286130176
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007380060180541625,
+ "loss": 1.58,
+ "theoretical_loss": 4.197954974052546,
+ "tokens_seen": 286195712
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007379899699097292,
+ "loss": 1.5693,
+ "theoretical_loss": 4.19784210620915,
+ "tokens_seen": 286261248
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007379739217652959,
+ "loss": 1.6604,
+ "theoretical_loss": 4.197729271435736,
+ "tokens_seen": 286326784
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007379578736208626,
+ "loss": 1.6563,
+ "theoretical_loss": 4.197616469715046,
+ "tokens_seen": 286392320
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007379418254764293,
+ "loss": 1.75,
+ "theoretical_loss": 4.19750370102984,
+ "tokens_seen": 286457856
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000737925777331996,
+ "loss": 1.7021,
+ "theoretical_loss": 4.197390965362887,
+ "tokens_seen": 286523392
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007379097291875627,
+ "loss": 1.567,
+ "theoretical_loss": 4.197278262696971,
+ "tokens_seen": 286588928
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378936810431294,
+ "loss": 1.5716,
+ "theoretical_loss": 4.197165593014889,
+ "tokens_seen": 286654464
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378776328986961,
+ "loss": 1.6708,
+ "theoretical_loss": 4.197052956299449,
+ "tokens_seen": 286720000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378615847542628,
+ "loss": 1.5052,
+ "theoretical_loss": 4.196940352533474,
+ "tokens_seen": 286785536
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378455366098295,
+ "loss": 1.5812,
+ "theoretical_loss": 4.196827781699799,
+ "tokens_seen": 286851072
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378294884653962,
+ "loss": 1.6132,
+ "theoretical_loss": 4.196715243781273,
+ "tokens_seen": 286916608
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007378134403209629,
+ "loss": 1.6742,
+ "theoretical_loss": 4.196602738760754,
+ "tokens_seen": 286982144
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377973921765296,
+ "loss": 1.6104,
+ "theoretical_loss": 4.196490266621119,
+ "tokens_seen": 287047680
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377813440320963,
+ "loss": 1.6694,
+ "theoretical_loss": 4.196377827345252,
+ "tokens_seen": 287113216
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000737765295887663,
+ "loss": 1.5387,
+ "theoretical_loss": 4.196265420916053,
+ "tokens_seen": 287178752
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377492477432297,
+ "loss": 1.6054,
+ "theoretical_loss": 4.196153047316433,
+ "tokens_seen": 287244288
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377331995987964,
+ "loss": 1.5317,
+ "theoretical_loss": 4.196040706529319,
+ "tokens_seen": 287309824
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377171514543631,
+ "loss": 1.6415,
+ "theoretical_loss": 4.195928398537647,
+ "tokens_seen": 287375360
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007377011033099298,
+ "loss": 1.6341,
+ "theoretical_loss": 4.195816123324367,
+ "tokens_seen": 287440896
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007376850551654965,
+ "loss": 1.6622,
+ "theoretical_loss": 4.195703880872443,
+ "tokens_seen": 287506432
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007376690070210632,
+ "loss": 1.5524,
+ "theoretical_loss": 4.195591671164852,
+ "tokens_seen": 287571968
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007376529588766299,
+ "loss": 1.5814,
+ "theoretical_loss": 4.195479494184581,
+ "tokens_seen": 287637504
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007376369107321966,
+ "loss": 1.5762,
+ "theoretical_loss": 4.195367349914631,
+ "tokens_seen": 287703040
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007376208625877633,
+ "loss": 1.5798,
+ "theoretical_loss": 4.195255238338016,
+ "tokens_seen": 287768576
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.00073760481444333,
+ "loss": 1.5229,
+ "theoretical_loss": 4.195143159437764,
+ "tokens_seen": 287834112
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375887662988967,
+ "loss": 1.5433,
+ "theoretical_loss": 4.195031113196913,
+ "tokens_seen": 287899648
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375727181544634,
+ "loss": 1.6659,
+ "theoretical_loss": 4.194919099598516,
+ "tokens_seen": 287965184
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375566700100301,
+ "loss": 1.5775,
+ "theoretical_loss": 4.194807118625636,
+ "tokens_seen": 288030720
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375406218655968,
+ "loss": 1.6525,
+ "theoretical_loss": 4.194695170261353,
+ "tokens_seen": 288096256
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375245737211636,
+ "loss": 1.5722,
+ "theoretical_loss": 4.194583254488754,
+ "tokens_seen": 288161792
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007375085255767302,
+ "loss": 1.5539,
+ "theoretical_loss": 4.194471371290944,
+ "tokens_seen": 288227328
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000737492477432297,
+ "loss": 1.6119,
+ "theoretical_loss": 4.194359520651036,
+ "tokens_seen": 288292864
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 102045,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5714504718780518,
+ "objective/train/theoretical_loss": 4.1942477025521585,
+ "objective/train/tokens_used": 308818400,
+ "theoretical_loss": 4.1942477025521585,
+ "tokens_seen": 288358400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007374764292878637,
+ "loss": 1.5811,
+ "theoretical_loss": 4.1942477025521585,
+ "tokens_seen": 288358400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007374603811434304,
+ "loss": 1.6291,
+ "theoretical_loss": 4.194135916977452,
+ "tokens_seen": 288423936
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000737444332998997,
+ "loss": 1.6362,
+ "theoretical_loss": 4.19402416391007,
+ "tokens_seen": 288489472
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007374282848545638,
+ "loss": 1.5195,
+ "theoretical_loss": 4.193912443333177,
+ "tokens_seen": 288555008
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007374122367101305,
+ "loss": 1.474,
+ "theoretical_loss": 4.193800755229951,
+ "tokens_seen": 288620544
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007373961885656972,
+ "loss": 1.6996,
+ "theoretical_loss": 4.193689099583582,
+ "tokens_seen": 288686080
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007373801404212639,
+ "loss": 1.5586,
+ "theoretical_loss": 4.193577476377275,
+ "tokens_seen": 288751616
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007373640922768306,
+ "loss": 1.5385,
+ "theoretical_loss": 4.193465885594242,
+ "tokens_seen": 288817152
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007373480441323973,
+ "loss": 1.5508,
+ "theoretical_loss": 4.1933543272177145,
+ "tokens_seen": 288882688
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000737331995987964,
+ "loss": 1.6673,
+ "theoretical_loss": 4.193242801230931,
+ "tokens_seen": 288948224
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007373159478435307,
+ "loss": 1.6181,
+ "theoretical_loss": 4.193131307617145,
+ "tokens_seen": 289013760
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372998996990974,
+ "loss": 1.6803,
+ "theoretical_loss": 4.193019846359622,
+ "tokens_seen": 289079296
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372838515546641,
+ "loss": 1.6104,
+ "theoretical_loss": 4.192908417441639,
+ "tokens_seen": 289144832
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372678034102308,
+ "loss": 1.6341,
+ "theoretical_loss": 4.192797020846487,
+ "tokens_seen": 289210368
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372517552657975,
+ "loss": 1.5669,
+ "theoretical_loss": 4.192685656557468,
+ "tokens_seen": 289275904
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372357071213642,
+ "loss": 1.6607,
+ "theoretical_loss": 4.1925743245578975,
+ "tokens_seen": 289341440
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372196589769309,
+ "loss": 1.5765,
+ "theoretical_loss": 4.1924630248311034,
+ "tokens_seen": 289406976
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007372036108324975,
+ "loss": 1.48,
+ "theoretical_loss": 4.192351757360425,
+ "tokens_seen": 289472512
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371875626880643,
+ "loss": 1.5329,
+ "theoretical_loss": 4.1922405221292145,
+ "tokens_seen": 289538048
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371715145436309,
+ "loss": 1.4819,
+ "theoretical_loss": 4.192129319120838,
+ "tokens_seen": 289603584
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371554663991977,
+ "loss": 1.6302,
+ "theoretical_loss": 4.1920181483186685,
+ "tokens_seen": 289669120
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371394182547643,
+ "loss": 1.5838,
+ "theoretical_loss": 4.1919070097060995,
+ "tokens_seen": 289734656
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371233701103311,
+ "loss": 1.5787,
+ "theoretical_loss": 4.19179590326653,
+ "tokens_seen": 289800192
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007371073219658977,
+ "loss": 1.5751,
+ "theoretical_loss": 4.191684828983375,
+ "tokens_seen": 289865728
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370912738214645,
+ "loss": 1.664,
+ "theoretical_loss": 4.191573786840061,
+ "tokens_seen": 289931264
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370752256770311,
+ "loss": 1.5749,
+ "theoretical_loss": 4.191462776820025,
+ "tokens_seen": 289996800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370591775325978,
+ "loss": 1.604,
+ "theoretical_loss": 4.191351798906719,
+ "tokens_seen": 290062336
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370431293881645,
+ "loss": 1.7533,
+ "theoretical_loss": 4.191240853083604,
+ "tokens_seen": 290127872
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370270812437312,
+ "loss": 1.6221,
+ "theoretical_loss": 4.191129939334159,
+ "tokens_seen": 290193408
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007370110330992979,
+ "loss": 1.5681,
+ "theoretical_loss": 4.191019057641868,
+ "tokens_seen": 290258944
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007369949849548646,
+ "loss": 1.6289,
+ "theoretical_loss": 4.1909082079902324,
+ "tokens_seen": 290324480
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007369789368104313,
+ "loss": 1.7017,
+ "theoretical_loss": 4.190797390362764,
+ "tokens_seen": 290390016
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000736962888665998,
+ "loss": 1.5707,
+ "theoretical_loss": 4.190686604742986,
+ "tokens_seen": 290455552
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007369468405215647,
+ "loss": 1.6355,
+ "theoretical_loss": 4.190575851114437,
+ "tokens_seen": 290521088
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007369307923771314,
+ "loss": 1.7072,
+ "theoretical_loss": 4.190465129460662,
+ "tokens_seen": 290586624
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007369147442326981,
+ "loss": 1.6748,
+ "theoretical_loss": 4.190354439765224,
+ "tokens_seen": 290652160
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368986960882648,
+ "loss": 1.5803,
+ "theoretical_loss": 4.190243782011696,
+ "tokens_seen": 290717696
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368826479438315,
+ "loss": 1.6931,
+ "theoretical_loss": 4.190133156183663,
+ "tokens_seen": 290783232
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368665997993982,
+ "loss": 1.5436,
+ "theoretical_loss": 4.190022562264721,
+ "tokens_seen": 290848768
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368505516549649,
+ "loss": 1.6706,
+ "theoretical_loss": 4.189912000238479,
+ "tokens_seen": 290914304
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368345035105316,
+ "loss": 1.5295,
+ "theoretical_loss": 4.189801470088559,
+ "tokens_seen": 290979840
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007368184553660983,
+ "loss": 1.6353,
+ "theoretical_loss": 4.189690971798596,
+ "tokens_seen": 291045376
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000736802407221665,
+ "loss": 1.6281,
+ "theoretical_loss": 4.189580505352232,
+ "tokens_seen": 291110912
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367863590772317,
+ "loss": 1.6095,
+ "theoretical_loss": 4.189470070733129,
+ "tokens_seen": 291176448
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367703109327984,
+ "loss": 1.6304,
+ "theoretical_loss": 4.189359667924952,
+ "tokens_seen": 291241984
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367542627883651,
+ "loss": 1.6004,
+ "theoretical_loss": 4.189249296911386,
+ "tokens_seen": 291307520
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367382146439318,
+ "loss": 1.6946,
+ "theoretical_loss": 4.189138957676125,
+ "tokens_seen": 291373056
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367221664994985,
+ "loss": 1.7297,
+ "theoretical_loss": 4.1890286502028715,
+ "tokens_seen": 291438592
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007367061183550652,
+ "loss": 1.6308,
+ "theoretical_loss": 4.188918374475347,
+ "tokens_seen": 291504128
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007366900702106319,
+ "loss": 1.6745,
+ "theoretical_loss": 4.188808130477279,
+ "tokens_seen": 291569664
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 103221,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6348395347595215,
+ "objective/train/theoretical_loss": 4.1886979181924096,
+ "objective/train/tokens_used": 312095200,
+ "theoretical_loss": 4.1886979181924096,
+ "tokens_seen": 291635200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007366740220661986,
+ "loss": 1.647,
+ "theoretical_loss": 4.1886979181924096,
+ "tokens_seen": 291635200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007366579739217653,
+ "loss": 1.7113,
+ "theoretical_loss": 4.188587737604493,
+ "tokens_seen": 291700736
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000736641925777332,
+ "loss": 1.6427,
+ "theoretical_loss": 4.188477588697295,
+ "tokens_seen": 291766272
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007366258776328987,
+ "loss": 1.5969,
+ "theoretical_loss": 4.188367471454593,
+ "tokens_seen": 291831808
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007366098294884654,
+ "loss": 1.4803,
+ "theoretical_loss": 4.188257385860177,
+ "tokens_seen": 291897344
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365937813440321,
+ "loss": 1.6042,
+ "theoretical_loss": 4.188147331897849,
+ "tokens_seen": 291962880
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365777331995988,
+ "loss": 1.6471,
+ "theoretical_loss": 4.1880373095514205,
+ "tokens_seen": 292028416
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365616850551655,
+ "loss": 1.6034,
+ "theoretical_loss": 4.18792731880472,
+ "tokens_seen": 292093952
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365456369107322,
+ "loss": 1.5614,
+ "theoretical_loss": 4.187817359641581,
+ "tokens_seen": 292159488
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365295887662989,
+ "loss": 1.5428,
+ "theoretical_loss": 4.1877074320458565,
+ "tokens_seen": 292225024
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007365135406218656,
+ "loss": 1.5486,
+ "theoretical_loss": 4.187597536001406,
+ "tokens_seen": 292290560
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364974924774323,
+ "loss": 1.5498,
+ "theoretical_loss": 4.187487671492102,
+ "tokens_seen": 292356096
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000736481444332999,
+ "loss": 1.6196,
+ "theoretical_loss": 4.18737783850183,
+ "tokens_seen": 292421632
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364653961885657,
+ "loss": 1.6716,
+ "theoretical_loss": 4.187268037014486,
+ "tokens_seen": 292487168
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364493480441324,
+ "loss": 1.6067,
+ "theoretical_loss": 4.187158267013979,
+ "tokens_seen": 292552704
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364332998996991,
+ "loss": 1.517,
+ "theoretical_loss": 4.18704852848423,
+ "tokens_seen": 292618240
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364172517552658,
+ "loss": 1.6312,
+ "theoretical_loss": 4.18693882140917,
+ "tokens_seen": 292683776
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007364012036108325,
+ "loss": 1.4959,
+ "theoretical_loss": 4.186829145772743,
+ "tokens_seen": 292749312
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363851554663992,
+ "loss": 1.5707,
+ "theoretical_loss": 4.186719501558905,
+ "tokens_seen": 292814848
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363691073219659,
+ "loss": 1.5797,
+ "theoretical_loss": 4.186609888751623,
+ "tokens_seen": 292880384
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363530591775326,
+ "loss": 1.6421,
+ "theoretical_loss": 4.186500307334878,
+ "tokens_seen": 292945920
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363370110330993,
+ "loss": 1.6258,
+ "theoretical_loss": 4.186390757292659,
+ "tokens_seen": 293011456
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363209628886661,
+ "loss": 1.5723,
+ "theoretical_loss": 4.18628123860897,
+ "tokens_seen": 293076992
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007363049147442327,
+ "loss": 1.638,
+ "theoretical_loss": 4.186171751267825,
+ "tokens_seen": 293142528
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007362888665997995,
+ "loss": 1.6085,
+ "theoretical_loss": 4.186062295253249,
+ "tokens_seen": 293208064
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007362728184553661,
+ "loss": 1.6619,
+ "theoretical_loss": 4.185952870549283,
+ "tokens_seen": 293273600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007362567703109329,
+ "loss": 1.5903,
+ "theoretical_loss": 4.185843477139974,
+ "tokens_seen": 293339136
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007362407221664995,
+ "loss": 1.555,
+ "theoretical_loss": 4.185734115009383,
+ "tokens_seen": 293404672
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007362246740220663,
+ "loss": 1.5025,
+ "theoretical_loss": 4.185624784141585,
+ "tokens_seen": 293470208
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000736208625877633,
+ "loss": 1.5115,
+ "theoretical_loss": 4.185515484520664,
+ "tokens_seen": 293535744
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361925777331997,
+ "loss": 1.6593,
+ "theoretical_loss": 4.1854062161307155,
+ "tokens_seen": 293601280
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361765295887663,
+ "loss": 1.7188,
+ "theoretical_loss": 4.185296978955848,
+ "tokens_seen": 293666816
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361604814443331,
+ "loss": 1.5893,
+ "theoretical_loss": 4.185187772980181,
+ "tokens_seen": 293732352
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361444332998997,
+ "loss": 1.4278,
+ "theoretical_loss": 4.185078598187846,
+ "tokens_seen": 293797888
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361283851554665,
+ "loss": 1.4631,
+ "theoretical_loss": 4.184969454562985,
+ "tokens_seen": 293863424
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007361123370110332,
+ "loss": 1.5732,
+ "theoretical_loss": 4.1848603420897525,
+ "tokens_seen": 293928960
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360962888665999,
+ "loss": 1.5025,
+ "theoretical_loss": 4.184751260752316,
+ "tokens_seen": 293994496
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360802407221666,
+ "loss": 1.5542,
+ "theoretical_loss": 4.18464221053485,
+ "tokens_seen": 294060032
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360641925777333,
+ "loss": 1.6657,
+ "theoretical_loss": 4.184533191421547,
+ "tokens_seen": 294125568
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360481444333,
+ "loss": 1.7267,
+ "theoretical_loss": 4.184424203396606,
+ "tokens_seen": 294191104
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360320962888667,
+ "loss": 1.4338,
+ "theoretical_loss": 4.184315246444239,
+ "tokens_seen": 294256640
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360160481444334,
+ "loss": 1.5626,
+ "theoretical_loss": 4.18420632054867,
+ "tokens_seen": 294322176
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007360000000000001,
+ "loss": 1.6043,
+ "theoretical_loss": 4.1840974256941355,
+ "tokens_seen": 294387712
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359839518555668,
+ "loss": 1.5946,
+ "theoretical_loss": 4.183988561864879,
+ "tokens_seen": 294453248
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359679037111335,
+ "loss": 1.5876,
+ "theoretical_loss": 4.183879729045163,
+ "tokens_seen": 294518784
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359518555667002,
+ "loss": 1.6075,
+ "theoretical_loss": 4.183770927219255,
+ "tokens_seen": 294584320
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359358074222669,
+ "loss": 1.5569,
+ "theoretical_loss": 4.183662156371436,
+ "tokens_seen": 294649856
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359197592778336,
+ "loss": 1.6115,
+ "theoretical_loss": 4.183553416485999,
+ "tokens_seen": 294715392
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007359037111334003,
+ "loss": 1.5446,
+ "theoretical_loss": 4.183444707547249,
+ "tokens_seen": 294780928
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735887662988967,
+ "loss": 1.6602,
+ "theoretical_loss": 4.1833360295395,
+ "tokens_seen": 294846464
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 104159,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.72190523147583,
+ "objective/train/theoretical_loss": 4.183227382447081,
+ "objective/train/tokens_used": 315372000,
+ "theoretical_loss": 4.183227382447081,
+ "tokens_seen": 294912000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007358716148445337,
+ "loss": 1.6478,
+ "theoretical_loss": 4.183227382447081,
+ "tokens_seen": 294912000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007358555667001004,
+ "loss": 1.6181,
+ "theoretical_loss": 4.183118766254328,
+ "tokens_seen": 294977536
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007358395185556671,
+ "loss": 1.6082,
+ "theoretical_loss": 4.183010180945593,
+ "tokens_seen": 295043072
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007358234704112338,
+ "loss": 1.5291,
+ "theoretical_loss": 4.182901626505237,
+ "tokens_seen": 295108608
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007358074222668004,
+ "loss": 1.6135,
+ "theoretical_loss": 4.182793102917632,
+ "tokens_seen": 295174144
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007357913741223672,
+ "loss": 1.5915,
+ "theoretical_loss": 4.182684610167162,
+ "tokens_seen": 295239680
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007357753259779338,
+ "loss": 1.6561,
+ "theoretical_loss": 4.1825761482382235,
+ "tokens_seen": 295305216
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007357592778335006,
+ "loss": 1.7016,
+ "theoretical_loss": 4.182467717115221,
+ "tokens_seen": 295370752
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007357432296890672,
+ "loss": 1.6305,
+ "theoretical_loss": 4.182359316782576,
+ "tokens_seen": 295436288
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735727181544634,
+ "loss": 1.5485,
+ "theoretical_loss": 4.1822509472247145,
+ "tokens_seen": 295501824
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007357111334002006,
+ "loss": 1.6077,
+ "theoretical_loss": 4.18214260842608,
+ "tokens_seen": 295567360
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007356950852557674,
+ "loss": 1.5885,
+ "theoretical_loss": 4.182034300371122,
+ "tokens_seen": 295632896
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735679037111334,
+ "loss": 1.564,
+ "theoretical_loss": 4.181926023044306,
+ "tokens_seen": 295698432
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007356629889669007,
+ "loss": 1.5082,
+ "theoretical_loss": 4.181817776430106,
+ "tokens_seen": 295763968
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007356469408224674,
+ "loss": 1.711,
+ "theoretical_loss": 4.181709560513008,
+ "tokens_seen": 295829504
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007356308926780341,
+ "loss": 1.616,
+ "theoretical_loss": 4.181601375277509,
+ "tokens_seen": 295895040
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007356148445336008,
+ "loss": 1.6577,
+ "theoretical_loss": 4.181493220708117,
+ "tokens_seen": 295960576
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355987963891675,
+ "loss": 1.6458,
+ "theoretical_loss": 4.181385096789353,
+ "tokens_seen": 296026112
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355827482447342,
+ "loss": 1.6822,
+ "theoretical_loss": 4.181277003505747,
+ "tokens_seen": 296091648
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355667001003009,
+ "loss": 1.5702,
+ "theoretical_loss": 4.181168940841843,
+ "tokens_seen": 296157184
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355506519558676,
+ "loss": 1.6557,
+ "theoretical_loss": 4.181060908782191,
+ "tokens_seen": 296222720
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355346038114343,
+ "loss": 1.5546,
+ "theoretical_loss": 4.18095290731136,
+ "tokens_seen": 296288256
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735518555667001,
+ "loss": 1.53,
+ "theoretical_loss": 4.180844936413922,
+ "tokens_seen": 296353792
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007355025075225677,
+ "loss": 1.562,
+ "theoretical_loss": 4.180736996074465,
+ "tokens_seen": 296419328
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354864593781344,
+ "loss": 1.7042,
+ "theoretical_loss": 4.180629086277589,
+ "tokens_seen": 296484864
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354704112337011,
+ "loss": 1.5895,
+ "theoretical_loss": 4.180521207007902,
+ "tokens_seen": 296550400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354543630892678,
+ "loss": 1.6513,
+ "theoretical_loss": 4.180413358250024,
+ "tokens_seen": 296615936
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354383149448345,
+ "loss": 1.6866,
+ "theoretical_loss": 4.180305539988588,
+ "tokens_seen": 296681472
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354222668004012,
+ "loss": 1.5783,
+ "theoretical_loss": 4.180197752208235,
+ "tokens_seen": 296747008
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007354062186559679,
+ "loss": 1.6043,
+ "theoretical_loss": 4.1800899948936205,
+ "tokens_seen": 296812544
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007353901705115346,
+ "loss": 1.565,
+ "theoretical_loss": 4.179982268029409,
+ "tokens_seen": 296878080
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007353741223671013,
+ "loss": 1.5138,
+ "theoretical_loss": 4.179874571600277,
+ "tokens_seen": 296943616
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735358074222668,
+ "loss": 1.6457,
+ "theoretical_loss": 4.1797669055909115,
+ "tokens_seen": 297009152
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007353420260782347,
+ "loss": 1.6078,
+ "theoretical_loss": 4.1796592699860105,
+ "tokens_seen": 297074688
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007353259779338014,
+ "loss": 1.691,
+ "theoretical_loss": 4.179551664770283,
+ "tokens_seen": 297140224
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007353099297893681,
+ "loss": 1.5473,
+ "theoretical_loss": 4.1794440899284515,
+ "tokens_seen": 297205760
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352938816449348,
+ "loss": 1.5952,
+ "theoretical_loss": 4.179336545445245,
+ "tokens_seen": 297271296
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352778335005015,
+ "loss": 1.5846,
+ "theoretical_loss": 4.1792290313054075,
+ "tokens_seen": 297336832
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352617853560682,
+ "loss": 1.5803,
+ "theoretical_loss": 4.179121547493692,
+ "tokens_seen": 297402368
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352457372116349,
+ "loss": 1.6163,
+ "theoretical_loss": 4.179014093994862,
+ "tokens_seen": 297467904
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352296890672016,
+ "loss": 1.59,
+ "theoretical_loss": 4.178906670793695,
+ "tokens_seen": 297533440
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007352136409227683,
+ "loss": 1.6357,
+ "theoretical_loss": 4.178799277874977,
+ "tokens_seen": 297598976
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735197592778335,
+ "loss": 1.6383,
+ "theoretical_loss": 4.178691915223505,
+ "tokens_seen": 297664512
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351815446339017,
+ "loss": 1.5541,
+ "theoretical_loss": 4.178584582824088,
+ "tokens_seen": 297730048
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351654964894684,
+ "loss": 1.7037,
+ "theoretical_loss": 4.1784772806615464,
+ "tokens_seen": 297795584
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351494483450351,
+ "loss": 1.6525,
+ "theoretical_loss": 4.178370008720709,
+ "tokens_seen": 297861120
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351334002006018,
+ "loss": 1.5659,
+ "theoretical_loss": 4.178262766986418,
+ "tokens_seen": 297926656
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351173520561686,
+ "loss": 1.5114,
+ "theoretical_loss": 4.178155555443525,
+ "tokens_seen": 297992192
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007351013039117352,
+ "loss": 1.7272,
+ "theoretical_loss": 4.178048374076894,
+ "tokens_seen": 298057728
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735085255767302,
+ "loss": 1.6131,
+ "theoretical_loss": 4.1779412228714,
+ "tokens_seen": 298123264
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 105423,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5215253829956055,
+ "objective/train/theoretical_loss": 4.177834101811927,
+ "objective/train/tokens_used": 318648800,
+ "theoretical_loss": 4.177834101811927,
+ "tokens_seen": 298188800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007350692076228686,
+ "loss": 1.5549,
+ "theoretical_loss": 4.177834101811927,
+ "tokens_seen": 298188800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007350531594784354,
+ "loss": 1.4696,
+ "theoretical_loss": 4.1777270108833715,
+ "tokens_seen": 298254336
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000735037111334002,
+ "loss": 1.6553,
+ "theoretical_loss": 4.177619950070639,
+ "tokens_seen": 298319872
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007350210631895688,
+ "loss": 1.6148,
+ "theoretical_loss": 4.177512919358649,
+ "tokens_seen": 298385408
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007350050150451354,
+ "loss": 1.6731,
+ "theoretical_loss": 4.17740591873233,
+ "tokens_seen": 298450944
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007349889669007022,
+ "loss": 1.6382,
+ "theoretical_loss": 4.177298948176619,
+ "tokens_seen": 298516480
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007349729187562688,
+ "loss": 1.4472,
+ "theoretical_loss": 4.17719200767647,
+ "tokens_seen": 298582016
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007349568706118356,
+ "loss": 1.6397,
+ "theoretical_loss": 4.177085097216842,
+ "tokens_seen": 298647552
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007349408224674022,
+ "loss": 1.6669,
+ "theoretical_loss": 4.1769782167827065,
+ "tokens_seen": 298713088
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734924774322969,
+ "loss": 1.6121,
+ "theoretical_loss": 4.176871366359047,
+ "tokens_seen": 298778624
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007349087261785356,
+ "loss": 1.4917,
+ "theoretical_loss": 4.176764545930858,
+ "tokens_seen": 298844160
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007348926780341024,
+ "loss": 1.5772,
+ "theoretical_loss": 4.176657755483143,
+ "tokens_seen": 298909696
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734876629889669,
+ "loss": 1.6754,
+ "theoretical_loss": 4.176550995000917,
+ "tokens_seen": 298975232
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007348605817452358,
+ "loss": 1.6027,
+ "theoretical_loss": 4.176444264469206,
+ "tokens_seen": 299040768
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007348445336008024,
+ "loss": 1.7495,
+ "theoretical_loss": 4.176337563873046,
+ "tokens_seen": 299106304
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007348284854563692,
+ "loss": 1.6497,
+ "theoretical_loss": 4.176230893197486,
+ "tokens_seen": 299171840
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007348124373119358,
+ "loss": 1.51,
+ "theoretical_loss": 4.1761242524275834,
+ "tokens_seen": 299237376
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007347963891675026,
+ "loss": 1.5526,
+ "theoretical_loss": 4.176017641548407,
+ "tokens_seen": 299302912
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007347803410230692,
+ "loss": 1.7034,
+ "theoretical_loss": 4.175911060545037,
+ "tokens_seen": 299368448
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734764292878636,
+ "loss": 1.6063,
+ "theoretical_loss": 4.175804509402562,
+ "tokens_seen": 299433984
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007347482447342027,
+ "loss": 1.4916,
+ "theoretical_loss": 4.175697988106085,
+ "tokens_seen": 299499520
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007347321965897694,
+ "loss": 1.7321,
+ "theoretical_loss": 4.175591496640718,
+ "tokens_seen": 299565056
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734716148445336,
+ "loss": 1.5392,
+ "theoretical_loss": 4.175485034991581,
+ "tokens_seen": 299630592
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007347001003009028,
+ "loss": 1.5464,
+ "theoretical_loss": 4.17537860314381,
+ "tokens_seen": 299696128
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007346840521564695,
+ "loss": 1.5023,
+ "theoretical_loss": 4.175272201082547,
+ "tokens_seen": 299761664
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007346680040120362,
+ "loss": 1.5762,
+ "theoretical_loss": 4.175165828792946,
+ "tokens_seen": 299827200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007346519558676029,
+ "loss": 1.6361,
+ "theoretical_loss": 4.1750594862601735,
+ "tokens_seen": 299892736
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007346359077231696,
+ "loss": 1.6891,
+ "theoretical_loss": 4.1749531734694045,
+ "tokens_seen": 299958272
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007346198595787363,
+ "loss": 1.5498,
+ "theoretical_loss": 4.174846890405825,
+ "tokens_seen": 300023808
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734603811434303,
+ "loss": 1.5768,
+ "theoretical_loss": 4.174740637054632,
+ "tokens_seen": 300089344
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345877632898697,
+ "loss": 1.5761,
+ "theoretical_loss": 4.174634413401034,
+ "tokens_seen": 300154880
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345717151454364,
+ "loss": 1.5583,
+ "theoretical_loss": 4.174528219430247,
+ "tokens_seen": 300220416
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345556670010031,
+ "loss": 1.5313,
+ "theoretical_loss": 4.174422055127502,
+ "tokens_seen": 300285952
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345396188565698,
+ "loss": 1.6252,
+ "theoretical_loss": 4.174315920478037,
+ "tokens_seen": 300351488
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345235707121365,
+ "loss": 1.5832,
+ "theoretical_loss": 4.174209815467102,
+ "tokens_seen": 300417024
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007345075225677032,
+ "loss": 1.5806,
+ "theoretical_loss": 4.174103740079958,
+ "tokens_seen": 300482560
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007344914744232699,
+ "loss": 1.6372,
+ "theoretical_loss": 4.1739976943018755,
+ "tokens_seen": 300548096
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007344754262788366,
+ "loss": 1.7141,
+ "theoretical_loss": 4.173891678118135,
+ "tokens_seen": 300613632
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007344593781344033,
+ "loss": 1.592,
+ "theoretical_loss": 4.173785691514031,
+ "tokens_seen": 300679168
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.00073444332998997,
+ "loss": 1.6567,
+ "theoretical_loss": 4.173679734474862,
+ "tokens_seen": 300744704
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007344272818455367,
+ "loss": 1.5908,
+ "theoretical_loss": 4.173573806985945,
+ "tokens_seen": 300810240
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007344112337011033,
+ "loss": 1.6119,
+ "theoretical_loss": 4.173467909032602,
+ "tokens_seen": 300875776
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343951855566701,
+ "loss": 1.6013,
+ "theoretical_loss": 4.173362040600166,
+ "tokens_seen": 300941312
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343791374122367,
+ "loss": 1.6848,
+ "theoretical_loss": 4.173256201673983,
+ "tokens_seen": 301006848
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343630892678035,
+ "loss": 1.5417,
+ "theoretical_loss": 4.173150392239406,
+ "tokens_seen": 301072384
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343470411233701,
+ "loss": 1.5092,
+ "theoretical_loss": 4.173044612281802,
+ "tokens_seen": 301137920
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343309929789369,
+ "loss": 1.5109,
+ "theoretical_loss": 4.172938861786546,
+ "tokens_seen": 301203456
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007343149448345035,
+ "loss": 1.5317,
+ "theoretical_loss": 4.1728331407390264,
+ "tokens_seen": 301268992
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342988966900703,
+ "loss": 1.58,
+ "theoretical_loss": 4.172727449124636,
+ "tokens_seen": 301334528
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342828485456369,
+ "loss": 1.625,
+ "theoretical_loss": 4.172621786928786,
+ "tokens_seen": 301400064
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 106715,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5944786071777344,
+ "objective/train/theoretical_loss": 4.17251615413689,
+ "objective/train/tokens_used": 321925600,
+ "theoretical_loss": 4.17251615413689,
+ "tokens_seen": 301465600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342668004012036,
+ "loss": 1.6601,
+ "theoretical_loss": 4.17251615413689,
+ "tokens_seen": 301465600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342507522567703,
+ "loss": 1.5005,
+ "theoretical_loss": 4.172410550734378,
+ "tokens_seen": 301531136
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734234704112337,
+ "loss": 1.5192,
+ "theoretical_loss": 4.172304976706689,
+ "tokens_seen": 301596672
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342186559679037,
+ "loss": 1.5185,
+ "theoretical_loss": 4.17219943203927,
+ "tokens_seen": 301662208
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007342026078234704,
+ "loss": 1.7129,
+ "theoretical_loss": 4.172093916717581,
+ "tokens_seen": 301727744
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341865596790371,
+ "loss": 1.5986,
+ "theoretical_loss": 4.171988430727091,
+ "tokens_seen": 301793280
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341705115346038,
+ "loss": 1.5291,
+ "theoretical_loss": 4.17188297405328,
+ "tokens_seen": 301858816
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341544633901705,
+ "loss": 1.6157,
+ "theoretical_loss": 4.171777546681638,
+ "tokens_seen": 301924352
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341384152457372,
+ "loss": 1.6012,
+ "theoretical_loss": 4.171672148597665,
+ "tokens_seen": 301989888
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341223671013039,
+ "loss": 1.7156,
+ "theoretical_loss": 4.171566779786872,
+ "tokens_seen": 302055424
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007341063189568706,
+ "loss": 1.5974,
+ "theoretical_loss": 4.17146144023478,
+ "tokens_seen": 302120960
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007340902708124373,
+ "loss": 1.5717,
+ "theoretical_loss": 4.171356129926921,
+ "tokens_seen": 302186496
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000734074222668004,
+ "loss": 1.647,
+ "theoretical_loss": 4.171250848848835,
+ "tokens_seen": 302252032
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007340581745235707,
+ "loss": 1.7013,
+ "theoretical_loss": 4.171145596986076,
+ "tokens_seen": 302317568
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007340421263791374,
+ "loss": 1.5124,
+ "theoretical_loss": 4.171040374324204,
+ "tokens_seen": 302383104
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007340260782347041,
+ "loss": 1.5927,
+ "theoretical_loss": 4.170935180848793,
+ "tokens_seen": 302448640
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007340100300902708,
+ "loss": 1.5633,
+ "theoretical_loss": 4.170830016545425,
+ "tokens_seen": 302514176
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339939819458375,
+ "loss": 1.7269,
+ "theoretical_loss": 4.1707248813996936,
+ "tokens_seen": 302579712
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339779338014042,
+ "loss": 1.6634,
+ "theoretical_loss": 4.170619775397201,
+ "tokens_seen": 302645248
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339618856569709,
+ "loss": 1.7733,
+ "theoretical_loss": 4.1705146985235615,
+ "tokens_seen": 302710784
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339458375125376,
+ "loss": 1.6171,
+ "theoretical_loss": 4.170409650764397,
+ "tokens_seen": 302776320
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339297893681043,
+ "loss": 1.49,
+ "theoretical_loss": 4.170304632105344,
+ "tokens_seen": 302841856
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007339137412236711,
+ "loss": 1.6787,
+ "theoretical_loss": 4.170199642532045,
+ "tokens_seen": 302907392
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338976930792377,
+ "loss": 1.5294,
+ "theoretical_loss": 4.170094682030154,
+ "tokens_seen": 302972928
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338816449348045,
+ "loss": 1.5229,
+ "theoretical_loss": 4.169989750585335,
+ "tokens_seen": 303038464
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338655967903711,
+ "loss": 1.6258,
+ "theoretical_loss": 4.169884848183264,
+ "tokens_seen": 303104000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338495486459379,
+ "loss": 1.5401,
+ "theoretical_loss": 4.169779974809624,
+ "tokens_seen": 303169536
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338335005015045,
+ "loss": 1.6634,
+ "theoretical_loss": 4.169675130450111,
+ "tokens_seen": 303235072
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338174523570713,
+ "loss": 1.5992,
+ "theoretical_loss": 4.16957031509043,
+ "tokens_seen": 303300608
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007338014042126379,
+ "loss": 1.601,
+ "theoretical_loss": 4.169465528716295,
+ "tokens_seen": 303366144
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337853560682047,
+ "loss": 1.4711,
+ "theoretical_loss": 4.169360771313434,
+ "tokens_seen": 303431680
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337693079237713,
+ "loss": 1.7924,
+ "theoretical_loss": 4.169256042867579,
+ "tokens_seen": 303497216
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337532597793381,
+ "loss": 1.6994,
+ "theoretical_loss": 4.169151343364476,
+ "tokens_seen": 303562752
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337372116349047,
+ "loss": 1.5778,
+ "theoretical_loss": 4.169046672789882,
+ "tokens_seen": 303628288
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337211634904715,
+ "loss": 1.6038,
+ "theoretical_loss": 4.168942031129562,
+ "tokens_seen": 303693824
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007337051153460381,
+ "loss": 1.6144,
+ "theoretical_loss": 4.168837418369292,
+ "tokens_seen": 303759360
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336890672016049,
+ "loss": 1.6138,
+ "theoretical_loss": 4.168732834494857,
+ "tokens_seen": 303824896
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336730190571715,
+ "loss": 1.5502,
+ "theoretical_loss": 4.1686282794920535,
+ "tokens_seen": 303890432
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336569709127383,
+ "loss": 1.7256,
+ "theoretical_loss": 4.168523753346687,
+ "tokens_seen": 303955968
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336409227683049,
+ "loss": 1.5377,
+ "theoretical_loss": 4.168419256044573,
+ "tokens_seen": 304021504
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336248746238717,
+ "loss": 1.632,
+ "theoretical_loss": 4.168314787571538,
+ "tokens_seen": 304087040
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007336088264794383,
+ "loss": 1.626,
+ "theoretical_loss": 4.168210347913418,
+ "tokens_seen": 304152576
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335927783350051,
+ "loss": 1.6017,
+ "theoretical_loss": 4.168105937056059,
+ "tokens_seen": 304218112
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335767301905717,
+ "loss": 1.7149,
+ "theoretical_loss": 4.168001554985317,
+ "tokens_seen": 304283648
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335606820461385,
+ "loss": 1.6081,
+ "theoretical_loss": 4.167897201687056,
+ "tokens_seen": 304349184
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335446339017051,
+ "loss": 1.6175,
+ "theoretical_loss": 4.167792877147155,
+ "tokens_seen": 304414720
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335285857572719,
+ "loss": 1.5451,
+ "theoretical_loss": 4.1676885813514986,
+ "tokens_seen": 304480256
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007335125376128385,
+ "loss": 1.6038,
+ "theoretical_loss": 4.167584314285982,
+ "tokens_seen": 304545792
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334964894684053,
+ "loss": 1.5534,
+ "theoretical_loss": 4.167480075936512,
+ "tokens_seen": 304611328
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000733480441323972,
+ "loss": 1.5889,
+ "theoretical_loss": 4.167375866289004,
+ "tokens_seen": 304676864
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 107694,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6512261629104614,
+ "objective/train/theoretical_loss": 4.167271685329383,
+ "objective/train/tokens_used": 325202400,
+ "theoretical_loss": 4.167271685329383,
+ "tokens_seen": 304742400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334643931795387,
+ "loss": 1.5713,
+ "theoretical_loss": 4.167271685329383,
+ "tokens_seen": 304742400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334483450351053,
+ "loss": 1.6476,
+ "theoretical_loss": 4.167167533043585,
+ "tokens_seen": 304807936
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334322968906721,
+ "loss": 1.63,
+ "theoretical_loss": 4.1670634094175565,
+ "tokens_seen": 304873472
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334162487462387,
+ "loss": 1.541,
+ "theoretical_loss": 4.166959314437252,
+ "tokens_seen": 304939008
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007334002006018055,
+ "loss": 1.6109,
+ "theoretical_loss": 4.166855248088637,
+ "tokens_seen": 305004544
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007333841524573721,
+ "loss": 1.6243,
+ "theoretical_loss": 4.1667512103576865,
+ "tokens_seen": 305070080
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007333681043129389,
+ "loss": 1.6751,
+ "theoretical_loss": 4.166647201230386,
+ "tokens_seen": 305135616
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007333520561685056,
+ "loss": 1.6589,
+ "theoretical_loss": 4.166543220692731,
+ "tokens_seen": 305201152
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007333360080240723,
+ "loss": 1.5901,
+ "theoretical_loss": 4.166439268730724,
+ "tokens_seen": 305266688
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000733319959879639,
+ "loss": 1.653,
+ "theoretical_loss": 4.166335345330383,
+ "tokens_seen": 305332224
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007333039117352057,
+ "loss": 1.6598,
+ "theoretical_loss": 4.16623145047773,
+ "tokens_seen": 305397760
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332878635907724,
+ "loss": 1.5896,
+ "theoretical_loss": 4.166127584158801,
+ "tokens_seen": 305463296
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332718154463391,
+ "loss": 1.6481,
+ "theoretical_loss": 4.166023746359639,
+ "tokens_seen": 305528832
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332557673019058,
+ "loss": 1.6581,
+ "theoretical_loss": 4.165919937066299,
+ "tokens_seen": 305594368
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332397191574725,
+ "loss": 1.6001,
+ "theoretical_loss": 4.165816156264844,
+ "tokens_seen": 305659904
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332236710130392,
+ "loss": 1.7288,
+ "theoretical_loss": 4.165712403941348,
+ "tokens_seen": 305725440
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007332076228686059,
+ "loss": 1.5835,
+ "theoretical_loss": 4.1656086800818946,
+ "tokens_seen": 305790976
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007331915747241726,
+ "loss": 1.6093,
+ "theoretical_loss": 4.165504984672577,
+ "tokens_seen": 305856512
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007331755265797393,
+ "loss": 1.4599,
+ "theoretical_loss": 4.165401317699498,
+ "tokens_seen": 305922048
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000733159478435306,
+ "loss": 1.6132,
+ "theoretical_loss": 4.16529767914877,
+ "tokens_seen": 305987584
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007331434302908727,
+ "loss": 1.5185,
+ "theoretical_loss": 4.165194069006516,
+ "tokens_seen": 306053120
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007331273821464394,
+ "loss": 1.6518,
+ "theoretical_loss": 4.165090487258867,
+ "tokens_seen": 306118656
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007331113340020061,
+ "loss": 1.5807,
+ "theoretical_loss": 4.164986933891968,
+ "tokens_seen": 306184192
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330952858575728,
+ "loss": 1.6028,
+ "theoretical_loss": 4.164883408891968,
+ "tokens_seen": 306249728
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330792377131395,
+ "loss": 1.6098,
+ "theoretical_loss": 4.164779912245029,
+ "tokens_seen": 306315264
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330631895687062,
+ "loss": 1.5847,
+ "theoretical_loss": 4.1646764439373225,
+ "tokens_seen": 306380800
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330471414242729,
+ "loss": 1.5105,
+ "theoretical_loss": 4.164573003955029,
+ "tokens_seen": 306446336
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330310932798396,
+ "loss": 1.5924,
+ "theoretical_loss": 4.164469592284338,
+ "tokens_seen": 306511872
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007330150451354062,
+ "loss": 1.7551,
+ "theoretical_loss": 4.164366208911453,
+ "tokens_seen": 306577408
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732998996990973,
+ "loss": 1.7007,
+ "theoretical_loss": 4.1642628538225805,
+ "tokens_seen": 306642944
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007329829488465396,
+ "loss": 1.6358,
+ "theoretical_loss": 4.1641595270039415,
+ "tokens_seen": 306708480
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007329669007021064,
+ "loss": 1.6819,
+ "theoretical_loss": 4.1640562284417655,
+ "tokens_seen": 306774016
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732950852557673,
+ "loss": 1.5444,
+ "theoretical_loss": 4.16395295812229,
+ "tokens_seen": 306839552
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007329348044132398,
+ "loss": 1.6675,
+ "theoretical_loss": 4.1638497160317645,
+ "tokens_seen": 306905088
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007329187562688064,
+ "loss": 1.6466,
+ "theoretical_loss": 4.163746502156448,
+ "tokens_seen": 306970624
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007329027081243732,
+ "loss": 1.4845,
+ "theoretical_loss": 4.163643316482606,
+ "tokens_seen": 307036160
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328866599799398,
+ "loss": 1.6061,
+ "theoretical_loss": 4.1635401589965175,
+ "tokens_seen": 307101696
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328706118355065,
+ "loss": 1.6429,
+ "theoretical_loss": 4.163437029684469,
+ "tokens_seen": 307167232
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328545636910732,
+ "loss": 1.6264,
+ "theoretical_loss": 4.163333928532758,
+ "tokens_seen": 307232768
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328385155466399,
+ "loss": 1.6924,
+ "theoretical_loss": 4.163230855527688,
+ "tokens_seen": 307298304
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328224674022066,
+ "loss": 1.6224,
+ "theoretical_loss": 4.163127810655578,
+ "tokens_seen": 307363840
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007328064192577733,
+ "loss": 1.6408,
+ "theoretical_loss": 4.163024793902752,
+ "tokens_seen": 307429376
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.00073279037111334,
+ "loss": 1.7319,
+ "theoretical_loss": 4.162921805255543,
+ "tokens_seen": 307494912
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007327743229689067,
+ "loss": 1.501,
+ "theoretical_loss": 4.162818844700298,
+ "tokens_seen": 307560448
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007327582748244734,
+ "loss": 1.6282,
+ "theoretical_loss": 4.162715912223369,
+ "tokens_seen": 307625984
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007327422266800401,
+ "loss": 1.5315,
+ "theoretical_loss": 4.162613007811122,
+ "tokens_seen": 307691520
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007327261785356068,
+ "loss": 1.5266,
+ "theoretical_loss": 4.1625101314499275,
+ "tokens_seen": 307757056
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007327101303911736,
+ "loss": 1.5706,
+ "theoretical_loss": 4.162407283126169,
+ "tokens_seen": 307822592
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007326940822467402,
+ "loss": 1.5297,
+ "theoretical_loss": 4.1623044628262384,
+ "tokens_seen": 307888128
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732678034102307,
+ "loss": 1.6028,
+ "theoretical_loss": 4.162201670536537,
+ "tokens_seen": 307953664
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 108952,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5811710357666016,
+ "objective/train/theoretical_loss": 4.162098906243477,
+ "objective/train/tokens_used": 328479200,
+ "theoretical_loss": 4.162098906243477,
+ "tokens_seen": 308019200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007326619859578736,
+ "loss": 1.6008,
+ "theoretical_loss": 4.162098906243477,
+ "tokens_seen": 308019200
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007326459378134404,
+ "loss": 1.5509,
+ "theoretical_loss": 4.161996169933477,
+ "tokens_seen": 308084736
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732629889669007,
+ "loss": 1.6285,
+ "theoretical_loss": 4.161893461592968,
+ "tokens_seen": 308150272
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007326138415245738,
+ "loss": 1.6352,
+ "theoretical_loss": 4.16179078120839,
+ "tokens_seen": 308215808
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325977933801404,
+ "loss": 1.5552,
+ "theoretical_loss": 4.161688128766191,
+ "tokens_seen": 308281344
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325817452357072,
+ "loss": 1.5523,
+ "theoretical_loss": 4.16158550425283,
+ "tokens_seen": 308346880
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325656970912738,
+ "loss": 1.6311,
+ "theoretical_loss": 4.161482907654775,
+ "tokens_seen": 308412416
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325496489468406,
+ "loss": 1.6177,
+ "theoretical_loss": 4.161380338958502,
+ "tokens_seen": 308477952
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325336008024072,
+ "loss": 1.6076,
+ "theoretical_loss": 4.161277798150498,
+ "tokens_seen": 308543488
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732517552657974,
+ "loss": 1.5029,
+ "theoretical_loss": 4.16117528521726,
+ "tokens_seen": 308609024
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007325015045135406,
+ "loss": 1.6536,
+ "theoretical_loss": 4.161072800145292,
+ "tokens_seen": 308674560
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007324854563691074,
+ "loss": 1.5278,
+ "theoretical_loss": 4.16097034292111,
+ "tokens_seen": 308740096
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732469408224674,
+ "loss": 1.4431,
+ "theoretical_loss": 4.160867913531238,
+ "tokens_seen": 308805632
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007324533600802408,
+ "loss": 1.6229,
+ "theoretical_loss": 4.160765511962209,
+ "tokens_seen": 308871168
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007324373119358074,
+ "loss": 1.5469,
+ "theoretical_loss": 4.160663138200567,
+ "tokens_seen": 308936704
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007324212637913742,
+ "loss": 1.6081,
+ "theoretical_loss": 4.1605607922328645,
+ "tokens_seen": 309002240
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007324052156469408,
+ "loss": 1.6369,
+ "theoretical_loss": 4.160458474045662,
+ "tokens_seen": 309067776
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007323891675025076,
+ "loss": 1.736,
+ "theoretical_loss": 4.16035618362553,
+ "tokens_seen": 309133312
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007323731193580742,
+ "loss": 1.4865,
+ "theoretical_loss": 4.16025392095905,
+ "tokens_seen": 309198848
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732357071213641,
+ "loss": 1.6449,
+ "theoretical_loss": 4.160151686032814,
+ "tokens_seen": 309264384
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007323410230692076,
+ "loss": 1.622,
+ "theoretical_loss": 4.160049478833416,
+ "tokens_seen": 309329920
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007323249749247744,
+ "loss": 1.681,
+ "theoretical_loss": 4.159947299347468,
+ "tokens_seen": 309395456
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732308926780341,
+ "loss": 1.5762,
+ "theoretical_loss": 4.159845147561587,
+ "tokens_seen": 309460992
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322928786359078,
+ "loss": 1.5544,
+ "theoretical_loss": 4.1597430234624,
+ "tokens_seen": 309526528
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322768304914744,
+ "loss": 1.4294,
+ "theoretical_loss": 4.159640927036541,
+ "tokens_seen": 309592064
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322607823470412,
+ "loss": 1.5333,
+ "theoretical_loss": 4.159538858270659,
+ "tokens_seen": 309657600
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322447342026078,
+ "loss": 1.5659,
+ "theoretical_loss": 4.159436817151407,
+ "tokens_seen": 309723136
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322286860581746,
+ "loss": 1.6312,
+ "theoretical_loss": 4.159334803665448,
+ "tokens_seen": 309788672
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007322126379137412,
+ "loss": 1.5842,
+ "theoretical_loss": 4.159232817799458,
+ "tokens_seen": 309854208
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732196589769308,
+ "loss": 1.5808,
+ "theoretical_loss": 4.1591308595401175,
+ "tokens_seen": 309919744
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007321805416248746,
+ "loss": 1.5817,
+ "theoretical_loss": 4.159028928874118,
+ "tokens_seen": 309985280
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007321644934804414,
+ "loss": 1.6191,
+ "theoretical_loss": 4.158927025788162,
+ "tokens_seen": 310050816
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732148445336008,
+ "loss": 1.624,
+ "theoretical_loss": 4.158825150268958,
+ "tokens_seen": 310116352
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007321323971915748,
+ "loss": 1.5717,
+ "theoretical_loss": 4.158723302303227,
+ "tokens_seen": 310181888
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007321163490471414,
+ "loss": 1.6036,
+ "theoretical_loss": 4.158621481877697,
+ "tokens_seen": 310247424
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007321003009027082,
+ "loss": 1.6511,
+ "theoretical_loss": 4.158519688979106,
+ "tokens_seen": 310312960
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007320842527582748,
+ "loss": 1.5586,
+ "theoretical_loss": 4.1584179235942,
+ "tokens_seen": 310378496
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007320682046138416,
+ "loss": 1.6026,
+ "theoretical_loss": 4.158316185709737,
+ "tokens_seen": 310444032
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007320521564694082,
+ "loss": 1.7038,
+ "theoretical_loss": 4.158214475312481,
+ "tokens_seen": 310509568
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000732036108324975,
+ "loss": 1.7325,
+ "theoretical_loss": 4.158112792389206,
+ "tokens_seen": 310575104
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007320200601805416,
+ "loss": 1.6455,
+ "theoretical_loss": 4.158011136926698,
+ "tokens_seen": 310640640
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007320040120361084,
+ "loss": 1.5333,
+ "theoretical_loss": 4.157909508911747,
+ "tokens_seen": 310706176
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000731987963891675,
+ "loss": 1.663,
+ "theoretical_loss": 4.157807908331157,
+ "tokens_seen": 310771712
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007319719157472418,
+ "loss": 1.6373,
+ "theoretical_loss": 4.157706335171738,
+ "tokens_seen": 310837248
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007319558676028085,
+ "loss": 1.6038,
+ "theoretical_loss": 4.15760478942031,
+ "tokens_seen": 310902784
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007319398194583752,
+ "loss": 1.6834,
+ "theoretical_loss": 4.157503271063703,
+ "tokens_seen": 310968320
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007319237713139419,
+ "loss": 1.529,
+ "theoretical_loss": 4.157401780088756,
+ "tokens_seen": 311033856
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007319077231695086,
+ "loss": 1.671,
+ "theoretical_loss": 4.157300316482315,
+ "tokens_seen": 311099392
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007318916750250753,
+ "loss": 1.6511,
+ "theoretical_loss": 4.157198880231238,
+ "tokens_seen": 311164928
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000731875626880642,
+ "loss": 1.5657,
+ "theoretical_loss": 4.157097471322389,
+ "tokens_seen": 311230464
+ },
+ {
+ "epoch": 0.09,
+ "objective/train/docs_used": 110010,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5765893459320068,
+ "objective/train/theoretical_loss": 4.156996089742645,
+ "objective/train/tokens_used": 331756000,
+ "theoretical_loss": 4.156996089742645,
+ "tokens_seen": 311296000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007318595787362087,
+ "loss": 1.56,
+ "theoretical_loss": 4.156996089742645,
+ "tokens_seen": 311296000
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007318435305917754,
+ "loss": 1.7442,
+ "theoretical_loss": 4.156894735478888,
+ "tokens_seen": 311361536
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007318274824473421,
+ "loss": 1.5361,
+ "theoretical_loss": 4.156793408518011,
+ "tokens_seen": 311427072
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007318114343029088,
+ "loss": 1.6328,
+ "theoretical_loss": 4.156692108846916,
+ "tokens_seen": 311492608
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007317953861584755,
+ "loss": 1.6673,
+ "theoretical_loss": 4.156590836452514,
+ "tokens_seen": 311558144
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007317793380140422,
+ "loss": 1.7033,
+ "theoretical_loss": 4.156489591321726,
+ "tokens_seen": 311623680
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007317632898696089,
+ "loss": 1.4632,
+ "theoretical_loss": 4.15638837344148,
+ "tokens_seen": 311689216
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007317472417251756,
+ "loss": 1.6694,
+ "theoretical_loss": 4.156287182798714,
+ "tokens_seen": 311754752
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007317311935807423,
+ "loss": 1.552,
+ "theoretical_loss": 4.156186019380375,
+ "tokens_seen": 311820288
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.000731715145436309,
+ "loss": 1.6538,
+ "theoretical_loss": 4.156084883173419,
+ "tokens_seen": 311885824
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316990972918757,
+ "loss": 1.5271,
+ "theoretical_loss": 4.155983774164811,
+ "tokens_seen": 311951360
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316830491474424,
+ "loss": 1.4608,
+ "theoretical_loss": 4.1558826923415255,
+ "tokens_seen": 312016896
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316670010030091,
+ "loss": 1.6252,
+ "theoretical_loss": 4.155781637690545,
+ "tokens_seen": 312082432
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316509528585758,
+ "loss": 1.5328,
+ "theoretical_loss": 4.1556806101988615,
+ "tokens_seen": 312147968
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316349047141425,
+ "loss": 1.5291,
+ "theoretical_loss": 4.155579609853476,
+ "tokens_seen": 312213504
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316188565697092,
+ "loss": 1.5796,
+ "theoretical_loss": 4.155478636641398,
+ "tokens_seen": 312279040
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007316028084252759,
+ "loss": 1.7283,
+ "theoretical_loss": 4.155377690549646,
+ "tokens_seen": 312344576
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315867602808425,
+ "loss": 1.5384,
+ "theoretical_loss": 4.155276771565248,
+ "tokens_seen": 312410112
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315707121364093,
+ "loss": 1.6876,
+ "theoretical_loss": 4.155175879675241,
+ "tokens_seen": 312475648
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315546639919759,
+ "loss": 1.5711,
+ "theoretical_loss": 4.155075014866671,
+ "tokens_seen": 312541184
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315386158475427,
+ "loss": 1.6038,
+ "theoretical_loss": 4.15497417712659,
+ "tokens_seen": 312606720
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315225677031093,
+ "loss": 1.6181,
+ "theoretical_loss": 4.1548733664420645,
+ "tokens_seen": 312672256
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007315065195586761,
+ "loss": 1.6023,
+ "theoretical_loss": 4.154772582800165,
+ "tokens_seen": 312737792
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314904714142427,
+ "loss": 1.6421,
+ "theoretical_loss": 4.154671826187972,
+ "tokens_seen": 312803328
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314744232698095,
+ "loss": 1.6457,
+ "theoretical_loss": 4.154571096592576,
+ "tokens_seen": 312868864
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314583751253761,
+ "loss": 1.6198,
+ "theoretical_loss": 4.154470394001077,
+ "tokens_seen": 312934400
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314423269809429,
+ "loss": 1.5943,
+ "theoretical_loss": 4.154369718400583,
+ "tokens_seen": 312999936
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314262788365095,
+ "loss": 1.5763,
+ "theoretical_loss": 4.154269069778207,
+ "tokens_seen": 313065472
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007314102306920763,
+ "loss": 1.6197,
+ "theoretical_loss": 4.154168448121078,
+ "tokens_seen": 313131008
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007313941825476429,
+ "loss": 1.6308,
+ "theoretical_loss": 4.15406785341633,
+ "tokens_seen": 313196544
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007313781344032097,
+ "loss": 1.488,
+ "theoretical_loss": 4.153967285651105,
+ "tokens_seen": 313262080
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007313620862587763,
+ "loss": 1.7241,
+ "theoretical_loss": 4.153866744812555,
+ "tokens_seen": 313327616
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007313460381143431,
+ "loss": 1.488,
+ "theoretical_loss": 4.1537662308878405,
+ "tokens_seen": 313393152
+ },
+ {
+ "epoch": 0.09,
+ "learning_rate": 0.0007313299899699097,
+ "loss": 1.685,
+ "theoretical_loss": 4.153665743864131,
+ "tokens_seen": 313458688
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007313139418254765,
+ "loss": 1.5907,
+ "theoretical_loss": 4.153565283728606,
+ "tokens_seen": 313524224
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312978936810431,
+ "loss": 1.6424,
+ "theoretical_loss": 4.153464850468452,
+ "tokens_seen": 313589760
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312818455366099,
+ "loss": 1.6169,
+ "theoretical_loss": 4.1533644440708635,
+ "tokens_seen": 313655296
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312657973921765,
+ "loss": 1.6373,
+ "theoretical_loss": 4.153264064523047,
+ "tokens_seen": 313720832
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312497492477433,
+ "loss": 1.4888,
+ "theoretical_loss": 4.1531637118122156,
+ "tokens_seen": 313786368
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312337011033099,
+ "loss": 1.6178,
+ "theoretical_loss": 4.153063385925591,
+ "tokens_seen": 313851904
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312176529588767,
+ "loss": 1.6062,
+ "theoretical_loss": 4.152963086850405,
+ "tokens_seen": 313917440
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007312016048144433,
+ "loss": 1.7214,
+ "theoretical_loss": 4.152862814573895,
+ "tokens_seen": 313982976
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311855566700101,
+ "loss": 1.5936,
+ "theoretical_loss": 4.152762569083313,
+ "tokens_seen": 314048512
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311695085255767,
+ "loss": 1.6886,
+ "theoretical_loss": 4.1526623503659135,
+ "tokens_seen": 314114048
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311534603811435,
+ "loss": 1.543,
+ "theoretical_loss": 4.152562158408962,
+ "tokens_seen": 314179584
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311374122367101,
+ "loss": 1.5865,
+ "theoretical_loss": 4.1524619931997355,
+ "tokens_seen": 314245120
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311213640922769,
+ "loss": 1.5747,
+ "theoretical_loss": 4.152361854725515,
+ "tokens_seen": 314310656
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007311053159478435,
+ "loss": 1.6538,
+ "theoretical_loss": 4.152261742973594,
+ "tokens_seen": 314376192
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310892678034103,
+ "loss": 1.6593,
+ "theoretical_loss": 4.152161657931273,
+ "tokens_seen": 314441728
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310732196589769,
+ "loss": 1.6103,
+ "theoretical_loss": 4.15206159958586,
+ "tokens_seen": 314507264
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 111213,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5043652057647705,
+ "objective/train/theoretical_loss": 4.1519615679246735,
+ "objective/train/tokens_used": 335032800,
+ "theoretical_loss": 4.1519615679246735,
+ "tokens_seen": 314572800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310571715145437,
+ "loss": 1.5245,
+ "theoretical_loss": 4.1519615679246735,
+ "tokens_seen": 314572800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310411233701103,
+ "loss": 1.5779,
+ "theoretical_loss": 4.151861562935041,
+ "tokens_seen": 314638336
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310250752256771,
+ "loss": 1.5589,
+ "theoretical_loss": 4.151761584604298,
+ "tokens_seen": 314703872
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007310090270812437,
+ "loss": 1.5793,
+ "theoretical_loss": 4.1516616329197875,
+ "tokens_seen": 314769408
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309929789368105,
+ "loss": 1.6619,
+ "theoretical_loss": 4.151561707868863,
+ "tokens_seen": 314834944
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309769307923771,
+ "loss": 1.6171,
+ "theoretical_loss": 4.151461809438885,
+ "tokens_seen": 314900480
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309608826479439,
+ "loss": 1.5683,
+ "theoretical_loss": 4.1513619376172235,
+ "tokens_seen": 314966016
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309448345035105,
+ "loss": 1.6547,
+ "theoretical_loss": 4.151262092391257,
+ "tokens_seen": 315031552
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309287863590773,
+ "loss": 1.6485,
+ "theoretical_loss": 4.1511622737483735,
+ "tokens_seen": 315097088
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007309127382146439,
+ "loss": 1.5209,
+ "theoretical_loss": 4.151062481675968,
+ "tokens_seen": 315162624
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308966900702107,
+ "loss": 1.5688,
+ "theoretical_loss": 4.150962716161445,
+ "tokens_seen": 315228160
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308806419257773,
+ "loss": 1.5452,
+ "theoretical_loss": 4.150862977192217,
+ "tokens_seen": 315293696
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308645937813441,
+ "loss": 1.555,
+ "theoretical_loss": 4.150763264755706,
+ "tokens_seen": 315359232
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308485456369107,
+ "loss": 1.7569,
+ "theoretical_loss": 4.150663578839342,
+ "tokens_seen": 315424768
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308324974924775,
+ "loss": 1.598,
+ "theoretical_loss": 4.150563919430562,
+ "tokens_seen": 315490304
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308164493480441,
+ "loss": 1.4641,
+ "theoretical_loss": 4.150464286516816,
+ "tokens_seen": 315555840
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007308004012036109,
+ "loss": 1.5865,
+ "theoretical_loss": 4.150364680085558,
+ "tokens_seen": 315621376
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307843530591775,
+ "loss": 1.5584,
+ "theoretical_loss": 4.150265100124253,
+ "tokens_seen": 315686912
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307683049147443,
+ "loss": 1.6787,
+ "theoretical_loss": 4.150165546620373,
+ "tokens_seen": 315752448
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307522567703109,
+ "loss": 1.7157,
+ "theoretical_loss": 4.1500660195614,
+ "tokens_seen": 315817984
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307362086258777,
+ "loss": 1.5774,
+ "theoretical_loss": 4.149966518934824,
+ "tokens_seen": 315883520
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307201604814443,
+ "loss": 1.5534,
+ "theoretical_loss": 4.149867044728143,
+ "tokens_seen": 315949056
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007307041123370111,
+ "loss": 1.597,
+ "theoretical_loss": 4.149767596928863,
+ "tokens_seen": 316014592
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306880641925777,
+ "loss": 1.8193,
+ "theoretical_loss": 4.149668175524502,
+ "tokens_seen": 316080128
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306720160481445,
+ "loss": 1.5885,
+ "theoretical_loss": 4.14956878050258,
+ "tokens_seen": 316145664
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306559679037111,
+ "loss": 1.6594,
+ "theoretical_loss": 4.1494694118506334,
+ "tokens_seen": 316211200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306399197592779,
+ "loss": 1.6892,
+ "theoretical_loss": 4.1493700695562,
+ "tokens_seen": 316276736
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306238716148446,
+ "loss": 1.5594,
+ "theoretical_loss": 4.14927075360683,
+ "tokens_seen": 316342272
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007306078234704113,
+ "loss": 1.6457,
+ "theoretical_loss": 4.149171463990082,
+ "tokens_seen": 316407808
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000730591775325978,
+ "loss": 1.5986,
+ "theoretical_loss": 4.149072200693521,
+ "tokens_seen": 316473344
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007305757271815447,
+ "loss": 1.6219,
+ "theoretical_loss": 4.148972963704722,
+ "tokens_seen": 316538880
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007305596790371114,
+ "loss": 1.6861,
+ "theoretical_loss": 4.148873753011269,
+ "tokens_seen": 316604416
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007305436308926781,
+ "loss": 1.6092,
+ "theoretical_loss": 4.148774568600752,
+ "tokens_seen": 316669952
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007305275827482448,
+ "loss": 1.5524,
+ "theoretical_loss": 4.148675410460772,
+ "tokens_seen": 316735488
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007305115346038115,
+ "loss": 1.5586,
+ "theoretical_loss": 4.148576278578937,
+ "tokens_seen": 316801024
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007304954864593782,
+ "loss": 1.5904,
+ "theoretical_loss": 4.148477172942863,
+ "tokens_seen": 316866560
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007304794383149449,
+ "loss": 1.745,
+ "theoretical_loss": 4.1483780935401775,
+ "tokens_seen": 316932096
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007304633901705116,
+ "loss": 1.5839,
+ "theoretical_loss": 4.148279040358511,
+ "tokens_seen": 316997632
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007304473420260783,
+ "loss": 1.5903,
+ "theoretical_loss": 4.148180013385507,
+ "tokens_seen": 317063168
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000730431293881645,
+ "loss": 1.7813,
+ "theoretical_loss": 4.148081012608817,
+ "tokens_seen": 317128704
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007304152457372117,
+ "loss": 1.8107,
+ "theoretical_loss": 4.147982038016096,
+ "tokens_seen": 317194240
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303991975927784,
+ "loss": 1.7651,
+ "theoretical_loss": 4.147883089595015,
+ "tokens_seen": 317259776
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303831494483451,
+ "loss": 1.8327,
+ "theoretical_loss": 4.147784167333247,
+ "tokens_seen": 317325312
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303671013039118,
+ "loss": 1.8399,
+ "theoretical_loss": 4.147685271218477,
+ "tokens_seen": 317390848
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303510531594785,
+ "loss": 1.8931,
+ "theoretical_loss": 4.147586401238396,
+ "tokens_seen": 317456384
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303350050150452,
+ "loss": 1.8639,
+ "theoretical_loss": 4.147487557380705,
+ "tokens_seen": 317521920
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303189568706119,
+ "loss": 1.8167,
+ "theoretical_loss": 4.147388739633113,
+ "tokens_seen": 317587456
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007303029087261786,
+ "loss": 1.86,
+ "theoretical_loss": 4.147289947983337,
+ "tokens_seen": 317652992
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007302868605817453,
+ "loss": 1.9685,
+ "theoretical_loss": 4.1471911824191015,
+ "tokens_seen": 317718528
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000730270812437312,
+ "loss": 1.7142,
+ "theoretical_loss": 4.147092442928141,
+ "tokens_seen": 317784064
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 112065,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 2.024470567703247,
+ "objective/train/theoretical_loss": 4.146993729498197,
+ "objective/train/tokens_used": 338309600,
+ "theoretical_loss": 4.146993729498197,
+ "tokens_seen": 317849600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007302547642928787,
+ "loss": 1.9022,
+ "theoretical_loss": 4.146993729498197,
+ "tokens_seen": 317849600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007302387161484454,
+ "loss": 1.9061,
+ "theoretical_loss": 4.146895042117021,
+ "tokens_seen": 317915136
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007302226680040121,
+ "loss": 1.9165,
+ "theoretical_loss": 4.146796380772369,
+ "tokens_seen": 317980672
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007302066198595788,
+ "loss": 1.9143,
+ "theoretical_loss": 4.146697745452011,
+ "tokens_seen": 318046208
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007301905717151454,
+ "loss": 1.8095,
+ "theoretical_loss": 4.146599136143719,
+ "tokens_seen": 318111744
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007301745235707122,
+ "loss": 1.7185,
+ "theoretical_loss": 4.146500552835278,
+ "tokens_seen": 318177280
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007301584754262788,
+ "loss": 1.8265,
+ "theoretical_loss": 4.146401995514479,
+ "tokens_seen": 318242816
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007301424272818456,
+ "loss": 1.8853,
+ "theoretical_loss": 4.146303464169123,
+ "tokens_seen": 318308352
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007301263791374122,
+ "loss": 1.8499,
+ "theoretical_loss": 4.146204958787017,
+ "tokens_seen": 318373888
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000730110330992979,
+ "loss": 1.6803,
+ "theoretical_loss": 4.146106479355978,
+ "tokens_seen": 318439424
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007300942828485456,
+ "loss": 1.8647,
+ "theoretical_loss": 4.1460080258638285,
+ "tokens_seen": 318504960
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007300782347041124,
+ "loss": 1.8665,
+ "theoretical_loss": 4.145909598298404,
+ "tokens_seen": 318570496
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000730062186559679,
+ "loss": 1.8231,
+ "theoretical_loss": 4.145811196647544,
+ "tokens_seen": 318636032
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007300461384152458,
+ "loss": 1.8109,
+ "theoretical_loss": 4.145712820899098,
+ "tokens_seen": 318701568
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007300300902708124,
+ "loss": 1.7429,
+ "theoretical_loss": 4.145614471040923,
+ "tokens_seen": 318767104
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007300140421263792,
+ "loss": 1.8226,
+ "theoretical_loss": 4.145516147060884,
+ "tokens_seen": 318832640
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007299979939819458,
+ "loss": 1.701,
+ "theoretical_loss": 4.145417848946857,
+ "tokens_seen": 318898176
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007299819458375126,
+ "loss": 1.8668,
+ "theoretical_loss": 4.145319576686721,
+ "tokens_seen": 318963712
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007299658976930792,
+ "loss": 1.6904,
+ "theoretical_loss": 4.1452213302683685,
+ "tokens_seen": 319029248
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729949849548646,
+ "loss": 1.9191,
+ "theoretical_loss": 4.145123109679696,
+ "tokens_seen": 319094784
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007299338014042126,
+ "loss": 1.9909,
+ "theoretical_loss": 4.1450249149086105,
+ "tokens_seen": 319160320
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007299177532597794,
+ "loss": 1.9859,
+ "theoretical_loss": 4.144926745943026,
+ "tokens_seen": 319225856
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729901705115346,
+ "loss": 1.8796,
+ "theoretical_loss": 4.1448286027708665,
+ "tokens_seen": 319291392
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298856569709128,
+ "loss": 1.7589,
+ "theoretical_loss": 4.144730485380062,
+ "tokens_seen": 319356928
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298696088264794,
+ "loss": 1.881,
+ "theoretical_loss": 4.144632393758551,
+ "tokens_seen": 319422464
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298535606820462,
+ "loss": 1.9404,
+ "theoretical_loss": 4.144534327894281,
+ "tokens_seen": 319488000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298375125376128,
+ "loss": 1.8255,
+ "theoretical_loss": 4.144436287775206,
+ "tokens_seen": 319553536
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298214643931796,
+ "loss": 1.8244,
+ "theoretical_loss": 4.144338273389291,
+ "tokens_seen": 319619072
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007298054162487462,
+ "loss": 1.7481,
+ "theoretical_loss": 4.144240284724505,
+ "tokens_seen": 319684608
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729789368104313,
+ "loss": 1.8876,
+ "theoretical_loss": 4.144142321768831,
+ "tokens_seen": 319750144
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007297733199598796,
+ "loss": 1.7792,
+ "theoretical_loss": 4.144044384510253,
+ "tokens_seen": 319815680
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007297572718154464,
+ "loss": 1.7007,
+ "theoretical_loss": 4.1439464729367685,
+ "tokens_seen": 319881216
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729741223671013,
+ "loss": 1.5911,
+ "theoretical_loss": 4.14384858703638,
+ "tokens_seen": 319946752
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007297251755265798,
+ "loss": 1.7342,
+ "theoretical_loss": 4.1437507267971,
+ "tokens_seen": 320012288
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007297091273821464,
+ "loss": 1.7468,
+ "theoretical_loss": 4.143652892206948,
+ "tokens_seen": 320077824
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007296930792377132,
+ "loss": 1.7422,
+ "theoretical_loss": 4.1435550832539505,
+ "tokens_seen": 320143360
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007296770310932798,
+ "loss": 1.6767,
+ "theoretical_loss": 4.143457299926146,
+ "tokens_seen": 320208896
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007296609829488466,
+ "loss": 1.7012,
+ "theoretical_loss": 4.143359542211576,
+ "tokens_seen": 320274432
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007296449348044132,
+ "loss": 1.6575,
+ "theoretical_loss": 4.143261810098293,
+ "tokens_seen": 320339968
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.00072962888665998,
+ "loss": 1.8258,
+ "theoretical_loss": 4.1431641035743585,
+ "tokens_seen": 320405504
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007296128385155466,
+ "loss": 1.7058,
+ "theoretical_loss": 4.143066422627838,
+ "tokens_seen": 320471040
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295967903711134,
+ "loss": 1.7022,
+ "theoretical_loss": 4.142968767246808,
+ "tokens_seen": 320536576
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.00072958074222668,
+ "loss": 1.658,
+ "theoretical_loss": 4.142871137419354,
+ "tokens_seen": 320602112
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295646940822468,
+ "loss": 1.7651,
+ "theoretical_loss": 4.142773533133565,
+ "tokens_seen": 320667648
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295486459378134,
+ "loss": 1.7998,
+ "theoretical_loss": 4.142675954377543,
+ "tokens_seen": 320733184
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295325977933802,
+ "loss": 1.8644,
+ "theoretical_loss": 4.142578401139396,
+ "tokens_seen": 320798720
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295165496489468,
+ "loss": 1.778,
+ "theoretical_loss": 4.142480873407239,
+ "tokens_seen": 320864256
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007295005015045136,
+ "loss": 1.7293,
+ "theoretical_loss": 4.1423833711691955,
+ "tokens_seen": 320929792
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007294844533600802,
+ "loss": 1.6623,
+ "theoretical_loss": 4.142285894413398,
+ "tokens_seen": 320995328
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729468405215647,
+ "loss": 1.768,
+ "theoretical_loss": 4.142188443127985,
+ "tokens_seen": 321060864
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 113306,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5380922555923462,
+ "objective/train/theoretical_loss": 4.142091017301105,
+ "objective/train/tokens_used": 341586400,
+ "theoretical_loss": 4.142091017301105,
+ "tokens_seen": 321126400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007294523570712136,
+ "loss": 1.6093,
+ "theoretical_loss": 4.142091017301105,
+ "tokens_seen": 321126400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007294363089267804,
+ "loss": 1.6369,
+ "theoretical_loss": 4.141993616920914,
+ "tokens_seen": 321191936
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729420260782347,
+ "loss": 1.71,
+ "theoretical_loss": 4.141896241975575,
+ "tokens_seen": 321257472
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007294042126379138,
+ "loss": 1.4959,
+ "theoretical_loss": 4.141798892453259,
+ "tokens_seen": 321323008
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007293881644934804,
+ "loss": 1.7129,
+ "theoretical_loss": 4.141701568342145,
+ "tokens_seen": 321388544
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007293721163490472,
+ "loss": 1.6549,
+ "theoretical_loss": 4.141604269630422,
+ "tokens_seen": 321454080
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007293560682046138,
+ "loss": 1.5904,
+ "theoretical_loss": 4.1415069963062825,
+ "tokens_seen": 321519616
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007293400200601806,
+ "loss": 1.6822,
+ "theoretical_loss": 4.1414097483579315,
+ "tokens_seen": 321585152
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007293239719157472,
+ "loss": 1.6719,
+ "theoretical_loss": 4.1413125257735794,
+ "tokens_seen": 321650688
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729307923771314,
+ "loss": 1.5757,
+ "theoretical_loss": 4.141215328541445,
+ "tokens_seen": 321716224
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007292918756268806,
+ "loss": 1.5815,
+ "theoretical_loss": 4.141118156649753,
+ "tokens_seen": 321781760
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007292758274824474,
+ "loss": 1.6802,
+ "theoretical_loss": 4.1410210100867415,
+ "tokens_seen": 321847296
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729259779338014,
+ "loss": 1.6105,
+ "theoretical_loss": 4.14092388884065,
+ "tokens_seen": 321912832
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007292437311935808,
+ "loss": 1.5999,
+ "theoretical_loss": 4.1408267928997295,
+ "tokens_seen": 321978368
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007292276830491475,
+ "loss": 1.6539,
+ "theoretical_loss": 4.140729722252239,
+ "tokens_seen": 322043904
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007292116349047142,
+ "loss": 1.6241,
+ "theoretical_loss": 4.140632676886442,
+ "tokens_seen": 322109440
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007291955867602809,
+ "loss": 1.6331,
+ "theoretical_loss": 4.140535656790615,
+ "tokens_seen": 322174976
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007291795386158476,
+ "loss": 1.6988,
+ "theoretical_loss": 4.140438661953038,
+ "tokens_seen": 322240512
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007291634904714143,
+ "loss": 1.4855,
+ "theoretical_loss": 4.140341692362,
+ "tokens_seen": 322306048
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729147442326981,
+ "loss": 1.682,
+ "theoretical_loss": 4.140244748005799,
+ "tokens_seen": 322371584
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007291313941825477,
+ "loss": 1.7468,
+ "theoretical_loss": 4.14014782887274,
+ "tokens_seen": 322437120
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007291153460381144,
+ "loss": 1.6284,
+ "theoretical_loss": 4.140050934951135,
+ "tokens_seen": 322502656
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290992978936812,
+ "loss": 1.6126,
+ "theoretical_loss": 4.139954066229304,
+ "tokens_seen": 322568192
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290832497492478,
+ "loss": 1.6776,
+ "theoretical_loss": 4.139857222695578,
+ "tokens_seen": 322633728
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290672016048146,
+ "loss": 1.6517,
+ "theoretical_loss": 4.139760404338291,
+ "tokens_seen": 322699264
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290511534603812,
+ "loss": 1.6009,
+ "theoretical_loss": 4.139663611145786,
+ "tokens_seen": 322764800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000729035105315948,
+ "loss": 1.6975,
+ "theoretical_loss": 4.139566843106417,
+ "tokens_seen": 322830336
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290190571715146,
+ "loss": 1.6493,
+ "theoretical_loss": 4.139470100208542,
+ "tokens_seen": 322895872
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007290030090270814,
+ "loss": 1.7209,
+ "theoretical_loss": 4.139373382440528,
+ "tokens_seen": 322961408
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000728986960882648,
+ "loss": 1.6022,
+ "theoretical_loss": 4.1392766897907505,
+ "tokens_seen": 323026944
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007289709127382148,
+ "loss": 1.6735,
+ "theoretical_loss": 4.139180022247592,
+ "tokens_seen": 323092480
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007289548645937814,
+ "loss": 1.5358,
+ "theoretical_loss": 4.139083379799443,
+ "tokens_seen": 323158016
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007289388164493481,
+ "loss": 1.5424,
+ "theoretical_loss": 4.1389867624347,
+ "tokens_seen": 323223552
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007289227683049148,
+ "loss": 1.6485,
+ "theoretical_loss": 4.138890170141771,
+ "tokens_seen": 323289088
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007289067201604815,
+ "loss": 1.5949,
+ "theoretical_loss": 4.138793602909068,
+ "tokens_seen": 323354624
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007288906720160482,
+ "loss": 1.7056,
+ "theoretical_loss": 4.138697060725013,
+ "tokens_seen": 323420160
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007288746238716149,
+ "loss": 1.6545,
+ "theoretical_loss": 4.138600543578034,
+ "tokens_seen": 323485696
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007288585757271816,
+ "loss": 1.7005,
+ "theoretical_loss": 4.138504051456568,
+ "tokens_seen": 323551232
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007288425275827483,
+ "loss": 1.6943,
+ "theoretical_loss": 4.13840758434906,
+ "tokens_seen": 323616768
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000728826479438315,
+ "loss": 1.5391,
+ "theoretical_loss": 4.1383111422439605,
+ "tokens_seen": 323682304
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007288104312938817,
+ "loss": 1.6891,
+ "theoretical_loss": 4.13821472512973,
+ "tokens_seen": 323747840
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287943831494483,
+ "loss": 1.6866,
+ "theoretical_loss": 4.138118332994837,
+ "tokens_seen": 323813376
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287783350050151,
+ "loss": 1.69,
+ "theoretical_loss": 4.138021965827753,
+ "tokens_seen": 323878912
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287622868605817,
+ "loss": 1.5937,
+ "theoretical_loss": 4.1379256236169635,
+ "tokens_seen": 323944448
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287462387161485,
+ "loss": 1.6,
+ "theoretical_loss": 4.137829306350957,
+ "tokens_seen": 324009984
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287301905717151,
+ "loss": 1.6746,
+ "theoretical_loss": 4.137733014018233,
+ "tokens_seen": 324075520
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007287141424272819,
+ "loss": 1.6548,
+ "theoretical_loss": 4.137636746607295,
+ "tokens_seen": 324141056
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286980942828485,
+ "loss": 1.5518,
+ "theoretical_loss": 4.137540504106657,
+ "tokens_seen": 324206592
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286820461384153,
+ "loss": 1.7232,
+ "theoretical_loss": 4.137444286504841,
+ "tokens_seen": 324272128
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286659979939819,
+ "loss": 1.6377,
+ "theoretical_loss": 4.137348093790372,
+ "tokens_seen": 324337664
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 114269,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6439099311828613,
+ "objective/train/theoretical_loss": 4.13725192595179,
+ "objective/train/tokens_used": 344863200,
+ "theoretical_loss": 4.13725192595179,
+ "tokens_seen": 324403200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286499498495487,
+ "loss": 1.5966,
+ "theoretical_loss": 4.13725192595179,
+ "tokens_seen": 324403200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286339017051153,
+ "loss": 1.7042,
+ "theoretical_loss": 4.1371557829776355,
+ "tokens_seen": 324468736
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286178535606821,
+ "loss": 1.6878,
+ "theoretical_loss": 4.13705966485646,
+ "tokens_seen": 324534272
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007286018054162487,
+ "loss": 1.641,
+ "theoretical_loss": 4.136963571576825,
+ "tokens_seen": 324599808
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285857572718155,
+ "loss": 1.6288,
+ "theoretical_loss": 4.136867503127292,
+ "tokens_seen": 324665344
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285697091273821,
+ "loss": 1.5733,
+ "theoretical_loss": 4.136771459496439,
+ "tokens_seen": 324730880
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285536609829489,
+ "loss": 1.5923,
+ "theoretical_loss": 4.136675440672844,
+ "tokens_seen": 324796416
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285376128385155,
+ "loss": 1.7097,
+ "theoretical_loss": 4.136579446645098,
+ "tokens_seen": 324861952
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285215646940823,
+ "loss": 1.6223,
+ "theoretical_loss": 4.136483477401798,
+ "tokens_seen": 324927488
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007285055165496489,
+ "loss": 1.6171,
+ "theoretical_loss": 4.136387532931546,
+ "tokens_seen": 324993024
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284894684052157,
+ "loss": 1.555,
+ "theoretical_loss": 4.136291613222955,
+ "tokens_seen": 325058560
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284734202607823,
+ "loss": 1.5637,
+ "theoretical_loss": 4.136195718264644,
+ "tokens_seen": 325124096
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284573721163491,
+ "loss": 1.5865,
+ "theoretical_loss": 4.136099848045239,
+ "tokens_seen": 325189632
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284413239719157,
+ "loss": 1.695,
+ "theoretical_loss": 4.136004002553375,
+ "tokens_seen": 325255168
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284252758274825,
+ "loss": 1.5485,
+ "theoretical_loss": 4.135908181777693,
+ "tokens_seen": 325320704
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007284092276830491,
+ "loss": 1.6128,
+ "theoretical_loss": 4.135812385706842,
+ "tokens_seen": 325386240
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283931795386159,
+ "loss": 1.7098,
+ "theoretical_loss": 4.135716614329479,
+ "tokens_seen": 325451776
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283771313941825,
+ "loss": 1.7113,
+ "theoretical_loss": 4.135620867634269,
+ "tokens_seen": 325517312
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283610832497493,
+ "loss": 1.6371,
+ "theoretical_loss": 4.135525145609881,
+ "tokens_seen": 325582848
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283450351053159,
+ "loss": 1.7067,
+ "theoretical_loss": 4.135429448244997,
+ "tokens_seen": 325648384
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283289869608827,
+ "loss": 1.609,
+ "theoretical_loss": 4.135333775528302,
+ "tokens_seen": 325713920
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007283129388164493,
+ "loss": 1.4926,
+ "theoretical_loss": 4.13523812744849,
+ "tokens_seen": 325779456
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282968906720161,
+ "loss": 1.7475,
+ "theoretical_loss": 4.135142503994263,
+ "tokens_seen": 325844992
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282808425275827,
+ "loss": 1.539,
+ "theoretical_loss": 4.13504690515433,
+ "tokens_seen": 325910528
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282647943831495,
+ "loss": 1.6961,
+ "theoretical_loss": 4.134951330917408,
+ "tokens_seen": 325976064
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282487462387161,
+ "loss": 1.5638,
+ "theoretical_loss": 4.134855781272218,
+ "tokens_seen": 326041600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282326980942829,
+ "loss": 1.6864,
+ "theoretical_loss": 4.134760256207495,
+ "tokens_seen": 326107136
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282166499498495,
+ "loss": 1.4862,
+ "theoretical_loss": 4.1346647557119764,
+ "tokens_seen": 326172672
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007282006018054163,
+ "loss": 1.6338,
+ "theoretical_loss": 4.134569279774407,
+ "tokens_seen": 326238208
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281845536609829,
+ "loss": 1.6086,
+ "theoretical_loss": 4.134473828383541,
+ "tokens_seen": 326303744
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281685055165497,
+ "loss": 1.4595,
+ "theoretical_loss": 4.13437840152814,
+ "tokens_seen": 326369280
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281524573721163,
+ "loss": 1.6468,
+ "theoretical_loss": 4.134282999196973,
+ "tokens_seen": 326434816
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281364092276831,
+ "loss": 1.6356,
+ "theoretical_loss": 4.134187621378813,
+ "tokens_seen": 326500352
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281203610832497,
+ "loss": 1.546,
+ "theoretical_loss": 4.134092268062446,
+ "tokens_seen": 326565888
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007281043129388165,
+ "loss": 1.4938,
+ "theoretical_loss": 4.133996939236661,
+ "tokens_seen": 326631424
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280882647943831,
+ "loss": 1.6343,
+ "theoretical_loss": 4.133901634890256,
+ "tokens_seen": 326696960
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280722166499499,
+ "loss": 1.5978,
+ "theoretical_loss": 4.1338063550120365,
+ "tokens_seen": 326762496
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280561685055165,
+ "loss": 1.7451,
+ "theoretical_loss": 4.133711099590815,
+ "tokens_seen": 326828032
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280401203610833,
+ "loss": 1.6231,
+ "theoretical_loss": 4.133615868615411,
+ "tokens_seen": 326893568
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280240722166499,
+ "loss": 1.6108,
+ "theoretical_loss": 4.133520662074654,
+ "tokens_seen": 326959104
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007280080240722167,
+ "loss": 1.6044,
+ "theoretical_loss": 4.133425479957375,
+ "tokens_seen": 327024640
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279919759277833,
+ "loss": 1.6349,
+ "theoretical_loss": 4.133330322252419,
+ "tokens_seen": 327090176
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279759277833501,
+ "loss": 1.6286,
+ "theoretical_loss": 4.1332351889486345,
+ "tokens_seen": 327155712
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279598796389167,
+ "loss": 1.4971,
+ "theoretical_loss": 4.133140080034878,
+ "tokens_seen": 327221248
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279438314944835,
+ "loss": 1.5259,
+ "theoretical_loss": 4.133044995500013,
+ "tokens_seen": 327286784
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279277833500503,
+ "loss": 1.6473,
+ "theoretical_loss": 4.1329499353329116,
+ "tokens_seen": 327352320
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007279117352056169,
+ "loss": 1.5083,
+ "theoretical_loss": 4.132854899522453,
+ "tokens_seen": 327417856
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278956870611837,
+ "loss": 1.7212,
+ "theoretical_loss": 4.132759888057521,
+ "tokens_seen": 327483392
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278796389167503,
+ "loss": 1.6026,
+ "theoretical_loss": 4.13266490092701,
+ "tokens_seen": 327548928
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278635907723171,
+ "loss": 1.5403,
+ "theoretical_loss": 4.132569938119821,
+ "tokens_seen": 327614464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.7371483758339369,
+ "debugging/entropy-1-grams": 5.361149495192262,
+ "debugging/length": 581.1666666666666,
+ "debugging/num_segments": 12,
+ "debugging/score": 0.06659683474910605,
+ "debugging/score_std": 0.015715120837281396,
+ "epoch": 0.1,
+ "objective/train/docs_used": 115553,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6277862787246704,
+ "objective/train/theoretical_loss": 4.132474999624861,
+ "objective/train/tokens_used": 348140000,
+ "theoretical_loss": 4.132474999624861,
+ "tokens_seen": 327680000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278475426278837,
+ "loss": 1.6737,
+ "theoretical_loss": 4.132474999624861,
+ "tokens_seen": 327680000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278314944834505,
+ "loss": 1.5806,
+ "theoretical_loss": 4.132380085431046,
+ "tokens_seen": 327745536
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007278154463390171,
+ "loss": 1.641,
+ "theoretical_loss": 4.132285195527297,
+ "tokens_seen": 327811072
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277993981945839,
+ "loss": 1.6191,
+ "theoretical_loss": 4.132190329902544,
+ "tokens_seen": 327876608
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277833500501505,
+ "loss": 1.5115,
+ "theoretical_loss": 4.132095488545724,
+ "tokens_seen": 327942144
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277673019057173,
+ "loss": 1.6416,
+ "theoretical_loss": 4.132000671445781,
+ "tokens_seen": 328007680
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277512537612839,
+ "loss": 1.5818,
+ "theoretical_loss": 4.131905878591667,
+ "tokens_seen": 328073216
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277352056168507,
+ "loss": 1.5892,
+ "theoretical_loss": 4.13181110997234,
+ "tokens_seen": 328138752
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277191574724173,
+ "loss": 1.6217,
+ "theoretical_loss": 4.131716365576765,
+ "tokens_seen": 328204288
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007277031093279841,
+ "loss": 1.4981,
+ "theoretical_loss": 4.131621645393917,
+ "tokens_seen": 328269824
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276870611835507,
+ "loss": 1.5857,
+ "theoretical_loss": 4.131526949412774,
+ "tokens_seen": 328335360
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276710130391175,
+ "loss": 1.53,
+ "theoretical_loss": 4.131432277622325,
+ "tokens_seen": 328400896
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276549648946841,
+ "loss": 1.5972,
+ "theoretical_loss": 4.131337630011564,
+ "tokens_seen": 328466432
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276389167502509,
+ "loss": 1.5262,
+ "theoretical_loss": 4.131243006569493,
+ "tokens_seen": 328531968
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276228686058175,
+ "loss": 1.5769,
+ "theoretical_loss": 4.131148407285121,
+ "tokens_seen": 328597504
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007276068204613843,
+ "loss": 1.6371,
+ "theoretical_loss": 4.131053832147465,
+ "tokens_seen": 328663040
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007275907723169509,
+ "loss": 1.6341,
+ "theoretical_loss": 4.130959281145547,
+ "tokens_seen": 328728576
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007275747241725177,
+ "loss": 1.6886,
+ "theoretical_loss": 4.130864754268399,
+ "tokens_seen": 328794112
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007275586760280843,
+ "loss": 1.5881,
+ "theoretical_loss": 4.130770251505058,
+ "tokens_seen": 328859648
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727542627883651,
+ "loss": 1.5361,
+ "theoretical_loss": 4.130675772844569,
+ "tokens_seen": 328925184
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007275265797392177,
+ "loss": 1.6476,
+ "theoretical_loss": 4.130581318275984,
+ "tokens_seen": 328990720
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007275105315947844,
+ "loss": 1.6211,
+ "theoretical_loss": 4.130486887788362,
+ "tokens_seen": 329056256
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274944834503511,
+ "loss": 1.5303,
+ "theoretical_loss": 4.130392481370771,
+ "tokens_seen": 329121792
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274784353059178,
+ "loss": 1.5864,
+ "theoretical_loss": 4.130298099012283,
+ "tokens_seen": 329187328
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274623871614845,
+ "loss": 1.5658,
+ "theoretical_loss": 4.130203740701979,
+ "tokens_seen": 329252864
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274463390170512,
+ "loss": 1.6121,
+ "theoretical_loss": 4.1301094064289465,
+ "tokens_seen": 329318400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274302908726179,
+ "loss": 1.7239,
+ "theoretical_loss": 4.130015096182282,
+ "tokens_seen": 329383936
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007274142427281846,
+ "loss": 1.5618,
+ "theoretical_loss": 4.129920809951085,
+ "tokens_seen": 329449472
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007273981945837512,
+ "loss": 1.594,
+ "theoretical_loss": 4.129826547724467,
+ "tokens_seen": 329515008
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727382146439318,
+ "loss": 1.6085,
+ "theoretical_loss": 4.129732309491542,
+ "tokens_seen": 329580544
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007273660982948846,
+ "loss": 1.7381,
+ "theoretical_loss": 4.129638095241436,
+ "tokens_seen": 329646080
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007273500501504514,
+ "loss": 1.608,
+ "theoretical_loss": 4.129543904963278,
+ "tokens_seen": 329711616
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727334002006018,
+ "loss": 1.6218,
+ "theoretical_loss": 4.129449738646205,
+ "tokens_seen": 329777152
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007273179538615848,
+ "loss": 1.5262,
+ "theoretical_loss": 4.129355596279362,
+ "tokens_seen": 329842688
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007273019057171514,
+ "loss": 1.585,
+ "theoretical_loss": 4.129261477851902,
+ "tokens_seen": 329908224
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007272858575727182,
+ "loss": 1.517,
+ "theoretical_loss": 4.129167383352982,
+ "tokens_seen": 329973760
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007272698094282848,
+ "loss": 1.5909,
+ "theoretical_loss": 4.129073312771768,
+ "tokens_seen": 330039296
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007272537612838516,
+ "loss": 1.6328,
+ "theoretical_loss": 4.128979266097433,
+ "tokens_seen": 330104832
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007272377131394182,
+ "loss": 1.5654,
+ "theoretical_loss": 4.128885243319158,
+ "tokens_seen": 330170368
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727221664994985,
+ "loss": 1.6423,
+ "theoretical_loss": 4.12879124442613,
+ "tokens_seen": 330235904
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007272056168505516,
+ "loss": 1.5612,
+ "theoretical_loss": 4.128697269407541,
+ "tokens_seen": 330301440
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007271895687061184,
+ "loss": 1.6453,
+ "theoretical_loss": 4.128603318252593,
+ "tokens_seen": 330366976
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727173520561685,
+ "loss": 1.6937,
+ "theoretical_loss": 4.128509390950494,
+ "tokens_seen": 330432512
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007271574724172518,
+ "loss": 1.6754,
+ "theoretical_loss": 4.128415487490461,
+ "tokens_seen": 330498048
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007271414242728184,
+ "loss": 1.5799,
+ "theoretical_loss": 4.128321607861713,
+ "tokens_seen": 330563584
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007271253761283852,
+ "loss": 1.6589,
+ "theoretical_loss": 4.128227752053482,
+ "tokens_seen": 330629120
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007271093279839518,
+ "loss": 1.5626,
+ "theoretical_loss": 4.128133920055001,
+ "tokens_seen": 330694656
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007270932798395186,
+ "loss": 1.5973,
+ "theoretical_loss": 4.1280401118555154,
+ "tokens_seen": 330760192
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007270772316950852,
+ "loss": 1.5759,
+ "theoretical_loss": 4.127946327444274,
+ "tokens_seen": 330825728
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727061183550652,
+ "loss": 1.4935,
+ "theoretical_loss": 4.127852566810535,
+ "tokens_seen": 330891264
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 116426,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7715214490890503,
+ "objective/train/theoretical_loss": 4.127758829943562,
+ "objective/train/tokens_used": 351416800,
+ "theoretical_loss": 4.127758829943562,
+ "tokens_seen": 330956800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007270451354062186,
+ "loss": 1.6378,
+ "theoretical_loss": 4.127758829943562,
+ "tokens_seen": 330956800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007270290872617854,
+ "loss": 1.6115,
+ "theoretical_loss": 4.127665116832626,
+ "tokens_seen": 331022336
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000727013039117352,
+ "loss": 1.4952,
+ "theoretical_loss": 4.127571427467005,
+ "tokens_seen": 331087872
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269969909729188,
+ "loss": 1.5364,
+ "theoretical_loss": 4.1274777618359835,
+ "tokens_seen": 331153408
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269809428284854,
+ "loss": 1.5265,
+ "theoretical_loss": 4.127384119928854,
+ "tokens_seen": 331218944
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269648946840522,
+ "loss": 1.5606,
+ "theoretical_loss": 4.127290501734914,
+ "tokens_seen": 331284480
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269488465396188,
+ "loss": 1.6189,
+ "theoretical_loss": 4.127196907243472,
+ "tokens_seen": 331350016
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269327983951856,
+ "loss": 1.6621,
+ "theoretical_loss": 4.127103336443839,
+ "tokens_seen": 331415552
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007269167502507522,
+ "loss": 1.6471,
+ "theoretical_loss": 4.127009789325334,
+ "tokens_seen": 331481088
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726900702106319,
+ "loss": 1.5859,
+ "theoretical_loss": 4.126916265877284,
+ "tokens_seen": 331546624
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007268846539618856,
+ "loss": 1.5848,
+ "theoretical_loss": 4.126822766089024,
+ "tokens_seen": 331612160
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007268686058174524,
+ "loss": 1.7124,
+ "theoretical_loss": 4.126729289949893,
+ "tokens_seen": 331677696
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726852557673019,
+ "loss": 1.582,
+ "theoretical_loss": 4.126635837449238,
+ "tokens_seen": 331743232
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007268365095285858,
+ "loss": 1.5718,
+ "theoretical_loss": 4.126542408576414,
+ "tokens_seen": 331808768
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007268204613841524,
+ "loss": 1.5834,
+ "theoretical_loss": 4.126449003320783,
+ "tokens_seen": 331874304
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007268044132397192,
+ "loss": 1.5014,
+ "theoretical_loss": 4.126355621671711,
+ "tokens_seen": 331939840
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007267883650952858,
+ "loss": 1.4939,
+ "theoretical_loss": 4.126262263618574,
+ "tokens_seen": 332005376
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007267723169508526,
+ "loss": 1.5887,
+ "theoretical_loss": 4.1261689291507535,
+ "tokens_seen": 332070912
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007267562688064192,
+ "loss": 1.6777,
+ "theoretical_loss": 4.126075618257639,
+ "tokens_seen": 332136448
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726740220661986,
+ "loss": 1.6688,
+ "theoretical_loss": 4.125982330928624,
+ "tokens_seen": 332201984
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007267241725175528,
+ "loss": 1.6503,
+ "theoretical_loss": 4.125889067153113,
+ "tokens_seen": 332267520
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007267081243731194,
+ "loss": 1.5699,
+ "theoretical_loss": 4.125795826920514,
+ "tokens_seen": 332333056
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007266920762286862,
+ "loss": 1.5449,
+ "theoretical_loss": 4.125702610220244,
+ "tokens_seen": 332398592
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007266760280842528,
+ "loss": 1.6834,
+ "theoretical_loss": 4.1256094170417255,
+ "tokens_seen": 332464128
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007266599799398196,
+ "loss": 1.6401,
+ "theoretical_loss": 4.125516247374387,
+ "tokens_seen": 332529664
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007266439317953862,
+ "loss": 1.7152,
+ "theoretical_loss": 4.125423101207668,
+ "tokens_seen": 332595200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726627883650953,
+ "loss": 1.4565,
+ "theoretical_loss": 4.125329978531009,
+ "tokens_seen": 332660736
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007266118355065196,
+ "loss": 1.6387,
+ "theoretical_loss": 4.125236879333861,
+ "tokens_seen": 332726272
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007265957873620864,
+ "loss": 1.5454,
+ "theoretical_loss": 4.125143803605683,
+ "tokens_seen": 332791808
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726579739217653,
+ "loss": 1.6302,
+ "theoretical_loss": 4.125050751335936,
+ "tokens_seen": 332857344
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007265636910732198,
+ "loss": 1.6075,
+ "theoretical_loss": 4.124957722514091,
+ "tokens_seen": 332922880
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007265476429287864,
+ "loss": 1.6286,
+ "theoretical_loss": 4.124864717129629,
+ "tokens_seen": 332988416
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007265315947843532,
+ "loss": 1.4963,
+ "theoretical_loss": 4.124771735172031,
+ "tokens_seen": 333053952
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007265155466399198,
+ "loss": 1.559,
+ "theoretical_loss": 4.1246787766307875,
+ "tokens_seen": 333119488
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007264994984954866,
+ "loss": 1.6646,
+ "theoretical_loss": 4.124585841495398,
+ "tokens_seen": 333185024
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007264834503510532,
+ "loss": 1.4918,
+ "theoretical_loss": 4.124492929755367,
+ "tokens_seen": 333250560
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.00072646740220662,
+ "loss": 1.5617,
+ "theoretical_loss": 4.1244000414002056,
+ "tokens_seen": 333316096
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007264513540621866,
+ "loss": 1.6774,
+ "theoretical_loss": 4.124307176419432,
+ "tokens_seen": 333381632
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007264353059177534,
+ "loss": 1.5307,
+ "theoretical_loss": 4.12421433480257,
+ "tokens_seen": 333447168
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.00072641925777332,
+ "loss": 1.5039,
+ "theoretical_loss": 4.124121516539153,
+ "tokens_seen": 333512704
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007264032096288868,
+ "loss": 1.6459,
+ "theoretical_loss": 4.124028721618719,
+ "tokens_seen": 333578240
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007263871614844534,
+ "loss": 1.5771,
+ "theoretical_loss": 4.123935950030812,
+ "tokens_seen": 333643776
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007263711133400202,
+ "loss": 1.5749,
+ "theoretical_loss": 4.123843201764984,
+ "tokens_seen": 333709312
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007263550651955868,
+ "loss": 1.5776,
+ "theoretical_loss": 4.1237504768107955,
+ "tokens_seen": 333774848
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007263390170511536,
+ "loss": 1.6285,
+ "theoretical_loss": 4.12365777515781,
+ "tokens_seen": 333840384
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007263229689067202,
+ "loss": 1.739,
+ "theoretical_loss": 4.1235650967956,
+ "tokens_seen": 333905920
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726306920762287,
+ "loss": 1.3803,
+ "theoretical_loss": 4.123472441713743,
+ "tokens_seen": 333971456
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007262908726178536,
+ "loss": 1.6053,
+ "theoretical_loss": 4.123379809901827,
+ "tokens_seen": 334036992
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007262748244734204,
+ "loss": 1.5184,
+ "theoretical_loss": 4.123287201349443,
+ "tokens_seen": 334102528
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726258776328987,
+ "loss": 1.4989,
+ "theoretical_loss": 4.123194616046189,
+ "tokens_seen": 334168064
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 117616,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7517592906951904,
+ "objective/train/theoretical_loss": 4.123102053981672,
+ "objective/train/tokens_used": 354693600,
+ "theoretical_loss": 4.123102053981672,
+ "tokens_seen": 334233600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007262427281845538,
+ "loss": 1.6572,
+ "theoretical_loss": 4.123102053981672,
+ "tokens_seen": 334233600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007262266800401204,
+ "loss": 1.5332,
+ "theoretical_loss": 4.123009515145503,
+ "tokens_seen": 334299136
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007262106318956872,
+ "loss": 1.5541,
+ "theoretical_loss": 4.122916999527301,
+ "tokens_seen": 334364672
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007261945837512538,
+ "loss": 1.6208,
+ "theoretical_loss": 4.122824507116691,
+ "tokens_seen": 334430208
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007261785356068206,
+ "loss": 1.5833,
+ "theoretical_loss": 4.122732037903308,
+ "tokens_seen": 334495744
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007261624874623872,
+ "loss": 1.6827,
+ "theoretical_loss": 4.122639591876789,
+ "tokens_seen": 334561280
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726146439317954,
+ "loss": 1.5785,
+ "theoretical_loss": 4.122547169026778,
+ "tokens_seen": 334626816
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007261303911735206,
+ "loss": 1.6094,
+ "theoretical_loss": 4.12245476934293,
+ "tokens_seen": 334692352
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007261143430290873,
+ "loss": 1.5737,
+ "theoretical_loss": 4.122362392814903,
+ "tokens_seen": 334757888
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000726098294884654,
+ "loss": 1.5096,
+ "theoretical_loss": 4.122270039432362,
+ "tokens_seen": 334823424
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260822467402207,
+ "loss": 1.5614,
+ "theoretical_loss": 4.122177709184978,
+ "tokens_seen": 334888960
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260661985957874,
+ "loss": 1.6424,
+ "theoretical_loss": 4.1220854020624325,
+ "tokens_seen": 334954496
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260501504513541,
+ "loss": 1.6124,
+ "theoretical_loss": 4.121993118054409,
+ "tokens_seen": 335020032
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260341023069208,
+ "loss": 1.5596,
+ "theoretical_loss": 4.1219008571505995,
+ "tokens_seen": 335085568
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260180541624875,
+ "loss": 1.6681,
+ "theoretical_loss": 4.121808619340704,
+ "tokens_seen": 335151104
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007260020060180541,
+ "loss": 1.5883,
+ "theoretical_loss": 4.121716404614426,
+ "tokens_seen": 335216640
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259859578736209,
+ "loss": 1.4965,
+ "theoretical_loss": 4.121624212961478,
+ "tokens_seen": 335282176
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259699097291875,
+ "loss": 1.6037,
+ "theoretical_loss": 4.121532044371579,
+ "tokens_seen": 335347712
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259538615847543,
+ "loss": 1.7225,
+ "theoretical_loss": 4.121439898834454,
+ "tokens_seen": 335413248
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259378134403209,
+ "loss": 1.69,
+ "theoretical_loss": 4.1213477763398325,
+ "tokens_seen": 335478784
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259217652958877,
+ "loss": 1.5917,
+ "theoretical_loss": 4.121255676877455,
+ "tokens_seen": 335544320
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007259057171514543,
+ "loss": 1.7592,
+ "theoretical_loss": 4.1211636004370655,
+ "tokens_seen": 335609856
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258896690070211,
+ "loss": 1.722,
+ "theoretical_loss": 4.121071547008415,
+ "tokens_seen": 335675392
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258736208625877,
+ "loss": 1.5877,
+ "theoretical_loss": 4.120979516581262,
+ "tokens_seen": 335740928
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258575727181545,
+ "loss": 1.5069,
+ "theoretical_loss": 4.120887509145371,
+ "tokens_seen": 335806464
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258415245737211,
+ "loss": 1.5241,
+ "theoretical_loss": 4.120795524690512,
+ "tokens_seen": 335872000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258254764292879,
+ "loss": 1.5952,
+ "theoretical_loss": 4.1207035632064635,
+ "tokens_seen": 335937536
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007258094282848545,
+ "loss": 1.4847,
+ "theoretical_loss": 4.120611624683008,
+ "tokens_seen": 336003072
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257933801404213,
+ "loss": 1.7648,
+ "theoretical_loss": 4.120519709109938,
+ "tokens_seen": 336068608
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257773319959879,
+ "loss": 1.5907,
+ "theoretical_loss": 4.12042781647705,
+ "tokens_seen": 336134144
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257612838515547,
+ "loss": 1.4965,
+ "theoretical_loss": 4.120335946774147,
+ "tokens_seen": 336199680
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257452357071213,
+ "loss": 1.6617,
+ "theoretical_loss": 4.12024409999104,
+ "tokens_seen": 336265216
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257291875626881,
+ "loss": 1.5381,
+ "theoretical_loss": 4.120152276117545,
+ "tokens_seen": 336330752
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007257131394182547,
+ "loss": 1.5592,
+ "theoretical_loss": 4.120060475143486,
+ "tokens_seen": 336396288
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256970912738215,
+ "loss": 1.5193,
+ "theoretical_loss": 4.119968697058692,
+ "tokens_seen": 336461824
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256810431293881,
+ "loss": 1.5876,
+ "theoretical_loss": 4.119876941852999,
+ "tokens_seen": 336527360
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256649949849549,
+ "loss": 1.652,
+ "theoretical_loss": 4.11978520951625,
+ "tokens_seen": 336592896
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256489468405215,
+ "loss": 1.5175,
+ "theoretical_loss": 4.119693500038294,
+ "tokens_seen": 336658432
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256328986960883,
+ "loss": 1.5785,
+ "theoretical_loss": 4.119601813408988,
+ "tokens_seen": 336723968
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256168505516549,
+ "loss": 1.6262,
+ "theoretical_loss": 4.1195101496181925,
+ "tokens_seen": 336789504
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007256008024072217,
+ "loss": 1.5974,
+ "theoretical_loss": 4.119418508655777,
+ "tokens_seen": 336855040
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255847542627883,
+ "loss": 1.5781,
+ "theoretical_loss": 4.119326890511615,
+ "tokens_seen": 336920576
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255687061183551,
+ "loss": 1.4977,
+ "theoretical_loss": 4.1192352951755895,
+ "tokens_seen": 336986112
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255526579739217,
+ "loss": 1.5431,
+ "theoretical_loss": 4.119143722637588,
+ "tokens_seen": 337051648
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255366098294885,
+ "loss": 1.5899,
+ "theoretical_loss": 4.119052172887506,
+ "tokens_seen": 337117184
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255205616850552,
+ "loss": 1.6135,
+ "theoretical_loss": 4.118960645915243,
+ "tokens_seen": 337182720
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007255045135406219,
+ "loss": 1.6049,
+ "theoretical_loss": 4.118869141710706,
+ "tokens_seen": 337248256
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007254884653961886,
+ "loss": 1.6426,
+ "theoretical_loss": 4.11877766026381,
+ "tokens_seen": 337313792
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007254724172517553,
+ "loss": 1.5858,
+ "theoretical_loss": 4.118686201564475,
+ "tokens_seen": 337379328
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000725456369107322,
+ "loss": 1.7204,
+ "theoretical_loss": 4.1185947656026265,
+ "tokens_seen": 337444864
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 118525,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.7721240520477295,
+ "objective/train/theoretical_loss": 4.118503352368197,
+ "objective/train/tokens_used": 357970400,
+ "theoretical_loss": 4.118503352368197,
+ "tokens_seen": 337510400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007254403209628887,
+ "loss": 1.6626,
+ "theoretical_loss": 4.118503352368197,
+ "tokens_seen": 337510400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007254242728184554,
+ "loss": 1.506,
+ "theoretical_loss": 4.118411961851129,
+ "tokens_seen": 337575936
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007254082246740221,
+ "loss": 1.5272,
+ "theoretical_loss": 4.118320594041366,
+ "tokens_seen": 337641472
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253921765295888,
+ "loss": 1.5772,
+ "theoretical_loss": 4.118229248928859,
+ "tokens_seen": 337707008
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253761283851555,
+ "loss": 1.5166,
+ "theoretical_loss": 4.11813792650357,
+ "tokens_seen": 337772544
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253600802407223,
+ "loss": 1.6504,
+ "theoretical_loss": 4.118046626755461,
+ "tokens_seen": 337838080
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253440320962889,
+ "loss": 1.543,
+ "theoretical_loss": 4.117955349674505,
+ "tokens_seen": 337903616
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253279839518557,
+ "loss": 1.6382,
+ "theoretical_loss": 4.117864095250679,
+ "tokens_seen": 337969152
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007253119358074223,
+ "loss": 1.5598,
+ "theoretical_loss": 4.117772863473968,
+ "tokens_seen": 338034688
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000725295887662989,
+ "loss": 1.626,
+ "theoretical_loss": 4.117681654334361,
+ "tokens_seen": 338100224
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007252798395185557,
+ "loss": 1.5688,
+ "theoretical_loss": 4.117590467821857,
+ "tokens_seen": 338165760
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007252637913741225,
+ "loss": 1.5726,
+ "theoretical_loss": 4.117499303926457,
+ "tokens_seen": 338231296
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007252477432296891,
+ "loss": 1.5728,
+ "theoretical_loss": 4.1174081626381716,
+ "tokens_seen": 338296832
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007252316950852559,
+ "loss": 1.5736,
+ "theoretical_loss": 4.117317043947017,
+ "tokens_seen": 338362368
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007252156469408225,
+ "loss": 1.6962,
+ "theoretical_loss": 4.117225947843014,
+ "tokens_seen": 338427904
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251995987963893,
+ "loss": 1.5178,
+ "theoretical_loss": 4.117134874316194,
+ "tokens_seen": 338493440
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251835506519559,
+ "loss": 1.5646,
+ "theoretical_loss": 4.117043823356589,
+ "tokens_seen": 338558976
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251675025075227,
+ "loss": 1.7531,
+ "theoretical_loss": 4.116952794954242,
+ "tokens_seen": 338624512
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251514543630893,
+ "loss": 1.4881,
+ "theoretical_loss": 4.1168617890992,
+ "tokens_seen": 338690048
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251354062186561,
+ "loss": 1.5268,
+ "theoretical_loss": 4.116770805781517,
+ "tokens_seen": 338755584
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251193580742227,
+ "loss": 1.404,
+ "theoretical_loss": 4.116679844991253,
+ "tokens_seen": 338821120
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007251033099297895,
+ "loss": 1.6254,
+ "theoretical_loss": 4.116588906718475,
+ "tokens_seen": 338886656
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250872617853561,
+ "loss": 1.5061,
+ "theoretical_loss": 4.116497990953255,
+ "tokens_seen": 338952192
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250712136409229,
+ "loss": 1.4944,
+ "theoretical_loss": 4.1164070976856735,
+ "tokens_seen": 339017728
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250551654964895,
+ "loss": 1.6102,
+ "theoretical_loss": 4.1163162269058144,
+ "tokens_seen": 339083264
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250391173520563,
+ "loss": 1.5782,
+ "theoretical_loss": 4.11622537860377,
+ "tokens_seen": 339148800
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250230692076229,
+ "loss": 1.7109,
+ "theoretical_loss": 4.116134552769638,
+ "tokens_seen": 339214336
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007250070210631897,
+ "loss": 1.6032,
+ "theoretical_loss": 4.116043749393523,
+ "tokens_seen": 339279872
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249909729187563,
+ "loss": 1.533,
+ "theoretical_loss": 4.115952968465535,
+ "tokens_seen": 339345408
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249749247743231,
+ "loss": 1.4966,
+ "theoretical_loss": 4.115862209975791,
+ "tokens_seen": 339410944
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249588766298897,
+ "loss": 1.5367,
+ "theoretical_loss": 4.115771473914415,
+ "tokens_seen": 339476480
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249428284854565,
+ "loss": 1.6097,
+ "theoretical_loss": 4.115680760271535,
+ "tokens_seen": 339542016
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249267803410231,
+ "loss": 1.67,
+ "theoretical_loss": 4.115590069037287,
+ "tokens_seen": 339607552
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007249107321965899,
+ "loss": 1.7401,
+ "theoretical_loss": 4.115499400201813,
+ "tokens_seen": 339673088
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248946840521565,
+ "loss": 1.6455,
+ "theoretical_loss": 4.1154087537552595,
+ "tokens_seen": 339738624
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248786359077233,
+ "loss": 1.6652,
+ "theoretical_loss": 4.1153181296877825,
+ "tokens_seen": 339804160
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248625877632899,
+ "loss": 1.4987,
+ "theoretical_loss": 4.115227527989543,
+ "tokens_seen": 339869696
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248465396188567,
+ "loss": 1.6819,
+ "theoretical_loss": 4.115136948650705,
+ "tokens_seen": 339935232
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248304914744233,
+ "loss": 1.512,
+ "theoretical_loss": 4.115046391661443,
+ "tokens_seen": 340000768
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007248144433299901,
+ "loss": 1.5822,
+ "theoretical_loss": 4.114955857011937,
+ "tokens_seen": 340066304
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247983951855567,
+ "loss": 1.6232,
+ "theoretical_loss": 4.114865344692371,
+ "tokens_seen": 340131840
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247823470411235,
+ "loss": 1.5821,
+ "theoretical_loss": 4.1147748546929375,
+ "tokens_seen": 340197376
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247662988966901,
+ "loss": 1.484,
+ "theoretical_loss": 4.114684387003833,
+ "tokens_seen": 340262912
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247502507522569,
+ "loss": 1.6424,
+ "theoretical_loss": 4.114593941615262,
+ "tokens_seen": 340328448
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247342026078235,
+ "loss": 1.4232,
+ "theoretical_loss": 4.114503518517435,
+ "tokens_seen": 340393984
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247181544633902,
+ "loss": 1.6698,
+ "theoretical_loss": 4.114413117700568,
+ "tokens_seen": 340459520
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007247021063189569,
+ "loss": 1.6268,
+ "theoretical_loss": 4.114322739154883,
+ "tokens_seen": 340525056
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007246860581745236,
+ "loss": 1.7974,
+ "theoretical_loss": 4.114232382870609,
+ "tokens_seen": 340590592
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007246700100300903,
+ "loss": 1.6961,
+ "theoretical_loss": 4.114142048837982,
+ "tokens_seen": 340656128
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724653961885657,
+ "loss": 1.6857,
+ "theoretical_loss": 4.114051737047241,
+ "tokens_seen": 340721664
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 119921,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6314363479614258,
+ "objective/train/theoretical_loss": 4.1139614474886335,
+ "objective/train/tokens_used": 361247200,
+ "theoretical_loss": 4.1139614474886335,
+ "tokens_seen": 340787200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007246379137412237,
+ "loss": 1.5892,
+ "theoretical_loss": 4.1139614474886335,
+ "tokens_seen": 340787200
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007246218655967904,
+ "loss": 1.6219,
+ "theoretical_loss": 4.1138711801524135,
+ "tokens_seen": 340852736
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007246058174523571,
+ "loss": 1.5791,
+ "theoretical_loss": 4.11378093502884,
+ "tokens_seen": 340918272
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245897693079238,
+ "loss": 1.498,
+ "theoretical_loss": 4.113690712108179,
+ "tokens_seen": 340983808
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245737211634904,
+ "loss": 1.5305,
+ "theoretical_loss": 4.113600511380701,
+ "tokens_seen": 341049344
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245576730190572,
+ "loss": 1.4932,
+ "theoretical_loss": 4.113510332836685,
+ "tokens_seen": 341114880
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245416248746238,
+ "loss": 1.5771,
+ "theoretical_loss": 4.1134201764664144,
+ "tokens_seen": 341180416
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245255767301906,
+ "loss": 1.6052,
+ "theoretical_loss": 4.113330042260181,
+ "tokens_seen": 341245952
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007245095285857572,
+ "loss": 1.4641,
+ "theoretical_loss": 4.113239930208278,
+ "tokens_seen": 341311488
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724493480441324,
+ "loss": 1.6998,
+ "theoretical_loss": 4.113149840301009,
+ "tokens_seen": 341377024
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007244774322968906,
+ "loss": 1.6401,
+ "theoretical_loss": 4.113059772528683,
+ "tokens_seen": 341442560
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007244613841524574,
+ "loss": 1.5859,
+ "theoretical_loss": 4.112969726881613,
+ "tokens_seen": 341508096
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724445336008024,
+ "loss": 1.6477,
+ "theoretical_loss": 4.112879703350122,
+ "tokens_seen": 341573632
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007244292878635908,
+ "loss": 1.701,
+ "theoretical_loss": 4.112789701924535,
+ "tokens_seen": 341639168
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007244132397191574,
+ "loss": 1.5066,
+ "theoretical_loss": 4.112699722595185,
+ "tokens_seen": 341704704
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243971915747242,
+ "loss": 1.5769,
+ "theoretical_loss": 4.11260976535241,
+ "tokens_seen": 341770240
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243811434302908,
+ "loss": 1.613,
+ "theoretical_loss": 4.112519830186557,
+ "tokens_seen": 341835776
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243650952858576,
+ "loss": 1.5651,
+ "theoretical_loss": 4.112429917087974,
+ "tokens_seen": 341901312
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243490471414242,
+ "loss": 1.6294,
+ "theoretical_loss": 4.112340026047022,
+ "tokens_seen": 341966848
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724332998996991,
+ "loss": 1.5895,
+ "theoretical_loss": 4.112250157054061,
+ "tokens_seen": 342032384
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243169508525577,
+ "loss": 1.8001,
+ "theoretical_loss": 4.112160310099461,
+ "tokens_seen": 342097920
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007243009027081244,
+ "loss": 1.5778,
+ "theoretical_loss": 4.112070485173598,
+ "tokens_seen": 342163456
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242848545636911,
+ "loss": 1.6557,
+ "theoretical_loss": 4.111980682266853,
+ "tokens_seen": 342228992
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242688064192578,
+ "loss": 1.5965,
+ "theoretical_loss": 4.111890901369612,
+ "tokens_seen": 342294528
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242527582748245,
+ "loss": 1.6546,
+ "theoretical_loss": 4.11180114247227,
+ "tokens_seen": 342360064
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242367101303912,
+ "loss": 1.6265,
+ "theoretical_loss": 4.1117114055652255,
+ "tokens_seen": 342425600
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242206619859579,
+ "loss": 1.5941,
+ "theoretical_loss": 4.111621690638884,
+ "tokens_seen": 342491136
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007242046138415246,
+ "loss": 1.4372,
+ "theoretical_loss": 4.111531997683658,
+ "tokens_seen": 342556672
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007241885656970913,
+ "loss": 1.5377,
+ "theoretical_loss": 4.1114423266899625,
+ "tokens_seen": 342622208
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724172517552658,
+ "loss": 1.6013,
+ "theoretical_loss": 4.111352677648224,
+ "tokens_seen": 342687744
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007241564694082247,
+ "loss": 1.5588,
+ "theoretical_loss": 4.111263050548869,
+ "tokens_seen": 342753280
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007241404212637914,
+ "loss": 1.7092,
+ "theoretical_loss": 4.111173445382335,
+ "tokens_seen": 342818816
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007241243731193581,
+ "loss": 1.5877,
+ "theoretical_loss": 4.111083862139063,
+ "tokens_seen": 342884352
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007241083249749248,
+ "loss": 1.569,
+ "theoretical_loss": 4.110994300809501,
+ "tokens_seen": 342949888
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007240922768304915,
+ "loss": 1.5495,
+ "theoretical_loss": 4.1109047613841,
+ "tokens_seen": 343015424
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007240762286860582,
+ "loss": 1.5503,
+ "theoretical_loss": 4.110815243853323,
+ "tokens_seen": 343080960
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724060180541625,
+ "loss": 1.6105,
+ "theoretical_loss": 4.110725748207633,
+ "tokens_seen": 343146496
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007240441323971916,
+ "loss": 1.4616,
+ "theoretical_loss": 4.110636274437502,
+ "tokens_seen": 343212032
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007240280842527583,
+ "loss": 1.6878,
+ "theoretical_loss": 4.110546822533407,
+ "tokens_seen": 343277568
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000724012036108325,
+ "loss": 1.6503,
+ "theoretical_loss": 4.110457392485832,
+ "tokens_seen": 343343104
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239959879638917,
+ "loss": 1.7304,
+ "theoretical_loss": 4.110367984285265,
+ "tokens_seen": 343408640
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239799398194584,
+ "loss": 1.6316,
+ "theoretical_loss": 4.1102785979222025,
+ "tokens_seen": 343474176
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239638916750252,
+ "loss": 1.6789,
+ "theoretical_loss": 4.110189233387146,
+ "tokens_seen": 343539712
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239478435305918,
+ "loss": 1.4762,
+ "theoretical_loss": 4.1100998906706,
+ "tokens_seen": 343605248
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239317953861586,
+ "loss": 1.6151,
+ "theoretical_loss": 4.110010569763081,
+ "tokens_seen": 343670784
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007239157472417252,
+ "loss": 1.5582,
+ "theoretical_loss": 4.109921270655106,
+ "tokens_seen": 343736320
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723899699097292,
+ "loss": 1.5535,
+ "theoretical_loss": 4.109831993337201,
+ "tokens_seen": 343801856
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007238836509528586,
+ "loss": 1.6652,
+ "theoretical_loss": 4.109742737799895,
+ "tokens_seen": 343867392
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007238676028084254,
+ "loss": 1.5522,
+ "theoretical_loss": 4.109653504033726,
+ "tokens_seen": 343932928
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723851554663992,
+ "loss": 1.4251,
+ "theoretical_loss": 4.109564292029235,
+ "tokens_seen": 343998464
+ },
+ {
+ "epoch": 0.1,
+ "objective/train/docs_used": 121082,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.569899082183838,
+ "objective/train/theoretical_loss": 4.109475101776974,
+ "objective/train/tokens_used": 364524000,
+ "theoretical_loss": 4.109475101776974,
+ "tokens_seen": 344064000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007238355065195588,
+ "loss": 1.6178,
+ "theoretical_loss": 4.109475101776974,
+ "tokens_seen": 344064000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007238194583751254,
+ "loss": 1.6877,
+ "theoretical_loss": 4.109385933267495,
+ "tokens_seen": 344129536
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007238034102306922,
+ "loss": 1.4787,
+ "theoretical_loss": 4.109296786491359,
+ "tokens_seen": 344195072
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007237873620862588,
+ "loss": 1.5375,
+ "theoretical_loss": 4.1092076614391315,
+ "tokens_seen": 344260608
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007237713139418256,
+ "loss": 1.7169,
+ "theoretical_loss": 4.109118558101386,
+ "tokens_seen": 344326144
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007237552657973922,
+ "loss": 1.5425,
+ "theoretical_loss": 4.1090294764687,
+ "tokens_seen": 344391680
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723739217652959,
+ "loss": 1.6181,
+ "theoretical_loss": 4.108940416531656,
+ "tokens_seen": 344457216
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007237231695085256,
+ "loss": 1.5927,
+ "theoretical_loss": 4.108851378280846,
+ "tokens_seen": 344522752
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007237071213640924,
+ "loss": 1.7046,
+ "theoretical_loss": 4.108762361706865,
+ "tokens_seen": 344588288
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723691073219659,
+ "loss": 1.6399,
+ "theoretical_loss": 4.108673366800313,
+ "tokens_seen": 344653824
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007236750250752258,
+ "loss": 1.5645,
+ "theoretical_loss": 4.1085843935518,
+ "tokens_seen": 344719360
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007236589769307924,
+ "loss": 1.4864,
+ "theoretical_loss": 4.108495441951937,
+ "tokens_seen": 344784896
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007236429287863592,
+ "loss": 1.6584,
+ "theoretical_loss": 4.108406511991344,
+ "tokens_seen": 344850432
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007236268806419258,
+ "loss": 1.5342,
+ "theoretical_loss": 4.108317603660646,
+ "tokens_seen": 344915968
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007236108324974926,
+ "loss": 1.6394,
+ "theoretical_loss": 4.108228716950474,
+ "tokens_seen": 344981504
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007235947843530592,
+ "loss": 1.5845,
+ "theoretical_loss": 4.108139851851464,
+ "tokens_seen": 345047040
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723578736208626,
+ "loss": 1.4446,
+ "theoretical_loss": 4.108051008354258,
+ "tokens_seen": 345112576
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007235626880641926,
+ "loss": 1.5017,
+ "theoretical_loss": 4.107962186449506,
+ "tokens_seen": 345178112
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007235466399197594,
+ "loss": 1.5043,
+ "theoretical_loss": 4.107873386127861,
+ "tokens_seen": 345243648
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723530591775326,
+ "loss": 1.4736,
+ "theoretical_loss": 4.107784607379983,
+ "tokens_seen": 345309184
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007235145436308928,
+ "loss": 1.4901,
+ "theoretical_loss": 4.107695850196539,
+ "tokens_seen": 345374720
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234984954864594,
+ "loss": 1.5762,
+ "theoretical_loss": 4.107607114568197,
+ "tokens_seen": 345440256
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234824473420262,
+ "loss": 1.5355,
+ "theoretical_loss": 4.107518400485638,
+ "tokens_seen": 345505792
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234663991975928,
+ "loss": 1.6054,
+ "theoretical_loss": 4.107429707939545,
+ "tokens_seen": 345571328
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234503510531596,
+ "loss": 1.6409,
+ "theoretical_loss": 4.107341036920604,
+ "tokens_seen": 345636864
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234343029087262,
+ "loss": 1.5958,
+ "theoretical_loss": 4.107252387419513,
+ "tokens_seen": 345702400
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723418254764293,
+ "loss": 1.6453,
+ "theoretical_loss": 4.107163759426971,
+ "tokens_seen": 345767936
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007234022066198596,
+ "loss": 1.6439,
+ "theoretical_loss": 4.1070751529336835,
+ "tokens_seen": 345833472
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007233861584754264,
+ "loss": 1.5924,
+ "theoretical_loss": 4.106986567930364,
+ "tokens_seen": 345899008
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.000723370110330993,
+ "loss": 1.6914,
+ "theoretical_loss": 4.1068980044077295,
+ "tokens_seen": 345964544
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007233540621865598,
+ "loss": 1.5754,
+ "theoretical_loss": 4.1068094623565035,
+ "tokens_seen": 346030080
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007233380140421264,
+ "loss": 1.5374,
+ "theoretical_loss": 4.106720941767417,
+ "tokens_seen": 346095616
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007233219658976931,
+ "loss": 1.6939,
+ "theoretical_loss": 4.106632442631203,
+ "tokens_seen": 346161152
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007233059177532598,
+ "loss": 1.6309,
+ "theoretical_loss": 4.106543964938604,
+ "tokens_seen": 346226688
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007232898696088265,
+ "loss": 1.5436,
+ "theoretical_loss": 4.106455508680366,
+ "tokens_seen": 346292224
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007232738214643932,
+ "loss": 1.5128,
+ "theoretical_loss": 4.106367073847241,
+ "tokens_seen": 346357760
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007232577733199599,
+ "loss": 1.4556,
+ "theoretical_loss": 4.106278660429988,
+ "tokens_seen": 346423296
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 0.0007232417251755266,
+ "loss": 1.5254,
+ "theoretical_loss": 4.10619026841937,
+ "tokens_seen": 346488832
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007232256770310933,
+ "loss": 1.7047,
+ "theoretical_loss": 4.106101897806156,
+ "tokens_seen": 346554368
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00072320962888666,
+ "loss": 1.5841,
+ "theoretical_loss": 4.1060135485811236,
+ "tokens_seen": 346619904
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231935807422267,
+ "loss": 1.6565,
+ "theoretical_loss": 4.105925220735052,
+ "tokens_seen": 346685440
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231775325977933,
+ "loss": 1.5367,
+ "theoretical_loss": 4.105836914258728,
+ "tokens_seen": 346750976
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231614844533601,
+ "loss": 1.5487,
+ "theoretical_loss": 4.105748629142945,
+ "tokens_seen": 346816512
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231454363089267,
+ "loss": 1.5987,
+ "theoretical_loss": 4.1056603653785,
+ "tokens_seen": 346882048
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231293881644935,
+ "loss": 1.6058,
+ "theoretical_loss": 4.1055721229561986,
+ "tokens_seen": 346947584
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007231133400200602,
+ "loss": 1.6939,
+ "theoretical_loss": 4.105483901866848,
+ "tokens_seen": 347013120
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230972918756269,
+ "loss": 1.628,
+ "theoretical_loss": 4.105395702101266,
+ "tokens_seen": 347078656
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230812437311936,
+ "loss": 1.6857,
+ "theoretical_loss": 4.1053075236502705,
+ "tokens_seen": 347144192
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230651955867603,
+ "loss": 1.535,
+ "theoretical_loss": 4.1052193665046905,
+ "tokens_seen": 347209728
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000723049147442327,
+ "loss": 1.5932,
+ "theoretical_loss": 4.105131230655358,
+ "tokens_seen": 347275264
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 122020,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6148931980133057,
+ "objective/train/theoretical_loss": 4.1050431160931105,
+ "objective/train/tokens_used": 367800800,
+ "theoretical_loss": 4.1050431160931105,
+ "tokens_seen": 347340800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230330992978937,
+ "loss": 1.5267,
+ "theoretical_loss": 4.1050431160931105,
+ "tokens_seen": 347340800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230170511534604,
+ "loss": 1.6963,
+ "theoretical_loss": 4.104955022808792,
+ "tokens_seen": 347406336
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007230010030090271,
+ "loss": 1.597,
+ "theoretical_loss": 4.104866950793251,
+ "tokens_seen": 347471872
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229849548645938,
+ "loss": 1.6458,
+ "theoretical_loss": 4.104778900037343,
+ "tokens_seen": 347537408
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229689067201605,
+ "loss": 1.5932,
+ "theoretical_loss": 4.104690870531929,
+ "tokens_seen": 347602944
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229528585757272,
+ "loss": 1.6048,
+ "theoretical_loss": 4.104602862267875,
+ "tokens_seen": 347668480
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229368104312939,
+ "loss": 1.5327,
+ "theoretical_loss": 4.1045148752360525,
+ "tokens_seen": 347734016
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229207622868606,
+ "loss": 1.6515,
+ "theoretical_loss": 4.104426909427339,
+ "tokens_seen": 347799552
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007229047141424273,
+ "loss": 1.7427,
+ "theoretical_loss": 4.104338964832619,
+ "tokens_seen": 347865088
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000722888665997994,
+ "loss": 1.4355,
+ "theoretical_loss": 4.10425104144278,
+ "tokens_seen": 347930624
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007228726178535607,
+ "loss": 1.5693,
+ "theoretical_loss": 4.104163139248717,
+ "tokens_seen": 347996160
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007228565697091274,
+ "loss": 1.7103,
+ "theoretical_loss": 4.1040752582413305,
+ "tokens_seen": 348061696
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007228405215646941,
+ "loss": 1.5576,
+ "theoretical_loss": 4.103987398411524,
+ "tokens_seen": 348127232
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007228244734202608,
+ "loss": 1.5204,
+ "theoretical_loss": 4.103899559750213,
+ "tokens_seen": 348192768
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007228084252758275,
+ "loss": 1.5165,
+ "theoretical_loss": 4.10381174224831,
+ "tokens_seen": 348258304
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007227923771313942,
+ "loss": 1.4986,
+ "theoretical_loss": 4.10372394589674,
+ "tokens_seen": 348323840
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007227763289869609,
+ "loss": 1.6011,
+ "theoretical_loss": 4.10363617068643,
+ "tokens_seen": 348389376
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007227602808425276,
+ "loss": 1.5675,
+ "theoretical_loss": 4.103548416608316,
+ "tokens_seen": 348454912
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007227442326980943,
+ "loss": 1.6869,
+ "theoretical_loss": 4.103460683653335,
+ "tokens_seen": 348520448
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000722728184553661,
+ "loss": 1.5663,
+ "theoretical_loss": 4.103372971812432,
+ "tokens_seen": 348585984
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007227121364092277,
+ "loss": 1.605,
+ "theoretical_loss": 4.103285281076558,
+ "tokens_seen": 348651520
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226960882647944,
+ "loss": 1.5792,
+ "theoretical_loss": 4.10319761143667,
+ "tokens_seen": 348717056
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226800401203611,
+ "loss": 1.5252,
+ "theoretical_loss": 4.103109962883728,
+ "tokens_seen": 348782592
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226639919759278,
+ "loss": 1.57,
+ "theoretical_loss": 4.1030223354087,
+ "tokens_seen": 348848128
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226479438314945,
+ "loss": 1.5449,
+ "theoretical_loss": 4.102934729002559,
+ "tokens_seen": 348913664
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226318956870612,
+ "loss": 1.4946,
+ "theoretical_loss": 4.102847143656284,
+ "tokens_seen": 348979200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007226158475426279,
+ "loss": 1.548,
+ "theoretical_loss": 4.102759579360857,
+ "tokens_seen": 349044736
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225997993981947,
+ "loss": 1.6362,
+ "theoretical_loss": 4.102672036107269,
+ "tokens_seen": 349110272
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225837512537613,
+ "loss": 1.5507,
+ "theoretical_loss": 4.102584513886514,
+ "tokens_seen": 349175808
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000722567703109328,
+ "loss": 1.5163,
+ "theoretical_loss": 4.102497012689594,
+ "tokens_seen": 349241344
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225516549648947,
+ "loss": 1.6004,
+ "theoretical_loss": 4.102409532507513,
+ "tokens_seen": 349306880
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225356068204615,
+ "loss": 1.4954,
+ "theoretical_loss": 4.102322073331285,
+ "tokens_seen": 349372416
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225195586760281,
+ "loss": 1.4697,
+ "theoretical_loss": 4.102234635151925,
+ "tokens_seen": 349437952
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007225035105315949,
+ "loss": 1.5609,
+ "theoretical_loss": 4.102147217960457,
+ "tokens_seen": 349503488
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224874623871615,
+ "loss": 1.5016,
+ "theoretical_loss": 4.1020598217479085,
+ "tokens_seen": 349569024
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224714142427283,
+ "loss": 1.6255,
+ "theoretical_loss": 4.101972446505314,
+ "tokens_seen": 349634560
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224553660982949,
+ "loss": 1.5068,
+ "theoretical_loss": 4.101885092223712,
+ "tokens_seen": 349700096
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224393179538617,
+ "loss": 1.6375,
+ "theoretical_loss": 4.101797758894147,
+ "tokens_seen": 349765632
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224232698094283,
+ "loss": 1.5521,
+ "theoretical_loss": 4.101710446507669,
+ "tokens_seen": 349831168
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007224072216649951,
+ "loss": 1.5221,
+ "theoretical_loss": 4.101623155055336,
+ "tokens_seen": 349896704
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223911735205617,
+ "loss": 1.5779,
+ "theoretical_loss": 4.101535884528206,
+ "tokens_seen": 349962240
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223751253761285,
+ "loss": 1.5925,
+ "theoretical_loss": 4.1014486349173485,
+ "tokens_seen": 350027776
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223590772316951,
+ "loss": 1.6624,
+ "theoretical_loss": 4.101361406213834,
+ "tokens_seen": 350093312
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223430290872619,
+ "loss": 1.5511,
+ "theoretical_loss": 4.101274198408741,
+ "tokens_seen": 350158848
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223269809428285,
+ "loss": 1.6351,
+ "theoretical_loss": 4.101187011493153,
+ "tokens_seen": 350224384
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007223109327983953,
+ "loss": 1.6788,
+ "theoretical_loss": 4.101099845458156,
+ "tokens_seen": 350289920
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222948846539619,
+ "loss": 1.6737,
+ "theoretical_loss": 4.101012700294848,
+ "tokens_seen": 350355456
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222788365095287,
+ "loss": 1.6357,
+ "theoretical_loss": 4.100925575994325,
+ "tokens_seen": 350420992
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222627883650953,
+ "loss": 1.5646,
+ "theoretical_loss": 4.100838472547695,
+ "tokens_seen": 350486528
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222467402206621,
+ "loss": 1.5503,
+ "theoretical_loss": 4.100751389946067,
+ "tokens_seen": 350552064
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 123333,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5236725807189941,
+ "objective/train/theoretical_loss": 4.100664328180557,
+ "objective/train/tokens_used": 371077600,
+ "theoretical_loss": 4.100664328180557,
+ "tokens_seen": 350617600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222306920762287,
+ "loss": 1.5919,
+ "theoretical_loss": 4.100664328180557,
+ "tokens_seen": 350617600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007222146439317955,
+ "loss": 1.6033,
+ "theoretical_loss": 4.100577287242286,
+ "tokens_seen": 350683136
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221985957873621,
+ "loss": 1.4229,
+ "theoretical_loss": 4.100490267122381,
+ "tokens_seen": 350748672
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221825476429289,
+ "loss": 1.4672,
+ "theoretical_loss": 4.100403267811975,
+ "tokens_seen": 350814208
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221664994984955,
+ "loss": 1.6758,
+ "theoretical_loss": 4.100316289302206,
+ "tokens_seen": 350879744
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221504513540623,
+ "loss": 1.5765,
+ "theoretical_loss": 4.100229331584215,
+ "tokens_seen": 350945280
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221344032096289,
+ "loss": 1.6201,
+ "theoretical_loss": 4.100142394649152,
+ "tokens_seen": 351010816
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221183550651957,
+ "loss": 1.5798,
+ "theoretical_loss": 4.100055478488171,
+ "tokens_seen": 351076352
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007221023069207623,
+ "loss": 1.5945,
+ "theoretical_loss": 4.099968583092431,
+ "tokens_seen": 351141888
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220862587763291,
+ "loss": 1.7934,
+ "theoretical_loss": 4.099881708453097,
+ "tokens_seen": 351207424
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220702106318957,
+ "loss": 1.5929,
+ "theoretical_loss": 4.099794854561339,
+ "tokens_seen": 351272960
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220541624874625,
+ "loss": 1.5479,
+ "theoretical_loss": 4.0997080214083335,
+ "tokens_seen": 351338496
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220381143430291,
+ "loss": 1.5969,
+ "theoretical_loss": 4.09962120898526,
+ "tokens_seen": 351404032
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220220661985959,
+ "loss": 1.8095,
+ "theoretical_loss": 4.099534417283305,
+ "tokens_seen": 351469568
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007220060180541625,
+ "loss": 1.5545,
+ "theoretical_loss": 4.099447646293662,
+ "tokens_seen": 351535104
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007219899699097293,
+ "loss": 1.4616,
+ "theoretical_loss": 4.099360896007527,
+ "tokens_seen": 351600640
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007219739217652959,
+ "loss": 1.5338,
+ "theoretical_loss": 4.099274166416102,
+ "tokens_seen": 351666176
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007219578736208627,
+ "loss": 1.5553,
+ "theoretical_loss": 4.099187457510595,
+ "tokens_seen": 351731712
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007219418254764293,
+ "loss": 1.6277,
+ "theoretical_loss": 4.099100769282221,
+ "tokens_seen": 351797248
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721925777331996,
+ "loss": 1.5872,
+ "theoretical_loss": 4.099014101722196,
+ "tokens_seen": 351862784
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007219097291875627,
+ "loss": 1.549,
+ "theoretical_loss": 4.098927454821747,
+ "tokens_seen": 351928320
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218936810431294,
+ "loss": 1.7034,
+ "theoretical_loss": 4.098840828572101,
+ "tokens_seen": 351993856
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218776328986961,
+ "loss": 1.5264,
+ "theoretical_loss": 4.0987542229644935,
+ "tokens_seen": 352059392
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218615847542628,
+ "loss": 1.5467,
+ "theoretical_loss": 4.0986676379901645,
+ "tokens_seen": 352124928
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218455366098295,
+ "loss": 1.5454,
+ "theoretical_loss": 4.098581073640361,
+ "tokens_seen": 352190464
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218294884653962,
+ "loss": 1.6108,
+ "theoretical_loss": 4.098494529906331,
+ "tokens_seen": 352256000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007218134403209629,
+ "loss": 1.625,
+ "theoretical_loss": 4.098408006779332,
+ "tokens_seen": 352321536
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217973921765296,
+ "loss": 1.6397,
+ "theoretical_loss": 4.098321504250626,
+ "tokens_seen": 352387072
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217813440320963,
+ "loss": 1.651,
+ "theoretical_loss": 4.098235022311479,
+ "tokens_seen": 352452608
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721765295887663,
+ "loss": 1.5686,
+ "theoretical_loss": 4.098148560953163,
+ "tokens_seen": 352518144
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217492477432297,
+ "loss": 1.5725,
+ "theoretical_loss": 4.098062120166956,
+ "tokens_seen": 352583680
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217331995987964,
+ "loss": 1.5605,
+ "theoretical_loss": 4.097975699944141,
+ "tokens_seen": 352649216
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217171514543631,
+ "loss": 1.4939,
+ "theoretical_loss": 4.097889300276005,
+ "tokens_seen": 352714752
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007217011033099298,
+ "loss": 1.425,
+ "theoretical_loss": 4.097802921153842,
+ "tokens_seen": 352780288
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007216850551654965,
+ "loss": 1.5532,
+ "theoretical_loss": 4.097716562568951,
+ "tokens_seen": 352845824
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007216690070210632,
+ "loss": 1.6839,
+ "theoretical_loss": 4.097630224512635,
+ "tokens_seen": 352911360
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007216529588766299,
+ "loss": 1.6294,
+ "theoretical_loss": 4.097543906976204,
+ "tokens_seen": 352976896
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007216369107321966,
+ "loss": 1.5198,
+ "theoretical_loss": 4.097457609950973,
+ "tokens_seen": 353042432
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007216208625877633,
+ "loss": 1.5761,
+ "theoretical_loss": 4.09737133342826,
+ "tokens_seen": 353107968
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00072160481444333,
+ "loss": 1.6117,
+ "theoretical_loss": 4.097285077399392,
+ "tokens_seen": 353173504
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215887662988967,
+ "loss": 1.6125,
+ "theoretical_loss": 4.097198841855699,
+ "tokens_seen": 353239040
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215727181544634,
+ "loss": 1.6285,
+ "theoretical_loss": 4.097112626788516,
+ "tokens_seen": 353304576
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215566700100301,
+ "loss": 1.5738,
+ "theoretical_loss": 4.097026432189185,
+ "tokens_seen": 353370112
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215406218655968,
+ "loss": 1.6719,
+ "theoretical_loss": 4.096940258049052,
+ "tokens_seen": 353435648
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215245737211635,
+ "loss": 1.5116,
+ "theoretical_loss": 4.096854104359467,
+ "tokens_seen": 353501184
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007215085255767302,
+ "loss": 1.4971,
+ "theoretical_loss": 4.096767971111789,
+ "tokens_seen": 353566720
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007214924774322969,
+ "loss": 1.6234,
+ "theoretical_loss": 4.096681858297378,
+ "tokens_seen": 353632256
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007214764292878636,
+ "loss": 1.5845,
+ "theoretical_loss": 4.096595765907603,
+ "tokens_seen": 353697792
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007214603811434303,
+ "loss": 1.4026,
+ "theoretical_loss": 4.096509693933835,
+ "tokens_seen": 353763328
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721444332998997,
+ "loss": 1.745,
+ "theoretical_loss": 4.096423642367453,
+ "tokens_seen": 353828864
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 124136,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5353221893310547,
+ "objective/train/theoretical_loss": 4.096337611199838,
+ "objective/train/tokens_used": 374354400,
+ "theoretical_loss": 4.096337611199838,
+ "tokens_seen": 353894400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007214282848545637,
+ "loss": 1.4744,
+ "theoretical_loss": 4.096337611199838,
+ "tokens_seen": 353894400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007214122367101304,
+ "loss": 1.5854,
+ "theoretical_loss": 4.096251600422381,
+ "tokens_seen": 353959936
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007213961885656971,
+ "loss": 1.5289,
+ "theoretical_loss": 4.096165610026474,
+ "tokens_seen": 354025472
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007213801404212638,
+ "loss": 1.5319,
+ "theoretical_loss": 4.096079640003515,
+ "tokens_seen": 354091008
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007213640922768305,
+ "loss": 1.469,
+ "theoretical_loss": 4.095993690344908,
+ "tokens_seen": 354156544
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007213480441323972,
+ "loss": 1.5532,
+ "theoretical_loss": 4.0959077610420636,
+ "tokens_seen": 354222080
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721331995987964,
+ "loss": 1.503,
+ "theoretical_loss": 4.095821852086394,
+ "tokens_seen": 354287616
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007213159478435306,
+ "loss": 1.4657,
+ "theoretical_loss": 4.095735963469322,
+ "tokens_seen": 354353152
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212998996990973,
+ "loss": 1.4184,
+ "theoretical_loss": 4.095650095182268,
+ "tokens_seen": 354418688
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721283851554664,
+ "loss": 1.525,
+ "theoretical_loss": 4.0955642472166645,
+ "tokens_seen": 354484224
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212678034102307,
+ "loss": 1.524,
+ "theoretical_loss": 4.095478419563947,
+ "tokens_seen": 354549760
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212517552657974,
+ "loss": 1.4833,
+ "theoretical_loss": 4.095392612215555,
+ "tokens_seen": 354615296
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212357071213642,
+ "loss": 1.486,
+ "theoretical_loss": 4.095306825162933,
+ "tokens_seen": 354680832
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212196589769308,
+ "loss": 1.6026,
+ "theoretical_loss": 4.095221058397534,
+ "tokens_seen": 354746368
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007212036108324976,
+ "loss": 1.493,
+ "theoretical_loss": 4.095135311910812,
+ "tokens_seen": 354811904
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007211875626880642,
+ "loss": 1.4913,
+ "theoretical_loss": 4.095049585694229,
+ "tokens_seen": 354877440
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721171514543631,
+ "loss": 1.5869,
+ "theoretical_loss": 4.09496387973925,
+ "tokens_seen": 354942976
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007211554663991976,
+ "loss": 1.4811,
+ "theoretical_loss": 4.094878194037348,
+ "tokens_seen": 355008512
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007211394182547644,
+ "loss": 1.5131,
+ "theoretical_loss": 4.094792528579999,
+ "tokens_seen": 355074048
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721123370110331,
+ "loss": 1.5777,
+ "theoretical_loss": 4.094706883358684,
+ "tokens_seen": 355139584
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007211073219658978,
+ "loss": 1.5027,
+ "theoretical_loss": 4.09462125836489,
+ "tokens_seen": 355205120
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007210912738214644,
+ "loss": 1.4404,
+ "theoretical_loss": 4.0945356535901105,
+ "tokens_seen": 355270656
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007210752256770312,
+ "loss": 1.5339,
+ "theoretical_loss": 4.094450069025841,
+ "tokens_seen": 355336192
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007210591775325978,
+ "loss": 1.454,
+ "theoretical_loss": 4.094364504663584,
+ "tokens_seen": 355401728
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007210431293881646,
+ "loss": 1.6805,
+ "theoretical_loss": 4.094278960494847,
+ "tokens_seen": 355467264
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007210270812437312,
+ "loss": 1.4953,
+ "theoretical_loss": 4.094193436511143,
+ "tokens_seen": 355532800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000721011033099298,
+ "loss": 1.4764,
+ "theoretical_loss": 4.094107932703991,
+ "tokens_seen": 355598336
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007209949849548646,
+ "loss": 1.6513,
+ "theoretical_loss": 4.09402244906491,
+ "tokens_seen": 355663872
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007209789368104314,
+ "loss": 1.6311,
+ "theoretical_loss": 4.093936985585432,
+ "tokens_seen": 355729408
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720962888665998,
+ "loss": 1.6119,
+ "theoretical_loss": 4.093851542257088,
+ "tokens_seen": 355794944
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007209468405215648,
+ "loss": 1.6171,
+ "theoretical_loss": 4.093766119071416,
+ "tokens_seen": 355860480
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007209307923771314,
+ "loss": 1.6012,
+ "theoretical_loss": 4.093680716019961,
+ "tokens_seen": 355926016
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007209147442326982,
+ "loss": 1.5785,
+ "theoretical_loss": 4.093595333094269,
+ "tokens_seen": 355991552
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007208986960882648,
+ "loss": 1.5399,
+ "theoretical_loss": 4.093509970285895,
+ "tokens_seen": 356057088
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007208826479438316,
+ "loss": 1.5283,
+ "theoretical_loss": 4.093424627586399,
+ "tokens_seen": 356122624
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007208665997993982,
+ "loss": 1.4739,
+ "theoretical_loss": 4.093339304987341,
+ "tokens_seen": 356188160
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720850551654965,
+ "loss": 1.4281,
+ "theoretical_loss": 4.093254002480293,
+ "tokens_seen": 356253696
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007208345035105316,
+ "loss": 1.7098,
+ "theoretical_loss": 4.093168720056828,
+ "tokens_seen": 356319232
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007208184553660984,
+ "loss": 1.4742,
+ "theoretical_loss": 4.093083457708524,
+ "tokens_seen": 356384768
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720802407221665,
+ "loss": 1.6874,
+ "theoretical_loss": 4.092998215426967,
+ "tokens_seen": 356450304
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207863590772318,
+ "loss": 1.5701,
+ "theoretical_loss": 4.092912993203744,
+ "tokens_seen": 356515840
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207703109327984,
+ "loss": 1.5932,
+ "theoretical_loss": 4.09282779103045,
+ "tokens_seen": 356581376
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207542627883652,
+ "loss": 1.5365,
+ "theoretical_loss": 4.092742608898685,
+ "tokens_seen": 356646912
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207382146439318,
+ "loss": 1.6133,
+ "theoretical_loss": 4.092657446800052,
+ "tokens_seen": 356712448
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207221664994986,
+ "loss": 1.5243,
+ "theoretical_loss": 4.092572304726161,
+ "tokens_seen": 356777984
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007207061183550653,
+ "loss": 1.5389,
+ "theoretical_loss": 4.092487182668627,
+ "tokens_seen": 356843520
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720690070210632,
+ "loss": 1.6216,
+ "theoretical_loss": 4.092402080619069,
+ "tokens_seen": 356909056
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007206740220661987,
+ "loss": 1.7329,
+ "theoretical_loss": 4.09231699856911,
+ "tokens_seen": 356974592
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007206579739217654,
+ "loss": 1.5615,
+ "theoretical_loss": 4.092231936510383,
+ "tokens_seen": 357040128
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720641925777332,
+ "loss": 1.5868,
+ "theoretical_loss": 4.09214689443452,
+ "tokens_seen": 357105664
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 125479,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5331692695617676,
+ "objective/train/theoretical_loss": 4.0920618723331605,
+ "objective/train/tokens_used": 377631200,
+ "theoretical_loss": 4.0920618723331605,
+ "tokens_seen": 357171200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007206258776328988,
+ "loss": 1.4486,
+ "theoretical_loss": 4.0920618723331605,
+ "tokens_seen": 357171200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007206098294884654,
+ "loss": 1.621,
+ "theoretical_loss": 4.091976870197951,
+ "tokens_seen": 357236736
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205937813440322,
+ "loss": 1.5364,
+ "theoretical_loss": 4.091891888020539,
+ "tokens_seen": 357302272
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205777331995988,
+ "loss": 1.4859,
+ "theoretical_loss": 4.091806925792582,
+ "tokens_seen": 357367808
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205616850551656,
+ "loss": 1.6564,
+ "theoretical_loss": 4.091721983505737,
+ "tokens_seen": 357433344
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205456369107322,
+ "loss": 1.4599,
+ "theoretical_loss": 4.091637061151671,
+ "tokens_seen": 357498880
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205295887662989,
+ "loss": 1.4924,
+ "theoretical_loss": 4.091552158722052,
+ "tokens_seen": 357564416
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007205135406218656,
+ "loss": 1.4918,
+ "theoretical_loss": 4.091467276208554,
+ "tokens_seen": 357629952
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204974924774323,
+ "loss": 1.5315,
+ "theoretical_loss": 4.091382413602861,
+ "tokens_seen": 357695488
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720481444332999,
+ "loss": 1.6625,
+ "theoretical_loss": 4.0912975708966535,
+ "tokens_seen": 357761024
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204653961885657,
+ "loss": 1.5066,
+ "theoretical_loss": 4.091212748081622,
+ "tokens_seen": 357826560
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204493480441324,
+ "loss": 1.5501,
+ "theoretical_loss": 4.091127945149463,
+ "tokens_seen": 357892096
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204332998996991,
+ "loss": 1.5951,
+ "theoretical_loss": 4.091043162091876,
+ "tokens_seen": 357957632
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204172517552658,
+ "loss": 1.6305,
+ "theoretical_loss": 4.090958398900564,
+ "tokens_seen": 358023168
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007204012036108325,
+ "loss": 1.4798,
+ "theoretical_loss": 4.090873655567238,
+ "tokens_seen": 358088704
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007203851554663992,
+ "loss": 1.5176,
+ "theoretical_loss": 4.090788932083613,
+ "tokens_seen": 358154240
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007203691073219659,
+ "loss": 1.5244,
+ "theoretical_loss": 4.090704228441408,
+ "tokens_seen": 358219776
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007203530591775326,
+ "loss": 1.4846,
+ "theoretical_loss": 4.090619544632347,
+ "tokens_seen": 358285312
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007203370110330993,
+ "loss": 1.5317,
+ "theoretical_loss": 4.0905348806481605,
+ "tokens_seen": 358350848
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720320962888666,
+ "loss": 1.5263,
+ "theoretical_loss": 4.090450236480583,
+ "tokens_seen": 358416384
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007203049147442327,
+ "loss": 1.4949,
+ "theoretical_loss": 4.090365612121355,
+ "tokens_seen": 358481920
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202888665997994,
+ "loss": 1.5811,
+ "theoretical_loss": 4.0902810075622185,
+ "tokens_seen": 358547456
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202728184553661,
+ "loss": 1.5137,
+ "theoretical_loss": 4.090196422794925,
+ "tokens_seen": 358612992
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202567703109328,
+ "loss": 1.6153,
+ "theoretical_loss": 4.0901118578112285,
+ "tokens_seen": 358678528
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202407221664995,
+ "loss": 1.5427,
+ "theoretical_loss": 4.090027312602888,
+ "tokens_seen": 358744064
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202246740220662,
+ "loss": 1.5705,
+ "theoretical_loss": 4.089942787161667,
+ "tokens_seen": 358809600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007202086258776329,
+ "loss": 1.6536,
+ "theoretical_loss": 4.089858281479335,
+ "tokens_seen": 358875136
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007201925777331996,
+ "loss": 1.5034,
+ "theoretical_loss": 4.089773795547668,
+ "tokens_seen": 358940672
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007201765295887663,
+ "loss": 1.6133,
+ "theoretical_loss": 4.089689329358443,
+ "tokens_seen": 359006208
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000720160481444333,
+ "loss": 1.6104,
+ "theoretical_loss": 4.089604882903444,
+ "tokens_seen": 359071744
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007201444332998997,
+ "loss": 1.5082,
+ "theoretical_loss": 4.089520456174461,
+ "tokens_seen": 359137280
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007201283851554664,
+ "loss": 1.6146,
+ "theoretical_loss": 4.089436049163287,
+ "tokens_seen": 359202816
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007201123370110331,
+ "loss": 1.6244,
+ "theoretical_loss": 4.089351661861722,
+ "tokens_seen": 359268352
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200962888665998,
+ "loss": 1.5419,
+ "theoretical_loss": 4.089267294261566,
+ "tokens_seen": 359333888
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200802407221665,
+ "loss": 1.6289,
+ "theoretical_loss": 4.089182946354631,
+ "tokens_seen": 359399424
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200641925777332,
+ "loss": 1.5434,
+ "theoretical_loss": 4.08909861813273,
+ "tokens_seen": 359464960
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200481444332999,
+ "loss": 1.4964,
+ "theoretical_loss": 4.0890143095876805,
+ "tokens_seen": 359530496
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200320962888666,
+ "loss": 1.6228,
+ "theoretical_loss": 4.088930020711305,
+ "tokens_seen": 359596032
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007200160481444333,
+ "loss": 1.5421,
+ "theoretical_loss": 4.088845751495433,
+ "tokens_seen": 359661568
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00072,
+ "loss": 1.4741,
+ "theoretical_loss": 4.088761501931897,
+ "tokens_seen": 359727104
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199839518555667,
+ "loss": 1.6506,
+ "theoretical_loss": 4.088677272012535,
+ "tokens_seen": 359792640
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199679037111334,
+ "loss": 1.6194,
+ "theoretical_loss": 4.088593061729188,
+ "tokens_seen": 359858176
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199518555667001,
+ "loss": 1.5317,
+ "theoretical_loss": 4.0885088710737065,
+ "tokens_seen": 359923712
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199358074222668,
+ "loss": 1.6348,
+ "theoretical_loss": 4.088424700037941,
+ "tokens_seen": 359989248
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199197592778335,
+ "loss": 1.5538,
+ "theoretical_loss": 4.08834054861375,
+ "tokens_seen": 360054784
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007199037111334002,
+ "loss": 1.5956,
+ "theoretical_loss": 4.088256416792994,
+ "tokens_seen": 360120320
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007198876629889669,
+ "loss": 1.4913,
+ "theoretical_loss": 4.088172304567541,
+ "tokens_seen": 360185856
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007198716148445337,
+ "loss": 1.6122,
+ "theoretical_loss": 4.088088211929263,
+ "tokens_seen": 360251392
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007198555667001003,
+ "loss": 1.4953,
+ "theoretical_loss": 4.088004138870037,
+ "tokens_seen": 360316928
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000719839518555667,
+ "loss": 1.5233,
+ "theoretical_loss": 4.087920085381745,
+ "tokens_seen": 360382464
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 126440,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6889604330062866,
+ "objective/train/theoretical_loss": 4.0878360514562715,
+ "objective/train/tokens_used": 380908000,
+ "theoretical_loss": 4.0878360514562715,
+ "tokens_seen": 360448000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007198234704112337,
+ "loss": 1.6539,
+ "theoretical_loss": 4.0878360514562715,
+ "tokens_seen": 360448000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007198074222668005,
+ "loss": 1.4704,
+ "theoretical_loss": 4.087752037085509,
+ "tokens_seen": 360513536
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197913741223671,
+ "loss": 1.5366,
+ "theoretical_loss": 4.087668042261353,
+ "tokens_seen": 360579072
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197753259779339,
+ "loss": 1.4443,
+ "theoretical_loss": 4.087584066975706,
+ "tokens_seen": 360644608
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197592778335005,
+ "loss": 1.5863,
+ "theoretical_loss": 4.087500111220472,
+ "tokens_seen": 360710144
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197432296890673,
+ "loss": 1.5266,
+ "theoretical_loss": 4.087416174987563,
+ "tokens_seen": 360775680
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197271815446339,
+ "loss": 1.6049,
+ "theoretical_loss": 4.087332258268893,
+ "tokens_seen": 360841216
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007197111334002007,
+ "loss": 1.5389,
+ "theoretical_loss": 4.087248361056384,
+ "tokens_seen": 360906752
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196950852557673,
+ "loss": 1.7153,
+ "theoretical_loss": 4.087164483341959,
+ "tokens_seen": 360972288
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196790371113341,
+ "loss": 1.4725,
+ "theoretical_loss": 4.087080625117549,
+ "tokens_seen": 361037824
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196629889669007,
+ "loss": 1.4997,
+ "theoretical_loss": 4.086996786375089,
+ "tokens_seen": 361103360
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196469408224675,
+ "loss": 1.5327,
+ "theoretical_loss": 4.0869129671065165,
+ "tokens_seen": 361168896
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196308926780341,
+ "loss": 1.5279,
+ "theoretical_loss": 4.086829167303778,
+ "tokens_seen": 361234432
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007196148445336009,
+ "loss": 1.6238,
+ "theoretical_loss": 4.086745386958821,
+ "tokens_seen": 361299968
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195987963891675,
+ "loss": 1.6077,
+ "theoretical_loss": 4.0866616260636,
+ "tokens_seen": 361365504
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195827482447343,
+ "loss": 1.5136,
+ "theoretical_loss": 4.086577884610074,
+ "tokens_seen": 361431040
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195667001003009,
+ "loss": 1.5304,
+ "theoretical_loss": 4.086494162590206,
+ "tokens_seen": 361496576
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195506519558677,
+ "loss": 1.6102,
+ "theoretical_loss": 4.086410459995964,
+ "tokens_seen": 361562112
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195346038114343,
+ "loss": 1.6007,
+ "theoretical_loss": 4.0863267768193205,
+ "tokens_seen": 361627648
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195185556670011,
+ "loss": 1.5662,
+ "theoretical_loss": 4.086243113052254,
+ "tokens_seen": 361693184
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007195025075225678,
+ "loss": 1.4989,
+ "theoretical_loss": 4.0861594686867475,
+ "tokens_seen": 361758720
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007194864593781345,
+ "loss": 1.5979,
+ "theoretical_loss": 4.086075843714786,
+ "tokens_seen": 361824256
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007194704112337012,
+ "loss": 1.6645,
+ "theoretical_loss": 4.085992238128364,
+ "tokens_seen": 361889792
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007194543630892679,
+ "loss": 1.7224,
+ "theoretical_loss": 4.085908651919477,
+ "tokens_seen": 361955328
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007194383149448346,
+ "loss": 1.5489,
+ "theoretical_loss": 4.085825085080128,
+ "tokens_seen": 362020864
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007194222668004013,
+ "loss": 1.5459,
+ "theoretical_loss": 4.08574153760232,
+ "tokens_seen": 362086400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000719406218655968,
+ "loss": 1.7234,
+ "theoretical_loss": 4.085658009478067,
+ "tokens_seen": 362151936
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193901705115347,
+ "loss": 1.4414,
+ "theoretical_loss": 4.0855745006993835,
+ "tokens_seen": 362217472
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193741223671014,
+ "loss": 1.5663,
+ "theoretical_loss": 4.08549101125829,
+ "tokens_seen": 362283008
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193580742226681,
+ "loss": 1.6014,
+ "theoretical_loss": 4.085407541146812,
+ "tokens_seen": 362348544
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193420260782348,
+ "loss": 1.6279,
+ "theoretical_loss": 4.085324090356979,
+ "tokens_seen": 362414080
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193259779338015,
+ "loss": 1.531,
+ "theoretical_loss": 4.085240658880827,
+ "tokens_seen": 362479616
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007193099297893682,
+ "loss": 1.5496,
+ "theoretical_loss": 4.085157246710394,
+ "tokens_seen": 362545152
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192938816449349,
+ "loss": 1.5263,
+ "theoretical_loss": 4.0850738538377245,
+ "tokens_seen": 362610688
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192778335005016,
+ "loss": 1.6056,
+ "theoretical_loss": 4.084990480254867,
+ "tokens_seen": 362676224
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192617853560683,
+ "loss": 1.6295,
+ "theoretical_loss": 4.084907125953876,
+ "tokens_seen": 362741760
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192457372116349,
+ "loss": 1.5451,
+ "theoretical_loss": 4.0848237909268095,
+ "tokens_seen": 362807296
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192296890672017,
+ "loss": 1.6121,
+ "theoretical_loss": 4.084740475165729,
+ "tokens_seen": 362872832
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007192136409227683,
+ "loss": 1.5199,
+ "theoretical_loss": 4.084657178662704,
+ "tokens_seen": 362938368
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191975927783351,
+ "loss": 1.5887,
+ "theoretical_loss": 4.084573901409807,
+ "tokens_seen": 363003904
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191815446339017,
+ "loss": 1.6182,
+ "theoretical_loss": 4.084490643399113,
+ "tokens_seen": 363069440
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191654964894685,
+ "loss": 1.5459,
+ "theoretical_loss": 4.084407404622706,
+ "tokens_seen": 363134976
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191494483450351,
+ "loss": 1.6282,
+ "theoretical_loss": 4.084324185072671,
+ "tokens_seen": 363200512
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191334002006019,
+ "loss": 1.6513,
+ "theoretical_loss": 4.084240984741099,
+ "tokens_seen": 363266048
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191173520561685,
+ "loss": 1.5754,
+ "theoretical_loss": 4.084157803620086,
+ "tokens_seen": 363331584
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007191013039117352,
+ "loss": 1.6044,
+ "theoretical_loss": 4.084074641701734,
+ "tokens_seen": 363397120
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007190852557673019,
+ "loss": 1.6218,
+ "theoretical_loss": 4.083991498978146,
+ "tokens_seen": 363462656
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007190692076228686,
+ "loss": 1.549,
+ "theoretical_loss": 4.083908375441433,
+ "tokens_seen": 363528192
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007190531594784353,
+ "loss": 1.5066,
+ "theoretical_loss": 4.083825271083709,
+ "tokens_seen": 363593728
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000719037111334002,
+ "loss": 1.7009,
+ "theoretical_loss": 4.083742185897093,
+ "tokens_seen": 363659264
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 127545,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4759583473205566,
+ "objective/train/theoretical_loss": 4.08365911987371,
+ "objective/train/tokens_used": 384184800,
+ "theoretical_loss": 4.08365911987371,
+ "tokens_seen": 363724800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007190210631895687,
+ "loss": 1.4025,
+ "theoretical_loss": 4.08365911987371,
+ "tokens_seen": 363724800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007190050150451354,
+ "loss": 1.6018,
+ "theoretical_loss": 4.083576073005687,
+ "tokens_seen": 363790336
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189889669007021,
+ "loss": 1.381,
+ "theoretical_loss": 4.083493045285158,
+ "tokens_seen": 363855872
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189729187562688,
+ "loss": 1.4797,
+ "theoretical_loss": 4.08341003670426,
+ "tokens_seen": 363921408
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189568706118355,
+ "loss": 1.477,
+ "theoretical_loss": 4.0833270472551355,
+ "tokens_seen": 363986944
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189408224674022,
+ "loss": 1.6203,
+ "theoretical_loss": 4.083244076929932,
+ "tokens_seen": 364052480
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189247743229689,
+ "loss": 1.669,
+ "theoretical_loss": 4.083161125720801,
+ "tokens_seen": 364118016
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007189087261785356,
+ "loss": 1.5458,
+ "theoretical_loss": 4.083078193619899,
+ "tokens_seen": 364183552
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007188926780341023,
+ "loss": 1.6065,
+ "theoretical_loss": 4.082995280619386,
+ "tokens_seen": 364249088
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718876629889669,
+ "loss": 1.4797,
+ "theoretical_loss": 4.0829123867114285,
+ "tokens_seen": 364314624
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007188605817452357,
+ "loss": 1.752,
+ "theoretical_loss": 4.082829511888197,
+ "tokens_seen": 364380160
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007188445336008024,
+ "loss": 1.5957,
+ "theoretical_loss": 4.082746656141865,
+ "tokens_seen": 364445696
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007188284854563691,
+ "loss": 1.6762,
+ "theoretical_loss": 4.082663819464613,
+ "tokens_seen": 364511232
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007188124373119358,
+ "loss": 1.6709,
+ "theoretical_loss": 4.082581001848624,
+ "tokens_seen": 364576768
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187963891675025,
+ "loss": 1.4828,
+ "theoretical_loss": 4.082498203286088,
+ "tokens_seen": 364642304
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187803410230692,
+ "loss": 1.5422,
+ "theoretical_loss": 4.082415423769197,
+ "tokens_seen": 364707840
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187642928786359,
+ "loss": 1.4115,
+ "theoretical_loss": 4.082332663290149,
+ "tokens_seen": 364773376
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187482447342026,
+ "loss": 1.5521,
+ "theoretical_loss": 4.082249921841147,
+ "tokens_seen": 364838912
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187321965897693,
+ "loss": 1.5435,
+ "theoretical_loss": 4.082167199414398,
+ "tokens_seen": 364904448
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718716148445336,
+ "loss": 1.5592,
+ "theoretical_loss": 4.082084496002113,
+ "tokens_seen": 364969984
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007187001003009027,
+ "loss": 1.521,
+ "theoretical_loss": 4.082001811596509,
+ "tokens_seen": 365035520
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007186840521564694,
+ "loss": 1.5971,
+ "theoretical_loss": 4.081919146189806,
+ "tokens_seen": 365101056
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007186680040120361,
+ "loss": 1.3846,
+ "theoretical_loss": 4.08183649977423,
+ "tokens_seen": 365166592
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007186519558676028,
+ "loss": 1.6659,
+ "theoretical_loss": 4.08175387234201,
+ "tokens_seen": 365232128
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007186359077231695,
+ "loss": 1.5476,
+ "theoretical_loss": 4.081671263885381,
+ "tokens_seen": 365297664
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007186198595787362,
+ "loss": 1.501,
+ "theoretical_loss": 4.081588674396583,
+ "tokens_seen": 365363200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718603811434303,
+ "loss": 1.5564,
+ "theoretical_loss": 4.081506103867858,
+ "tokens_seen": 365428736
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007185877632898696,
+ "loss": 1.6682,
+ "theoretical_loss": 4.081423552291456,
+ "tokens_seen": 365494272
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007185717151454363,
+ "loss": 1.5907,
+ "theoretical_loss": 4.081341019659628,
+ "tokens_seen": 365559808
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718555667001003,
+ "loss": 1.5072,
+ "theoretical_loss": 4.081258505964633,
+ "tokens_seen": 365625344
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007185396188565697,
+ "loss": 1.629,
+ "theoretical_loss": 4.081176011198731,
+ "tokens_seen": 365690880
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007185235707121364,
+ "loss": 1.6701,
+ "theoretical_loss": 4.08109353535419,
+ "tokens_seen": 365756416
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007185075225677032,
+ "loss": 1.6575,
+ "theoretical_loss": 4.0810110784232805,
+ "tokens_seen": 365821952
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007184914744232698,
+ "loss": 1.6648,
+ "theoretical_loss": 4.080928640398278,
+ "tokens_seen": 365887488
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007184754262788366,
+ "loss": 1.5334,
+ "theoretical_loss": 4.080846221271461,
+ "tokens_seen": 365953024
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007184593781344032,
+ "loss": 1.6324,
+ "theoretical_loss": 4.0807638210351165,
+ "tokens_seen": 366018560
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00071844332998997,
+ "loss": 1.7469,
+ "theoretical_loss": 4.080681439681533,
+ "tokens_seen": 366084096
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007184272818455366,
+ "loss": 1.6183,
+ "theoretical_loss": 4.080599077203003,
+ "tokens_seen": 366149632
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007184112337011034,
+ "loss": 1.4391,
+ "theoretical_loss": 4.080516733591827,
+ "tokens_seen": 366215168
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00071839518555667,
+ "loss": 1.578,
+ "theoretical_loss": 4.080434408840304,
+ "tokens_seen": 366280704
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007183791374122368,
+ "loss": 1.6214,
+ "theoretical_loss": 4.080352102940744,
+ "tokens_seen": 366346240
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007183630892678034,
+ "loss": 1.583,
+ "theoretical_loss": 4.080269815885458,
+ "tokens_seen": 366411776
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007183470411233702,
+ "loss": 1.6004,
+ "theoretical_loss": 4.080187547666763,
+ "tokens_seen": 366477312
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007183309929789368,
+ "loss": 1.5257,
+ "theoretical_loss": 4.080105298276978,
+ "tokens_seen": 366542848
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007183149448345036,
+ "loss": 1.4851,
+ "theoretical_loss": 4.08002306770843,
+ "tokens_seen": 366608384
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182988966900703,
+ "loss": 1.554,
+ "theoretical_loss": 4.079940855953447,
+ "tokens_seen": 366673920
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718282848545637,
+ "loss": 1.5353,
+ "theoretical_loss": 4.079858663004364,
+ "tokens_seen": 366739456
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182668004012037,
+ "loss": 1.5238,
+ "theoretical_loss": 4.079776488853521,
+ "tokens_seen": 366804992
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182507522567704,
+ "loss": 1.5154,
+ "theoretical_loss": 4.079694333493259,
+ "tokens_seen": 366870528
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182347041123371,
+ "loss": 1.6315,
+ "theoretical_loss": 4.079612196915927,
+ "tokens_seen": 366936064
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 128594,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5575971603393555,
+ "objective/train/theoretical_loss": 4.079530079113876,
+ "objective/train/tokens_used": 387461600,
+ "theoretical_loss": 4.079530079113876,
+ "tokens_seen": 367001600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182186559679038,
+ "loss": 1.559,
+ "theoretical_loss": 4.079530079113876,
+ "tokens_seen": 367001600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007182026078234705,
+ "loss": 1.6243,
+ "theoretical_loss": 4.079447980079465,
+ "tokens_seen": 367067136
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007181865596790372,
+ "loss": 1.5679,
+ "theoretical_loss": 4.079365899805053,
+ "tokens_seen": 367132672
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007181705115346039,
+ "loss": 1.5415,
+ "theoretical_loss": 4.079283838283007,
+ "tokens_seen": 367198208
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007181544633901706,
+ "loss": 1.63,
+ "theoretical_loss": 4.079201795505695,
+ "tokens_seen": 367263744
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007181384152457373,
+ "loss": 1.5294,
+ "theoretical_loss": 4.079119771465494,
+ "tokens_seen": 367329280
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000718122367101304,
+ "loss": 1.5443,
+ "theoretical_loss": 4.079037766154781,
+ "tokens_seen": 367394816
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007181063189568707,
+ "loss": 1.6435,
+ "theoretical_loss": 4.078955779565941,
+ "tokens_seen": 367460352
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180902708124374,
+ "loss": 1.4562,
+ "theoretical_loss": 4.078873811691361,
+ "tokens_seen": 367525888
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180742226680041,
+ "loss": 1.6974,
+ "theoretical_loss": 4.078791862523433,
+ "tokens_seen": 367591424
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180581745235708,
+ "loss": 1.5949,
+ "theoretical_loss": 4.078709932054554,
+ "tokens_seen": 367656960
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180421263791375,
+ "loss": 1.5187,
+ "theoretical_loss": 4.078628020277128,
+ "tokens_seen": 367722496
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180260782347042,
+ "loss": 1.547,
+ "theoretical_loss": 4.078546127183556,
+ "tokens_seen": 367788032
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007180100300902709,
+ "loss": 1.5075,
+ "theoretical_loss": 4.078464252766252,
+ "tokens_seen": 367853568
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007179939819458376,
+ "loss": 1.4956,
+ "theoretical_loss": 4.078382397017628,
+ "tokens_seen": 367919104
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007179779338014043,
+ "loss": 1.6077,
+ "theoretical_loss": 4.0783005599301045,
+ "tokens_seen": 367984640
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717961885656971,
+ "loss": 1.6948,
+ "theoretical_loss": 4.078218741496104,
+ "tokens_seen": 368050176
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007179458375125377,
+ "loss": 1.5008,
+ "theoretical_loss": 4.078136941708055,
+ "tokens_seen": 368115712
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007179297893681044,
+ "loss": 1.4755,
+ "theoretical_loss": 4.078055160558391,
+ "tokens_seen": 368181248
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007179137412236711,
+ "loss": 1.6208,
+ "theoretical_loss": 4.0779733980395445,
+ "tokens_seen": 368246784
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178976930792378,
+ "loss": 1.5518,
+ "theoretical_loss": 4.077891654143961,
+ "tokens_seen": 368312320
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178816449348045,
+ "loss": 1.746,
+ "theoretical_loss": 4.0778099288640846,
+ "tokens_seen": 368377856
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178655967903712,
+ "loss": 1.4664,
+ "theoretical_loss": 4.0777282221923645,
+ "tokens_seen": 368443392
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178495486459378,
+ "loss": 1.6716,
+ "theoretical_loss": 4.077646534121256,
+ "tokens_seen": 368508928
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178335005015046,
+ "loss": 1.6311,
+ "theoretical_loss": 4.077564864643217,
+ "tokens_seen": 368574464
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007178174523570712,
+ "loss": 1.5638,
+ "theoretical_loss": 4.077483213750711,
+ "tokens_seen": 368640000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717801404212638,
+ "loss": 1.6408,
+ "theoretical_loss": 4.077401581436206,
+ "tokens_seen": 368705536
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007177853560682046,
+ "loss": 1.5993,
+ "theoretical_loss": 4.077319967692173,
+ "tokens_seen": 368771072
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007177693079237714,
+ "loss": 1.5674,
+ "theoretical_loss": 4.077238372511088,
+ "tokens_seen": 368836608
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717753259779338,
+ "loss": 1.5769,
+ "theoretical_loss": 4.077156795885434,
+ "tokens_seen": 368902144
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007177372116349048,
+ "loss": 1.4508,
+ "theoretical_loss": 4.077075237807693,
+ "tokens_seen": 368967680
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007177211634904714,
+ "loss": 1.5182,
+ "theoretical_loss": 4.076993698270357,
+ "tokens_seen": 369033216
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007177051153460381,
+ "loss": 1.5911,
+ "theoretical_loss": 4.0769121772659185,
+ "tokens_seen": 369098752
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176890672016048,
+ "loss": 1.5392,
+ "theoretical_loss": 4.076830674786876,
+ "tokens_seen": 369164288
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176730190571715,
+ "loss": 1.5293,
+ "theoretical_loss": 4.076749190825732,
+ "tokens_seen": 369229824
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176569709127382,
+ "loss": 1.5865,
+ "theoretical_loss": 4.076667725374993,
+ "tokens_seen": 369295360
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176409227683049,
+ "loss": 1.5412,
+ "theoretical_loss": 4.076586278427171,
+ "tokens_seen": 369360896
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176248746238716,
+ "loss": 1.4512,
+ "theoretical_loss": 4.076504849974781,
+ "tokens_seen": 369426432
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007176088264794383,
+ "loss": 1.6467,
+ "theoretical_loss": 4.076423440010343,
+ "tokens_seen": 369491968
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717592778335005,
+ "loss": 1.5657,
+ "theoretical_loss": 4.076342048526382,
+ "tokens_seen": 369557504
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007175767301905717,
+ "loss": 1.5009,
+ "theoretical_loss": 4.076260675515427,
+ "tokens_seen": 369623040
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007175606820461384,
+ "loss": 1.4412,
+ "theoretical_loss": 4.07617932097001,
+ "tokens_seen": 369688576
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007175446339017051,
+ "loss": 1.5958,
+ "theoretical_loss": 4.076097984882669,
+ "tokens_seen": 369754112
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007175285857572718,
+ "loss": 1.5223,
+ "theoretical_loss": 4.076016667245945,
+ "tokens_seen": 369819648
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007175125376128385,
+ "loss": 1.5297,
+ "theoretical_loss": 4.075935368052384,
+ "tokens_seen": 369885184
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174964894684052,
+ "loss": 1.5543,
+ "theoretical_loss": 4.075854087294538,
+ "tokens_seen": 369950720
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174804413239719,
+ "loss": 1.4428,
+ "theoretical_loss": 4.075772824964961,
+ "tokens_seen": 370016256
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174643931795386,
+ "loss": 1.483,
+ "theoretical_loss": 4.07569158105621,
+ "tokens_seen": 370081792
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174483450351053,
+ "loss": 1.5444,
+ "theoretical_loss": 4.075610355560851,
+ "tokens_seen": 370147328
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717432296890672,
+ "loss": 1.5556,
+ "theoretical_loss": 4.075529148471451,
+ "tokens_seen": 370212864
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 129872,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5275757312774658,
+ "objective/train/theoretical_loss": 4.075447959780581,
+ "objective/train/tokens_used": 390738400,
+ "theoretical_loss": 4.075447959780581,
+ "tokens_seen": 370278400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174162487462387,
+ "loss": 1.517,
+ "theoretical_loss": 4.075447959780581,
+ "tokens_seen": 370278400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007174002006018054,
+ "loss": 1.5282,
+ "theoretical_loss": 4.075366789480818,
+ "tokens_seen": 370343936
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173841524573721,
+ "loss": 1.5652,
+ "theoretical_loss": 4.075285637564743,
+ "tokens_seen": 370409472
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173681043129388,
+ "loss": 1.5969,
+ "theoretical_loss": 4.07520450402494,
+ "tokens_seen": 370475008
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173520561685055,
+ "loss": 1.4542,
+ "theoretical_loss": 4.075123388854,
+ "tokens_seen": 370540544
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173360080240722,
+ "loss": 1.619,
+ "theoretical_loss": 4.075042292044515,
+ "tokens_seen": 370606080
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173199598796389,
+ "loss": 1.5211,
+ "theoretical_loss": 4.074961213589082,
+ "tokens_seen": 370671616
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007173039117352056,
+ "loss": 1.6464,
+ "theoretical_loss": 4.074880153480304,
+ "tokens_seen": 370737152
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007172878635907723,
+ "loss": 1.528,
+ "theoretical_loss": 4.074799111710789,
+ "tokens_seen": 370802688
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717271815446339,
+ "loss": 1.5398,
+ "theoretical_loss": 4.074718088273144,
+ "tokens_seen": 370868224
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007172557673019057,
+ "loss": 1.496,
+ "theoretical_loss": 4.074637083159988,
+ "tokens_seen": 370933760
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007172397191574724,
+ "loss": 1.4893,
+ "theoretical_loss": 4.0745560963639385,
+ "tokens_seen": 370999296
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007172236710130391,
+ "loss": 1.5709,
+ "theoretical_loss": 4.0744751278776175,
+ "tokens_seen": 371064832
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007172076228686058,
+ "loss": 1.6092,
+ "theoretical_loss": 4.074394177693655,
+ "tokens_seen": 371130368
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007171915747241725,
+ "loss": 1.464,
+ "theoretical_loss": 4.074313245804683,
+ "tokens_seen": 371195904
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007171755265797392,
+ "loss": 1.5461,
+ "theoretical_loss": 4.074232332203336,
+ "tokens_seen": 371261440
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007171594784353059,
+ "loss": 1.4682,
+ "theoretical_loss": 4.074151436882256,
+ "tokens_seen": 371326976
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007171434302908726,
+ "loss": 1.5951,
+ "theoretical_loss": 4.074070559834088,
+ "tokens_seen": 371392512
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007171273821464393,
+ "loss": 1.5379,
+ "theoretical_loss": 4.073989701051481,
+ "tokens_seen": 371458048
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000717111334002006,
+ "loss": 1.5889,
+ "theoretical_loss": 4.073908860527087,
+ "tokens_seen": 371523584
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170952858575728,
+ "loss": 1.5812,
+ "theoretical_loss": 4.0738280382535645,
+ "tokens_seen": 371589120
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170792377131395,
+ "loss": 1.403,
+ "theoretical_loss": 4.0737472342235765,
+ "tokens_seen": 371654656
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170631895687062,
+ "loss": 1.5452,
+ "theoretical_loss": 4.073666448429787,
+ "tokens_seen": 371720192
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170471414242729,
+ "loss": 1.5192,
+ "theoretical_loss": 4.073585680864868,
+ "tokens_seen": 371785728
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170310932798396,
+ "loss": 1.4643,
+ "theoretical_loss": 4.073504931521493,
+ "tokens_seen": 371851264
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007170150451354063,
+ "loss": 1.5878,
+ "theoretical_loss": 4.073424200392342,
+ "tokens_seen": 371916800
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000716998996990973,
+ "loss": 1.6744,
+ "theoretical_loss": 4.073343487470097,
+ "tokens_seen": 371982336
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169829488465397,
+ "loss": 1.5849,
+ "theoretical_loss": 4.073262792747445,
+ "tokens_seen": 372047872
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169669007021064,
+ "loss": 1.5167,
+ "theoretical_loss": 4.073182116217079,
+ "tokens_seen": 372113408
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169508525576731,
+ "loss": 1.5442,
+ "theoretical_loss": 4.073101457871693,
+ "tokens_seen": 372178944
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169348044132398,
+ "loss": 1.5109,
+ "theoretical_loss": 4.073020817703988,
+ "tokens_seen": 372244480
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169187562688065,
+ "loss": 1.5497,
+ "theoretical_loss": 4.072940195706668,
+ "tokens_seen": 372310016
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007169027081243732,
+ "loss": 1.5475,
+ "theoretical_loss": 4.072859591872441,
+ "tokens_seen": 372375552
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007168866599799399,
+ "loss": 1.5128,
+ "theoretical_loss": 4.07277900619402,
+ "tokens_seen": 372441088
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007168706118355066,
+ "loss": 1.5664,
+ "theoretical_loss": 4.072698438664121,
+ "tokens_seen": 372506624
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007168545636910733,
+ "loss": 1.5994,
+ "theoretical_loss": 4.072617889275467,
+ "tokens_seen": 372572160
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00071683851554664,
+ "loss": 1.5367,
+ "theoretical_loss": 4.07253735802078,
+ "tokens_seen": 372637696
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007168224674022067,
+ "loss": 1.595,
+ "theoretical_loss": 4.072456844892792,
+ "tokens_seen": 372703232
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007168064192577734,
+ "loss": 1.5945,
+ "theoretical_loss": 4.072376349884236,
+ "tokens_seen": 372768768
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167903711133401,
+ "loss": 1.4935,
+ "theoretical_loss": 4.072295872987848,
+ "tokens_seen": 372834304
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167743229689068,
+ "loss": 1.5228,
+ "theoretical_loss": 4.072215414196373,
+ "tokens_seen": 372899840
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167582748244735,
+ "loss": 1.5967,
+ "theoretical_loss": 4.072134973502554,
+ "tokens_seen": 372965376
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167422266800402,
+ "loss": 1.5616,
+ "theoretical_loss": 4.072054550899144,
+ "tokens_seen": 373030912
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167261785356069,
+ "loss": 1.4757,
+ "theoretical_loss": 4.071974146378896,
+ "tokens_seen": 373096448
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007167101303911736,
+ "loss": 1.5998,
+ "theoretical_loss": 4.071893759934568,
+ "tokens_seen": 373161984
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007166940822467403,
+ "loss": 1.4561,
+ "theoretical_loss": 4.0718133915589245,
+ "tokens_seen": 373227520
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000716678034102307,
+ "loss": 1.5062,
+ "theoretical_loss": 4.071733041244732,
+ "tokens_seen": 373293056
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007166619859578737,
+ "loss": 1.5552,
+ "theoretical_loss": 4.071652708984761,
+ "tokens_seen": 373358592
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007166459378134404,
+ "loss": 1.4439,
+ "theoretical_loss": 4.0715723947717875,
+ "tokens_seen": 373424128
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007166298896690071,
+ "loss": 1.5569,
+ "theoretical_loss": 4.07149209859859,
+ "tokens_seen": 373489664
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 131043,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4238359928131104,
+ "objective/train/theoretical_loss": 4.0714118204579535,
+ "objective/train/tokens_used": 394015200,
+ "theoretical_loss": 4.0714118204579535,
+ "tokens_seen": 373555200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007166138415245738,
+ "loss": 1.432,
+ "theoretical_loss": 4.0714118204579535,
+ "tokens_seen": 373555200
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165977933801405,
+ "loss": 1.4711,
+ "theoretical_loss": 4.0713315603426645,
+ "tokens_seen": 373620736
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165817452357072,
+ "loss": 1.7222,
+ "theoretical_loss": 4.071251318245516,
+ "tokens_seen": 373686272
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165656970912739,
+ "loss": 1.4663,
+ "theoretical_loss": 4.071171094159304,
+ "tokens_seen": 373751808
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165496489468406,
+ "loss": 1.5914,
+ "theoretical_loss": 4.071090888076828,
+ "tokens_seen": 373817344
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165336008024073,
+ "loss": 1.4531,
+ "theoretical_loss": 4.071010699990892,
+ "tokens_seen": 373882880
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000716517552657974,
+ "loss": 1.5228,
+ "theoretical_loss": 4.070930529894306,
+ "tokens_seen": 373948416
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007165015045135407,
+ "loss": 1.6131,
+ "theoretical_loss": 4.070850377779882,
+ "tokens_seen": 374013952
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164854563691074,
+ "loss": 1.578,
+ "theoretical_loss": 4.070770243640436,
+ "tokens_seen": 374079488
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164694082246741,
+ "loss": 1.6162,
+ "theoretical_loss": 4.07069012746879,
+ "tokens_seen": 374145024
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164533600802407,
+ "loss": 1.4642,
+ "theoretical_loss": 4.070610029257768,
+ "tokens_seen": 374210560
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164373119358075,
+ "loss": 1.596,
+ "theoretical_loss": 4.0705299490002,
+ "tokens_seen": 374276096
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164212637913741,
+ "loss": 1.5887,
+ "theoretical_loss": 4.070449886688919,
+ "tokens_seen": 374341632
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007164052156469409,
+ "loss": 1.466,
+ "theoretical_loss": 4.0703698423167625,
+ "tokens_seen": 374407168
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007163891675025075,
+ "loss": 1.55,
+ "theoretical_loss": 4.070289815876571,
+ "tokens_seen": 374472704
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007163731193580743,
+ "loss": 1.6113,
+ "theoretical_loss": 4.070209807361191,
+ "tokens_seen": 374538240
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007163570712136409,
+ "loss": 1.5242,
+ "theoretical_loss": 4.0701298167634725,
+ "tokens_seen": 374603776
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007163410230692077,
+ "loss": 1.5492,
+ "theoretical_loss": 4.0700498440762685,
+ "tokens_seen": 374669312
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007163249749247743,
+ "loss": 1.5498,
+ "theoretical_loss": 4.069969889292437,
+ "tokens_seen": 374734848
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000716308926780341,
+ "loss": 1.6181,
+ "theoretical_loss": 4.069889952404841,
+ "tokens_seen": 374800384
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162928786359077,
+ "loss": 1.4905,
+ "theoretical_loss": 4.069810033406345,
+ "tokens_seen": 374865920
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162768304914744,
+ "loss": 1.6594,
+ "theoretical_loss": 4.06973013228982,
+ "tokens_seen": 374931456
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162607823470411,
+ "loss": 1.59,
+ "theoretical_loss": 4.0696502490481405,
+ "tokens_seen": 374996992
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162447342026078,
+ "loss": 1.5982,
+ "theoretical_loss": 4.069570383674185,
+ "tokens_seen": 375062528
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162286860581745,
+ "loss": 1.5867,
+ "theoretical_loss": 4.069490536160835,
+ "tokens_seen": 375128064
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007162126379137412,
+ "loss": 1.5181,
+ "theoretical_loss": 4.069410706500978,
+ "tokens_seen": 375193600
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161965897693079,
+ "loss": 1.5475,
+ "theoretical_loss": 4.0693308946875035,
+ "tokens_seen": 375259136
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161805416248746,
+ "loss": 1.5137,
+ "theoretical_loss": 4.069251100713307,
+ "tokens_seen": 375324672
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161644934804413,
+ "loss": 1.558,
+ "theoretical_loss": 4.069171324571288,
+ "tokens_seen": 375390208
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000716148445336008,
+ "loss": 1.5597,
+ "theoretical_loss": 4.069091566254347,
+ "tokens_seen": 375455744
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161323971915747,
+ "loss": 1.5059,
+ "theoretical_loss": 4.069011825755393,
+ "tokens_seen": 375521280
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161163490471414,
+ "loss": 1.4777,
+ "theoretical_loss": 4.068932103067335,
+ "tokens_seen": 375586816
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007161003009027081,
+ "loss": 1.5031,
+ "theoretical_loss": 4.06885239818309,
+ "tokens_seen": 375652352
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160842527582748,
+ "loss": 1.5192,
+ "theoretical_loss": 4.068772711095575,
+ "tokens_seen": 375717888
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160682046138415,
+ "loss": 1.6938,
+ "theoretical_loss": 4.068693041797715,
+ "tokens_seen": 375783424
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160521564694082,
+ "loss": 1.6846,
+ "theoretical_loss": 4.068613390282437,
+ "tokens_seen": 375848960
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160361083249749,
+ "loss": 1.6436,
+ "theoretical_loss": 4.06853375654267,
+ "tokens_seen": 375914496
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160200601805416,
+ "loss": 1.4518,
+ "theoretical_loss": 4.068454140571351,
+ "tokens_seen": 375980032
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007160040120361083,
+ "loss": 1.6677,
+ "theoretical_loss": 4.068374542361418,
+ "tokens_seen": 376045568
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000715987963891675,
+ "loss": 1.6108,
+ "theoretical_loss": 4.068294961905817,
+ "tokens_seen": 376111104
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007159719157472417,
+ "loss": 1.5951,
+ "theoretical_loss": 4.068215399197492,
+ "tokens_seen": 376176640
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007159558676028084,
+ "loss": 1.6626,
+ "theoretical_loss": 4.068135854229396,
+ "tokens_seen": 376242176
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007159398194583751,
+ "loss": 1.5851,
+ "theoretical_loss": 4.068056326994485,
+ "tokens_seen": 376307712
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007159237713139418,
+ "loss": 1.549,
+ "theoretical_loss": 4.067976817485717,
+ "tokens_seen": 376373248
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007159077231695085,
+ "loss": 1.534,
+ "theoretical_loss": 4.067897325696056,
+ "tokens_seen": 376438784
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007158916750250753,
+ "loss": 1.6247,
+ "theoretical_loss": 4.06781785161847,
+ "tokens_seen": 376504320
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000715875626880642,
+ "loss": 1.574,
+ "theoretical_loss": 4.067738395245929,
+ "tokens_seen": 376569856
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007158595787362087,
+ "loss": 1.5068,
+ "theoretical_loss": 4.067658956571409,
+ "tokens_seen": 376635392
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007158435305917753,
+ "loss": 1.56,
+ "theoretical_loss": 4.067579535587891,
+ "tokens_seen": 376700928
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007158274824473421,
+ "loss": 1.5091,
+ "theoretical_loss": 4.067500132288357,
+ "tokens_seen": 376766464
+ },
+ {
+ "epoch": 0.11,
+ "objective/train/docs_used": 131935,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4308035373687744,
+ "objective/train/theoretical_loss": 4.067420746665794,
+ "objective/train/tokens_used": 397292000,
+ "theoretical_loss": 4.067420746665794,
+ "tokens_seen": 376832000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007158114343029087,
+ "loss": 1.5559,
+ "theoretical_loss": 4.067420746665794,
+ "tokens_seen": 376832000
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007157953861584755,
+ "loss": 1.5028,
+ "theoretical_loss": 4.067341378713195,
+ "tokens_seen": 376897536
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007157793380140421,
+ "loss": 1.5996,
+ "theoretical_loss": 4.0672620284235546,
+ "tokens_seen": 376963072
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007157632898696089,
+ "loss": 1.5491,
+ "theoretical_loss": 4.067182695789872,
+ "tokens_seen": 377028608
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007157472417251756,
+ "loss": 1.5053,
+ "theoretical_loss": 4.06710338080515,
+ "tokens_seen": 377094144
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007157311935807423,
+ "loss": 1.5987,
+ "theoretical_loss": 4.067024083462399,
+ "tokens_seen": 377159680
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000715715145436309,
+ "loss": 1.4151,
+ "theoretical_loss": 4.066944803754628,
+ "tokens_seen": 377225216
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156990972918757,
+ "loss": 1.4533,
+ "theoretical_loss": 4.0668655416748525,
+ "tokens_seen": 377290752
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156830491474424,
+ "loss": 1.5566,
+ "theoretical_loss": 4.066786297216092,
+ "tokens_seen": 377356288
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156670010030091,
+ "loss": 1.4728,
+ "theoretical_loss": 4.066707070371371,
+ "tokens_seen": 377421824
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156509528585758,
+ "loss": 1.5469,
+ "theoretical_loss": 4.066627861133716,
+ "tokens_seen": 377487360
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156349047141425,
+ "loss": 1.4933,
+ "theoretical_loss": 4.066548669496157,
+ "tokens_seen": 377552896
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156188565697092,
+ "loss": 1.5501,
+ "theoretical_loss": 4.066469495451732,
+ "tokens_seen": 377618432
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007156028084252759,
+ "loss": 1.6296,
+ "theoretical_loss": 4.066390338993479,
+ "tokens_seen": 377683968
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007155867602808426,
+ "loss": 1.6332,
+ "theoretical_loss": 4.0663112001144395,
+ "tokens_seen": 377749504
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007155707121364093,
+ "loss": 1.6292,
+ "theoretical_loss": 4.066232078807664,
+ "tokens_seen": 377815040
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000715554663991976,
+ "loss": 1.5593,
+ "theoretical_loss": 4.0661529750662,
+ "tokens_seen": 377880576
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007155386158475427,
+ "loss": 1.5156,
+ "theoretical_loss": 4.066073888883105,
+ "tokens_seen": 377946112
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007155225677031094,
+ "loss": 1.5514,
+ "theoretical_loss": 4.065994820251438,
+ "tokens_seen": 378011648
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007155065195586761,
+ "loss": 1.4833,
+ "theoretical_loss": 4.065915769164262,
+ "tokens_seen": 378077184
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154904714142428,
+ "loss": 1.6372,
+ "theoretical_loss": 4.065836735614642,
+ "tokens_seen": 378142720
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154744232698095,
+ "loss": 1.6646,
+ "theoretical_loss": 4.06575771959565,
+ "tokens_seen": 378208256
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154583751253762,
+ "loss": 1.6973,
+ "theoretical_loss": 4.065678721100362,
+ "tokens_seen": 378273792
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154423269809429,
+ "loss": 1.6011,
+ "theoretical_loss": 4.065599740121855,
+ "tokens_seen": 378339328
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154262788365096,
+ "loss": 1.5382,
+ "theoretical_loss": 4.065520776653212,
+ "tokens_seen": 378404864
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007154102306920763,
+ "loss": 1.5479,
+ "theoretical_loss": 4.06544183068752,
+ "tokens_seen": 378470400
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.000715394182547643,
+ "loss": 1.5046,
+ "theoretical_loss": 4.0653629022178706,
+ "tokens_seen": 378535936
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007153781344032097,
+ "loss": 1.502,
+ "theoretical_loss": 4.065283991237355,
+ "tokens_seen": 378601472
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007153620862587764,
+ "loss": 1.553,
+ "theoretical_loss": 4.065205097739075,
+ "tokens_seen": 378667008
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007153460381143431,
+ "loss": 1.5149,
+ "theoretical_loss": 4.065126221716131,
+ "tokens_seen": 378732544
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007153299899699098,
+ "loss": 1.5736,
+ "theoretical_loss": 4.0650473631616295,
+ "tokens_seen": 378798080
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007153139418254765,
+ "loss": 1.5831,
+ "theoretical_loss": 4.064968522068681,
+ "tokens_seen": 378863616
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152978936810432,
+ "loss": 1.6488,
+ "theoretical_loss": 4.0648896984304,
+ "tokens_seen": 378929152
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152818455366099,
+ "loss": 1.5114,
+ "theoretical_loss": 4.064810892239903,
+ "tokens_seen": 378994688
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152657973921766,
+ "loss": 1.4931,
+ "theoretical_loss": 4.064732103490312,
+ "tokens_seen": 379060224
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152497492477433,
+ "loss": 1.5598,
+ "theoretical_loss": 4.064653332174753,
+ "tokens_seen": 379125760
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.00071523370110331,
+ "loss": 1.6029,
+ "theoretical_loss": 4.064574578286356,
+ "tokens_seen": 379191296
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152176529588767,
+ "loss": 1.6075,
+ "theoretical_loss": 4.064495841818255,
+ "tokens_seen": 379256832
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007152016048144434,
+ "loss": 1.5738,
+ "theoretical_loss": 4.064417122763587,
+ "tokens_seen": 379322368
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007151855566700101,
+ "loss": 1.5304,
+ "theoretical_loss": 4.064338421115491,
+ "tokens_seen": 379387904
+ },
+ {
+ "epoch": 0.11,
+ "learning_rate": 0.0007151695085255768,
+ "loss": 1.5089,
+ "theoretical_loss": 4.064259736867116,
+ "tokens_seen": 379453440
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007151534603811435,
+ "loss": 1.5295,
+ "theoretical_loss": 4.0641810700116086,
+ "tokens_seen": 379518976
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007151374122367102,
+ "loss": 1.553,
+ "theoretical_loss": 4.064102420542122,
+ "tokens_seen": 379584512
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007151213640922769,
+ "loss": 1.5785,
+ "theoretical_loss": 4.064023788451814,
+ "tokens_seen": 379650048
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007151053159478436,
+ "loss": 1.5725,
+ "theoretical_loss": 4.063945173733844,
+ "tokens_seen": 379715584
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007150892678034103,
+ "loss": 1.3953,
+ "theoretical_loss": 4.063866576381377,
+ "tokens_seen": 379781120
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000715073219658977,
+ "loss": 1.4806,
+ "theoretical_loss": 4.063787996387583,
+ "tokens_seen": 379846656
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007150571715145436,
+ "loss": 1.5389,
+ "theoretical_loss": 4.063709433745632,
+ "tokens_seen": 379912192
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007150411233701104,
+ "loss": 1.5711,
+ "theoretical_loss": 4.063630888448701,
+ "tokens_seen": 379977728
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000715025075225677,
+ "loss": 1.6166,
+ "theoretical_loss": 4.06355236048997,
+ "tokens_seen": 380043264
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 133175,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.637108325958252,
+ "objective/train/theoretical_loss": 4.063473849862623,
+ "objective/train/tokens_used": 400568800,
+ "theoretical_loss": 4.063473849862623,
+ "tokens_seen": 380108800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007150090270812438,
+ "loss": 1.6212,
+ "theoretical_loss": 4.063473849862623,
+ "tokens_seen": 380108800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149929789368104,
+ "loss": 1.4958,
+ "theoretical_loss": 4.063395356559848,
+ "tokens_seen": 380174336
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149769307923772,
+ "loss": 1.5352,
+ "theoretical_loss": 4.063316880574836,
+ "tokens_seen": 380239872
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149608826479438,
+ "loss": 1.5525,
+ "theoretical_loss": 4.063238421900783,
+ "tokens_seen": 380305408
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149448345035106,
+ "loss": 1.6395,
+ "theoretical_loss": 4.063159980530887,
+ "tokens_seen": 380370944
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149287863590772,
+ "loss": 1.5751,
+ "theoretical_loss": 4.063081556458353,
+ "tokens_seen": 380436480
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007149127382146439,
+ "loss": 1.5307,
+ "theoretical_loss": 4.063003149676387,
+ "tokens_seen": 380502016
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148966900702106,
+ "loss": 1.6268,
+ "theoretical_loss": 4.062924760178198,
+ "tokens_seen": 380567552
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148806419257773,
+ "loss": 1.4769,
+ "theoretical_loss": 4.062846387957005,
+ "tokens_seen": 380633088
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714864593781344,
+ "loss": 1.4427,
+ "theoretical_loss": 4.062768033006023,
+ "tokens_seen": 380698624
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148485456369107,
+ "loss": 1.5785,
+ "theoretical_loss": 4.062689695318475,
+ "tokens_seen": 380764160
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148324974924774,
+ "loss": 1.7042,
+ "theoretical_loss": 4.062611374887587,
+ "tokens_seen": 380829696
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148164493480441,
+ "loss": 1.4121,
+ "theoretical_loss": 4.062533071706589,
+ "tokens_seen": 380895232
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007148004012036108,
+ "loss": 1.529,
+ "theoretical_loss": 4.062454785768716,
+ "tokens_seen": 380960768
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007147843530591775,
+ "loss": 1.4973,
+ "theoretical_loss": 4.062376517067204,
+ "tokens_seen": 381026304
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007147683049147442,
+ "loss": 1.5547,
+ "theoretical_loss": 4.062298265595294,
+ "tokens_seen": 381091840
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007147522567703109,
+ "loss": 1.4552,
+ "theoretical_loss": 4.062220031346234,
+ "tokens_seen": 381157376
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007147362086258776,
+ "loss": 1.5088,
+ "theoretical_loss": 4.06214181431327,
+ "tokens_seen": 381222912
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007147201604814443,
+ "loss": 1.5998,
+ "theoretical_loss": 4.062063614489656,
+ "tokens_seen": 381288448
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714704112337011,
+ "loss": 1.3875,
+ "theoretical_loss": 4.061985431868648,
+ "tokens_seen": 381353984
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146880641925778,
+ "loss": 1.5849,
+ "theoretical_loss": 4.061907266443508,
+ "tokens_seen": 381419520
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146720160481444,
+ "loss": 1.5982,
+ "theoretical_loss": 4.061829118207498,
+ "tokens_seen": 381485056
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146559679037112,
+ "loss": 1.5771,
+ "theoretical_loss": 4.061750987153888,
+ "tokens_seen": 381550592
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146399197592778,
+ "loss": 1.6583,
+ "theoretical_loss": 4.061672873275949,
+ "tokens_seen": 381616128
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146238716148446,
+ "loss": 1.5146,
+ "theoretical_loss": 4.061594776566958,
+ "tokens_seen": 381681664
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007146078234704112,
+ "loss": 1.6203,
+ "theoretical_loss": 4.061516697020191,
+ "tokens_seen": 381747200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714591775325978,
+ "loss": 1.4899,
+ "theoretical_loss": 4.061438634628933,
+ "tokens_seen": 381812736
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007145757271815446,
+ "loss": 1.4785,
+ "theoretical_loss": 4.061360589386473,
+ "tokens_seen": 381878272
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007145596790371114,
+ "loss": 1.5746,
+ "theoretical_loss": 4.061282561286098,
+ "tokens_seen": 381943808
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714543630892678,
+ "loss": 1.4623,
+ "theoretical_loss": 4.061204550321104,
+ "tokens_seen": 382009344
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007145275827482448,
+ "loss": 1.4579,
+ "theoretical_loss": 4.061126556484791,
+ "tokens_seen": 382074880
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007145115346038114,
+ "loss": 1.7142,
+ "theoretical_loss": 4.06104857977046,
+ "tokens_seen": 382140416
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007144954864593782,
+ "loss": 1.4523,
+ "theoretical_loss": 4.060970620171415,
+ "tokens_seen": 382205952
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007144794383149448,
+ "loss": 1.569,
+ "theoretical_loss": 4.060892677680967,
+ "tokens_seen": 382271488
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007144633901705116,
+ "loss": 1.56,
+ "theoretical_loss": 4.06081475229243,
+ "tokens_seen": 382337024
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007144473420260782,
+ "loss": 1.595,
+ "theoretical_loss": 4.060736843999121,
+ "tokens_seen": 382402560
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714431293881645,
+ "loss": 1.4555,
+ "theoretical_loss": 4.060658952794359,
+ "tokens_seen": 382468096
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007144152457372116,
+ "loss": 1.5569,
+ "theoretical_loss": 4.06058107867147,
+ "tokens_seen": 382533632
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143991975927784,
+ "loss": 1.5,
+ "theoretical_loss": 4.060503221623783,
+ "tokens_seen": 382599168
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714383149448345,
+ "loss": 1.5665,
+ "theoretical_loss": 4.060425381644629,
+ "tokens_seen": 382664704
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143671013039118,
+ "loss": 1.5303,
+ "theoretical_loss": 4.060347558727344,
+ "tokens_seen": 382730240
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143510531594785,
+ "loss": 1.5049,
+ "theoretical_loss": 4.060269752865268,
+ "tokens_seen": 382795776
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143350050150452,
+ "loss": 1.5552,
+ "theoretical_loss": 4.060191964051744,
+ "tokens_seen": 382861312
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143189568706119,
+ "loss": 1.3659,
+ "theoretical_loss": 4.060114192280119,
+ "tokens_seen": 382926848
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007143029087261786,
+ "loss": 1.5707,
+ "theoretical_loss": 4.0600364375437445,
+ "tokens_seen": 382992384
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007142868605817453,
+ "loss": 1.644,
+ "theoretical_loss": 4.059958699835974,
+ "tokens_seen": 383057920
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714270812437312,
+ "loss": 1.5578,
+ "theoretical_loss": 4.059880979150167,
+ "tokens_seen": 383123456
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007142547642928787,
+ "loss": 1.4528,
+ "theoretical_loss": 4.059803275479684,
+ "tokens_seen": 383188992
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007142387161484454,
+ "loss": 1.5817,
+ "theoretical_loss": 4.059725588817891,
+ "tokens_seen": 383254528
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007142226680040121,
+ "loss": 1.4266,
+ "theoretical_loss": 4.0596479191581585,
+ "tokens_seen": 383320064
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 134130,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6504217386245728,
+ "objective/train/theoretical_loss": 4.059570266493859,
+ "objective/train/tokens_used": 403845600,
+ "theoretical_loss": 4.059570266493859,
+ "tokens_seen": 383385600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007142066198595788,
+ "loss": 1.6386,
+ "theoretical_loss": 4.059570266493859,
+ "tokens_seen": 383385600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007141905717151455,
+ "loss": 1.5463,
+ "theoretical_loss": 4.059492630818369,
+ "tokens_seen": 383451136
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007141745235707122,
+ "loss": 1.6065,
+ "theoretical_loss": 4.05941501212507,
+ "tokens_seen": 383516672
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007141584754262789,
+ "loss": 1.4486,
+ "theoretical_loss": 4.059337410407346,
+ "tokens_seen": 383582208
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007141424272818456,
+ "loss": 1.5162,
+ "theoretical_loss": 4.059259825658584,
+ "tokens_seen": 383647744
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007141263791374123,
+ "loss": 1.5191,
+ "theoretical_loss": 4.059182257872177,
+ "tokens_seen": 383713280
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000714110330992979,
+ "loss": 1.6177,
+ "theoretical_loss": 4.059104707041519,
+ "tokens_seen": 383778816
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140942828485457,
+ "loss": 1.4974,
+ "theoretical_loss": 4.059027173160011,
+ "tokens_seen": 383844352
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140782347041124,
+ "loss": 1.5206,
+ "theoretical_loss": 4.0589496562210545,
+ "tokens_seen": 383909888
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140621865596791,
+ "loss": 1.5041,
+ "theoretical_loss": 4.058872156218055,
+ "tokens_seen": 383975424
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140461384152458,
+ "loss": 1.4481,
+ "theoretical_loss": 4.058794673144425,
+ "tokens_seen": 384040960
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140300902708125,
+ "loss": 1.4762,
+ "theoretical_loss": 4.0587172069935775,
+ "tokens_seen": 384106496
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007140140421263792,
+ "loss": 1.5379,
+ "theoretical_loss": 4.05863975775893,
+ "tokens_seen": 384172032
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139979939819459,
+ "loss": 1.6316,
+ "theoretical_loss": 4.058562325433902,
+ "tokens_seen": 384237568
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139819458375126,
+ "loss": 1.4715,
+ "theoretical_loss": 4.058484910011922,
+ "tokens_seen": 384303104
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139658976930793,
+ "loss": 1.535,
+ "theoretical_loss": 4.058407511486415,
+ "tokens_seen": 384368640
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000713949849548646,
+ "loss": 1.591,
+ "theoretical_loss": 4.058330129850816,
+ "tokens_seen": 384434176
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139338014042127,
+ "loss": 1.4796,
+ "theoretical_loss": 4.058252765098558,
+ "tokens_seen": 384499712
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139177532597794,
+ "loss": 1.3905,
+ "theoretical_loss": 4.058175417223084,
+ "tokens_seen": 384565248
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007139017051153461,
+ "loss": 1.6414,
+ "theoretical_loss": 4.058098086217836,
+ "tokens_seen": 384630784
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138856569709128,
+ "loss": 1.4973,
+ "theoretical_loss": 4.058020772076259,
+ "tokens_seen": 384696320
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138696088264795,
+ "loss": 1.6776,
+ "theoretical_loss": 4.057943474791806,
+ "tokens_seen": 384761856
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138535606820462,
+ "loss": 1.4589,
+ "theoretical_loss": 4.0578661943579295,
+ "tokens_seen": 384827392
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138375125376129,
+ "loss": 1.5205,
+ "theoretical_loss": 4.05778893076809,
+ "tokens_seen": 384892928
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138214643931796,
+ "loss": 1.6148,
+ "theoretical_loss": 4.057711684015747,
+ "tokens_seen": 384958464
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007138054162487463,
+ "loss": 1.4956,
+ "theoretical_loss": 4.057634454094365,
+ "tokens_seen": 385024000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000713789368104313,
+ "loss": 1.5085,
+ "theoretical_loss": 4.057557240997415,
+ "tokens_seen": 385089536
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007137733199598797,
+ "loss": 1.4795,
+ "theoretical_loss": 4.057480044718368,
+ "tokens_seen": 385155072
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007137572718154464,
+ "loss": 1.533,
+ "theoretical_loss": 4.057402865250701,
+ "tokens_seen": 385220608
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007137412236710131,
+ "loss": 1.5496,
+ "theoretical_loss": 4.057325702587893,
+ "tokens_seen": 385286144
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007137251755265798,
+ "loss": 1.6184,
+ "theoretical_loss": 4.057248556723429,
+ "tokens_seen": 385351680
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007137091273821465,
+ "loss": 1.5929,
+ "theoretical_loss": 4.057171427650793,
+ "tokens_seen": 385417216
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136930792377132,
+ "loss": 1.5673,
+ "theoretical_loss": 4.0570943153634795,
+ "tokens_seen": 385482752
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136770310932799,
+ "loss": 1.4974,
+ "theoretical_loss": 4.0570172198549805,
+ "tokens_seen": 385548288
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136609829488466,
+ "loss": 1.5252,
+ "theoretical_loss": 4.056940141118794,
+ "tokens_seen": 385613824
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136449348044133,
+ "loss": 1.5163,
+ "theoretical_loss": 4.056863079148422,
+ "tokens_seen": 385679360
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136288866599799,
+ "loss": 1.5341,
+ "theoretical_loss": 4.05678603393737,
+ "tokens_seen": 385744896
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007136128385155467,
+ "loss": 1.5027,
+ "theoretical_loss": 4.056709005479147,
+ "tokens_seen": 385810432
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135967903711133,
+ "loss": 1.4236,
+ "theoretical_loss": 4.056631993767264,
+ "tokens_seen": 385875968
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135807422266801,
+ "loss": 1.6033,
+ "theoretical_loss": 4.056554998795239,
+ "tokens_seen": 385941504
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135646940822467,
+ "loss": 1.5528,
+ "theoretical_loss": 4.05647802055659,
+ "tokens_seen": 386007040
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135486459378135,
+ "loss": 1.6238,
+ "theoretical_loss": 4.056401059044841,
+ "tokens_seen": 386072576
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135325977933801,
+ "loss": 1.5508,
+ "theoretical_loss": 4.056324114253519,
+ "tokens_seen": 386138112
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135165496489468,
+ "loss": 1.4138,
+ "theoretical_loss": 4.056247186176154,
+ "tokens_seen": 386203648
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007135005015045135,
+ "loss": 1.5629,
+ "theoretical_loss": 4.056170274806281,
+ "tokens_seen": 386269184
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134844533600803,
+ "loss": 1.5687,
+ "theoretical_loss": 4.056093380137436,
+ "tokens_seen": 386334720
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134684052156469,
+ "loss": 1.5048,
+ "theoretical_loss": 4.056016502163162,
+ "tokens_seen": 386400256
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134523570712137,
+ "loss": 1.6054,
+ "theoretical_loss": 4.055939640877003,
+ "tokens_seen": 386465792
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134363089267803,
+ "loss": 1.5434,
+ "theoretical_loss": 4.055862796272508,
+ "tokens_seen": 386531328
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134202607823471,
+ "loss": 1.4143,
+ "theoretical_loss": 4.055785968343228,
+ "tokens_seen": 386596864
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 135378,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6471995115280151,
+ "objective/train/theoretical_loss": 4.05570915708272,
+ "objective/train/tokens_used": 407122400,
+ "theoretical_loss": 4.05570915708272,
+ "tokens_seen": 386662400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007134042126379137,
+ "loss": 1.5954,
+ "theoretical_loss": 4.05570915708272,
+ "tokens_seen": 386662400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133881644934805,
+ "loss": 1.367,
+ "theoretical_loss": 4.055632362484542,
+ "tokens_seen": 386727936
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133721163490471,
+ "loss": 1.5976,
+ "theoretical_loss": 4.055555584542257,
+ "tokens_seen": 386793472
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133560682046139,
+ "loss": 1.4761,
+ "theoretical_loss": 4.055478823249432,
+ "tokens_seen": 386859008
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133400200601805,
+ "loss": 1.4676,
+ "theoretical_loss": 4.055402078599638,
+ "tokens_seen": 386924544
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133239719157473,
+ "loss": 1.4819,
+ "theoretical_loss": 4.055325350586445,
+ "tokens_seen": 386990080
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007133079237713139,
+ "loss": 1.3562,
+ "theoretical_loss": 4.055248639203433,
+ "tokens_seen": 387055616
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132918756268807,
+ "loss": 1.561,
+ "theoretical_loss": 4.055171944444182,
+ "tokens_seen": 387121152
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132758274824473,
+ "loss": 1.5905,
+ "theoretical_loss": 4.055095266302276,
+ "tokens_seen": 387186688
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132597793380141,
+ "loss": 1.5024,
+ "theoretical_loss": 4.055018604771302,
+ "tokens_seen": 387252224
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132437311935807,
+ "loss": 1.5481,
+ "theoretical_loss": 4.054941959844853,
+ "tokens_seen": 387317760
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132276830491475,
+ "loss": 1.5958,
+ "theoretical_loss": 4.054865331516523,
+ "tokens_seen": 387383296
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007132116349047141,
+ "loss": 1.4962,
+ "theoretical_loss": 4.054788719779911,
+ "tokens_seen": 387448832
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131955867602809,
+ "loss": 1.6014,
+ "theoretical_loss": 4.054712124628617,
+ "tokens_seen": 387514368
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131795386158475,
+ "loss": 1.569,
+ "theoretical_loss": 4.054635546056248,
+ "tokens_seen": 387579904
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131634904714143,
+ "loss": 1.4515,
+ "theoretical_loss": 4.054558984056414,
+ "tokens_seen": 387645440
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131474423269809,
+ "loss": 1.509,
+ "theoretical_loss": 4.054482438622727,
+ "tokens_seen": 387710976
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131313941825477,
+ "loss": 1.3481,
+ "theoretical_loss": 4.0544059097488026,
+ "tokens_seen": 387776512
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007131153460381143,
+ "loss": 1.5887,
+ "theoretical_loss": 4.05432939742826,
+ "tokens_seen": 387842048
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130992978936811,
+ "loss": 1.5527,
+ "theoretical_loss": 4.054252901654724,
+ "tokens_seen": 387907584
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130832497492477,
+ "loss": 1.5288,
+ "theoretical_loss": 4.054176422421821,
+ "tokens_seen": 387973120
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130672016048145,
+ "loss": 1.575,
+ "theoretical_loss": 4.05409995972318,
+ "tokens_seen": 388038656
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130511534603811,
+ "loss": 1.54,
+ "theoretical_loss": 4.054023513552437,
+ "tokens_seen": 388104192
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130351053159479,
+ "loss": 1.476,
+ "theoretical_loss": 4.0539470839032266,
+ "tokens_seen": 388169728
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130190571715146,
+ "loss": 1.5659,
+ "theoretical_loss": 4.053870670769192,
+ "tokens_seen": 388235264
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007130030090270813,
+ "loss": 1.5637,
+ "theoretical_loss": 4.053794274143977,
+ "tokens_seen": 388300800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712986960882648,
+ "loss": 1.5868,
+ "theoretical_loss": 4.053717894021229,
+ "tokens_seen": 388366336
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007129709127382147,
+ "loss": 1.4002,
+ "theoretical_loss": 4.0536415303946,
+ "tokens_seen": 388431872
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007129548645937814,
+ "loss": 1.591,
+ "theoretical_loss": 4.053565183257745,
+ "tokens_seen": 388497408
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007129388164493481,
+ "loss": 1.5506,
+ "theoretical_loss": 4.053488852604322,
+ "tokens_seen": 388562944
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007129227683049148,
+ "loss": 1.483,
+ "theoretical_loss": 4.053412538427993,
+ "tokens_seen": 388628480
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007129067201604815,
+ "loss": 1.6479,
+ "theoretical_loss": 4.053336240722423,
+ "tokens_seen": 388694016
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007128906720160482,
+ "loss": 1.4564,
+ "theoretical_loss": 4.053259959481283,
+ "tokens_seen": 388759552
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007128746238716149,
+ "loss": 1.532,
+ "theoretical_loss": 4.053183694698244,
+ "tokens_seen": 388825088
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007128585757271816,
+ "loss": 1.4543,
+ "theoretical_loss": 4.053107446366981,
+ "tokens_seen": 388890624
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007128425275827483,
+ "loss": 1.7257,
+ "theoretical_loss": 4.053031214481175,
+ "tokens_seen": 388956160
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712826479438315,
+ "loss": 1.5432,
+ "theoretical_loss": 4.052954999034509,
+ "tokens_seen": 389021696
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007128104312938817,
+ "loss": 1.3693,
+ "theoretical_loss": 4.052878800020668,
+ "tokens_seen": 389087232
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127943831494484,
+ "loss": 1.5255,
+ "theoretical_loss": 4.052802617433344,
+ "tokens_seen": 389152768
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127783350050151,
+ "loss": 1.5014,
+ "theoretical_loss": 4.0527264512662295,
+ "tokens_seen": 389218304
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127622868605818,
+ "loss": 1.5613,
+ "theoretical_loss": 4.052650301513021,
+ "tokens_seen": 389283840
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127462387161485,
+ "loss": 1.4304,
+ "theoretical_loss": 4.0525741681674186,
+ "tokens_seen": 389349376
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127301905717152,
+ "loss": 1.4755,
+ "theoretical_loss": 4.052498051223127,
+ "tokens_seen": 389414912
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007127141424272819,
+ "loss": 1.7319,
+ "theoretical_loss": 4.052421950673853,
+ "tokens_seen": 389480448
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126980942828486,
+ "loss": 1.6418,
+ "theoretical_loss": 4.052345866513308,
+ "tokens_seen": 389545984
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126820461384153,
+ "loss": 1.4938,
+ "theoretical_loss": 4.052269798735207,
+ "tokens_seen": 389611520
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712665997993982,
+ "loss": 1.4466,
+ "theoretical_loss": 4.052193747333265,
+ "tokens_seen": 389677056
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126499498495487,
+ "loss": 1.6005,
+ "theoretical_loss": 4.052117712301206,
+ "tokens_seen": 389742592
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126339017051154,
+ "loss": 1.4522,
+ "theoretical_loss": 4.052041693632754,
+ "tokens_seen": 389808128
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126178535606821,
+ "loss": 1.5045,
+ "theoretical_loss": 4.0519656913216355,
+ "tokens_seen": 389873664
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 136223,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4804840087890625,
+ "objective/train/theoretical_loss": 4.051889705361584,
+ "objective/train/tokens_used": 410399200,
+ "theoretical_loss": 4.051889705361584,
+ "tokens_seen": 389939200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007126018054162488,
+ "loss": 1.5287,
+ "theoretical_loss": 4.051889705361584,
+ "tokens_seen": 389939200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007125857572718155,
+ "loss": 1.4526,
+ "theoretical_loss": 4.051813735746334,
+ "tokens_seen": 390004736
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007125697091273822,
+ "loss": 1.6437,
+ "theoretical_loss": 4.051737782469624,
+ "tokens_seen": 390070272
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007125536609829489,
+ "loss": 1.6544,
+ "theoretical_loss": 4.051661845525196,
+ "tokens_seen": 390135808
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007125376128385156,
+ "loss": 1.4723,
+ "theoretical_loss": 4.051585924906796,
+ "tokens_seen": 390201344
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007125215646940823,
+ "loss": 1.7279,
+ "theoretical_loss": 4.051510020608171,
+ "tokens_seen": 390266880
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712505516549649,
+ "loss": 1.5472,
+ "theoretical_loss": 4.051434132623075,
+ "tokens_seen": 390332416
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124894684052157,
+ "loss": 1.699,
+ "theoretical_loss": 4.051358260945262,
+ "tokens_seen": 390397952
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124734202607824,
+ "loss": 1.6237,
+ "theoretical_loss": 4.051282405568494,
+ "tokens_seen": 390463488
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124573721163491,
+ "loss": 1.5859,
+ "theoretical_loss": 4.051206566486531,
+ "tokens_seen": 390529024
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124413239719158,
+ "loss": 1.5356,
+ "theoretical_loss": 4.0511307436931405,
+ "tokens_seen": 390594560
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124252758274825,
+ "loss": 1.558,
+ "theoretical_loss": 4.051054937182092,
+ "tokens_seen": 390660096
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007124092276830492,
+ "loss": 1.5667,
+ "theoretical_loss": 4.0509791469471566,
+ "tokens_seen": 390725632
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007123931795386159,
+ "loss": 1.5082,
+ "theoretical_loss": 4.050903372982113,
+ "tokens_seen": 390791168
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007123771313941826,
+ "loss": 1.6031,
+ "theoretical_loss": 4.050827615280739,
+ "tokens_seen": 390856704
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007123610832497493,
+ "loss": 1.5781,
+ "theoretical_loss": 4.050751873836819,
+ "tokens_seen": 390922240
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712345035105316,
+ "loss": 1.4028,
+ "theoretical_loss": 4.050676148644139,
+ "tokens_seen": 390987776
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007123289869608827,
+ "loss": 1.6489,
+ "theoretical_loss": 4.050600439696489,
+ "tokens_seen": 391053312
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007123129388164494,
+ "loss": 1.473,
+ "theoretical_loss": 4.050524746987664,
+ "tokens_seen": 391118848
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122968906720161,
+ "loss": 1.5548,
+ "theoretical_loss": 4.050449070511458,
+ "tokens_seen": 391184384
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122808425275828,
+ "loss": 1.5028,
+ "theoretical_loss": 4.050373410261673,
+ "tokens_seen": 391249920
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122647943831495,
+ "loss": 1.5715,
+ "theoretical_loss": 4.050297766232113,
+ "tokens_seen": 391315456
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122487462387162,
+ "loss": 1.7176,
+ "theoretical_loss": 4.050222138416583,
+ "tokens_seen": 391380992
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122326980942828,
+ "loss": 1.5573,
+ "theoretical_loss": 4.050146526808896,
+ "tokens_seen": 391446528
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122166499498496,
+ "loss": 1.6053,
+ "theoretical_loss": 4.050070931402864,
+ "tokens_seen": 391512064
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007122006018054162,
+ "loss": 1.4945,
+ "theoretical_loss": 4.049995352192305,
+ "tokens_seen": 391577600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712184553660983,
+ "loss": 1.6051,
+ "theoretical_loss": 4.04991978917104,
+ "tokens_seen": 391643136
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007121685055165496,
+ "loss": 1.6117,
+ "theoretical_loss": 4.049844242332892,
+ "tokens_seen": 391708672
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007121524573721164,
+ "loss": 1.5436,
+ "theoretical_loss": 4.049768711671689,
+ "tokens_seen": 391774208
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000712136409227683,
+ "loss": 1.5156,
+ "theoretical_loss": 4.049693197181262,
+ "tokens_seen": 391839744
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007121203610832498,
+ "loss": 1.5343,
+ "theoretical_loss": 4.049617698855444,
+ "tokens_seen": 391905280
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007121043129388164,
+ "loss": 1.5006,
+ "theoretical_loss": 4.049542216688074,
+ "tokens_seen": 391970816
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007120882647943832,
+ "loss": 1.5838,
+ "theoretical_loss": 4.049466750672994,
+ "tokens_seen": 392036352
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007120722166499498,
+ "loss": 1.6256,
+ "theoretical_loss": 4.049391300804045,
+ "tokens_seen": 392101888
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007120561685055166,
+ "loss": 1.65,
+ "theoretical_loss": 4.049315867075077,
+ "tokens_seen": 392167424
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007120401203610832,
+ "loss": 1.4785,
+ "theoretical_loss": 4.0492404494799406,
+ "tokens_seen": 392232960
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.00071202407221665,
+ "loss": 1.5928,
+ "theoretical_loss": 4.049165048012489,
+ "tokens_seen": 392298496
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007120080240722166,
+ "loss": 1.4771,
+ "theoretical_loss": 4.049089662666582,
+ "tokens_seen": 392364032
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007119919759277834,
+ "loss": 1.4962,
+ "theoretical_loss": 4.04901429343608,
+ "tokens_seen": 392429568
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.00071197592778335,
+ "loss": 1.5464,
+ "theoretical_loss": 4.048938940314848,
+ "tokens_seen": 392495104
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007119598796389168,
+ "loss": 1.439,
+ "theoretical_loss": 4.048863603296753,
+ "tokens_seen": 392560640
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007119438314944834,
+ "loss": 1.4998,
+ "theoretical_loss": 4.048788282375666,
+ "tokens_seen": 392626176
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007119277833500502,
+ "loss": 1.5768,
+ "theoretical_loss": 4.048712977545463,
+ "tokens_seen": 392691712
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007119117352056168,
+ "loss": 1.4402,
+ "theoretical_loss": 4.048637688800021,
+ "tokens_seen": 392757248
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007118956870611836,
+ "loss": 1.5783,
+ "theoretical_loss": 4.048562416133221,
+ "tokens_seen": 392822784
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007118796389167502,
+ "loss": 1.5998,
+ "theoretical_loss": 4.048487159538949,
+ "tokens_seen": 392888320
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711863590772317,
+ "loss": 1.5081,
+ "theoretical_loss": 4.048411919011091,
+ "tokens_seen": 392953856
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007118475426278836,
+ "loss": 1.5313,
+ "theoretical_loss": 4.04833669454354,
+ "tokens_seen": 393019392
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007118314944834504,
+ "loss": 1.5179,
+ "theoretical_loss": 4.0482614861301895,
+ "tokens_seen": 393084928
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711815446339017,
+ "loss": 1.5538,
+ "theoretical_loss": 4.048186293764939,
+ "tokens_seen": 393150464
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 137415,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5672664642333984,
+ "objective/train/theoretical_loss": 4.048111117441689,
+ "objective/train/tokens_used": 413676000,
+ "theoretical_loss": 4.048111117441689,
+ "tokens_seen": 393216000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117993981945838,
+ "loss": 1.6009,
+ "theoretical_loss": 4.048111117441689,
+ "tokens_seen": 393216000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117833500501504,
+ "loss": 1.6339,
+ "theoretical_loss": 4.048035957154343,
+ "tokens_seen": 393281536
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117673019057172,
+ "loss": 1.6105,
+ "theoretical_loss": 4.047960812896811,
+ "tokens_seen": 393347072
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117512537612838,
+ "loss": 1.6307,
+ "theoretical_loss": 4.047885684663004,
+ "tokens_seen": 393412608
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117352056168506,
+ "loss": 1.5179,
+ "theoretical_loss": 4.047810572446836,
+ "tokens_seen": 393478144
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007117191574724172,
+ "loss": 1.617,
+ "theoretical_loss": 4.047735476242225,
+ "tokens_seen": 393543680
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711703109327984,
+ "loss": 1.6172,
+ "theoretical_loss": 4.047660396043092,
+ "tokens_seen": 393609216
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007116870611835506,
+ "loss": 1.4355,
+ "theoretical_loss": 4.047585331843363,
+ "tokens_seen": 393674752
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007116710130391174,
+ "loss": 1.5347,
+ "theoretical_loss": 4.047510283636965,
+ "tokens_seen": 393740288
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711654964894684,
+ "loss": 1.4901,
+ "theoretical_loss": 4.04743525141783,
+ "tokens_seen": 393805824
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007116389167502508,
+ "loss": 1.5138,
+ "theoretical_loss": 4.047360235179891,
+ "tokens_seen": 393871360
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007116228686058175,
+ "loss": 1.5696,
+ "theoretical_loss": 4.047285234917087,
+ "tokens_seen": 393936896
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007116068204613842,
+ "loss": 1.4289,
+ "theoretical_loss": 4.04721025062336,
+ "tokens_seen": 394002432
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007115907723169509,
+ "loss": 1.5431,
+ "theoretical_loss": 4.047135282292653,
+ "tokens_seen": 394067968
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007115747241725176,
+ "loss": 1.5128,
+ "theoretical_loss": 4.0470603299189145,
+ "tokens_seen": 394133504
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007115586760280843,
+ "loss": 1.5557,
+ "theoretical_loss": 4.0469853934960955,
+ "tokens_seen": 394199040
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711542627883651,
+ "loss": 1.5148,
+ "theoretical_loss": 4.046910473018151,
+ "tokens_seen": 394264576
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007115265797392177,
+ "loss": 1.6591,
+ "theoretical_loss": 4.046835568479038,
+ "tokens_seen": 394330112
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007115105315947844,
+ "loss": 1.4581,
+ "theoretical_loss": 4.046760679872717,
+ "tokens_seen": 394395648
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114944834503511,
+ "loss": 1.4585,
+ "theoretical_loss": 4.046685807193153,
+ "tokens_seen": 394461184
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114784353059178,
+ "loss": 1.5061,
+ "theoretical_loss": 4.046610950434314,
+ "tokens_seen": 394526720
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114623871614845,
+ "loss": 1.5028,
+ "theoretical_loss": 4.04653610959017,
+ "tokens_seen": 394592256
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114463390170512,
+ "loss": 1.5819,
+ "theoretical_loss": 4.046461284654696,
+ "tokens_seen": 394657792
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114302908726179,
+ "loss": 1.5578,
+ "theoretical_loss": 4.046386475621869,
+ "tokens_seen": 394723328
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007114142427281846,
+ "loss": 1.5475,
+ "theoretical_loss": 4.046311682485669,
+ "tokens_seen": 394788864
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113981945837513,
+ "loss": 1.5736,
+ "theoretical_loss": 4.0462369052400815,
+ "tokens_seen": 394854400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711382146439318,
+ "loss": 1.5481,
+ "theoretical_loss": 4.0461621438790925,
+ "tokens_seen": 394919936
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113660982948847,
+ "loss": 1.6394,
+ "theoretical_loss": 4.046087398396693,
+ "tokens_seen": 394985472
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113500501504514,
+ "loss": 1.5032,
+ "theoretical_loss": 4.046012668786877,
+ "tokens_seen": 395051008
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113340020060181,
+ "loss": 1.5094,
+ "theoretical_loss": 4.045937955043641,
+ "tokens_seen": 395116544
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113179538615848,
+ "loss": 1.6007,
+ "theoretical_loss": 4.045863257160986,
+ "tokens_seen": 395182080
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007113019057171515,
+ "loss": 1.4421,
+ "theoretical_loss": 4.045788575132915,
+ "tokens_seen": 395247616
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007112858575727182,
+ "loss": 1.5609,
+ "theoretical_loss": 4.045713908953435,
+ "tokens_seen": 395313152
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007112698094282849,
+ "loss": 1.4736,
+ "theoretical_loss": 4.045639258616557,
+ "tokens_seen": 395378688
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007112537612838516,
+ "loss": 1.5692,
+ "theoretical_loss": 4.045564624116293,
+ "tokens_seen": 395444224
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007112377131394183,
+ "loss": 1.5839,
+ "theoretical_loss": 4.04549000544666,
+ "tokens_seen": 395509760
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711221664994985,
+ "loss": 1.3912,
+ "theoretical_loss": 4.045415402601678,
+ "tokens_seen": 395575296
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007112056168505517,
+ "loss": 1.4101,
+ "theoretical_loss": 4.045340815575372,
+ "tokens_seen": 395640832
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007111895687061184,
+ "loss": 1.582,
+ "theoretical_loss": 4.045266244361765,
+ "tokens_seen": 395706368
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007111735205616851,
+ "loss": 1.5496,
+ "theoretical_loss": 4.045191688954889,
+ "tokens_seen": 395771904
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007111574724172518,
+ "loss": 1.5711,
+ "theoretical_loss": 4.045117149348776,
+ "tokens_seen": 395837440
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007111414242728185,
+ "loss": 1.4717,
+ "theoretical_loss": 4.045042625537462,
+ "tokens_seen": 395902976
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007111253761283852,
+ "loss": 1.5493,
+ "theoretical_loss": 4.044968117514987,
+ "tokens_seen": 395968512
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711109327983952,
+ "loss": 1.5777,
+ "theoretical_loss": 4.044893625275392,
+ "tokens_seen": 396034048
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007110932798395186,
+ "loss": 1.4861,
+ "theoretical_loss": 4.044819148812726,
+ "tokens_seen": 396099584
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007110772316950854,
+ "loss": 1.5666,
+ "theoretical_loss": 4.044744688121035,
+ "tokens_seen": 396165120
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000711061183550652,
+ "loss": 1.451,
+ "theoretical_loss": 4.044670243194372,
+ "tokens_seen": 396230656
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007110451354062188,
+ "loss": 1.5889,
+ "theoretical_loss": 4.044595814026794,
+ "tokens_seen": 396296192
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007110290872617854,
+ "loss": 1.41,
+ "theoretical_loss": 4.044521400612357,
+ "tokens_seen": 396361728
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007110130391173522,
+ "loss": 1.5389,
+ "theoretical_loss": 4.044447002945125,
+ "tokens_seen": 396427264
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 138316,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4225183725357056,
+ "objective/train/theoretical_loss": 4.044372621019162,
+ "objective/train/tokens_used": 416952800,
+ "theoretical_loss": 4.044372621019162,
+ "tokens_seen": 396492800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109969909729188,
+ "loss": 1.3996,
+ "theoretical_loss": 4.044372621019162,
+ "tokens_seen": 396492800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109809428284855,
+ "loss": 1.6224,
+ "theoretical_loss": 4.044298254828538,
+ "tokens_seen": 396558336
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109648946840522,
+ "loss": 1.3741,
+ "theoretical_loss": 4.044223904367324,
+ "tokens_seen": 396623872
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109488465396189,
+ "loss": 1.5343,
+ "theoretical_loss": 4.044149569629594,
+ "tokens_seen": 396689408
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109327983951856,
+ "loss": 1.5944,
+ "theoretical_loss": 4.044075250609426,
+ "tokens_seen": 396754944
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007109167502507523,
+ "loss": 1.4888,
+ "theoretical_loss": 4.044000947300901,
+ "tokens_seen": 396820480
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000710900702106319,
+ "loss": 1.4342,
+ "theoretical_loss": 4.043926659698106,
+ "tokens_seen": 396886016
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108846539618857,
+ "loss": 1.6039,
+ "theoretical_loss": 4.043852387795125,
+ "tokens_seen": 396951552
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108686058174524,
+ "loss": 1.5694,
+ "theoretical_loss": 4.043778131586052,
+ "tokens_seen": 397017088
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108525576730191,
+ "loss": 1.6645,
+ "theoretical_loss": 4.043703891064979,
+ "tokens_seen": 397082624
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108365095285857,
+ "loss": 1.5529,
+ "theoretical_loss": 4.043629666226003,
+ "tokens_seen": 397148160
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108204613841525,
+ "loss": 1.4129,
+ "theoretical_loss": 4.043555457063226,
+ "tokens_seen": 397213696
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007108044132397191,
+ "loss": 1.5982,
+ "theoretical_loss": 4.043481263570751,
+ "tokens_seen": 397279232
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107883650952859,
+ "loss": 1.5183,
+ "theoretical_loss": 4.043407085742684,
+ "tokens_seen": 397344768
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107723169508525,
+ "loss": 1.5224,
+ "theoretical_loss": 4.0433329235731374,
+ "tokens_seen": 397410304
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107562688064193,
+ "loss": 1.576,
+ "theoretical_loss": 4.043258777056221,
+ "tokens_seen": 397475840
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107402206619859,
+ "loss": 1.5558,
+ "theoretical_loss": 4.043184646186053,
+ "tokens_seen": 397541376
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107241725175527,
+ "loss": 1.5146,
+ "theoretical_loss": 4.043110530956754,
+ "tokens_seen": 397606912
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007107081243731193,
+ "loss": 1.5506,
+ "theoretical_loss": 4.043036431362444,
+ "tokens_seen": 397672448
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106920762286861,
+ "loss": 1.5227,
+ "theoretical_loss": 4.042962347397252,
+ "tokens_seen": 397737984
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106760280842527,
+ "loss": 1.467,
+ "theoretical_loss": 4.042888279055304,
+ "tokens_seen": 397803520
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106599799398195,
+ "loss": 1.4999,
+ "theoretical_loss": 4.0428142263307345,
+ "tokens_seen": 397869056
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106439317953861,
+ "loss": 1.5015,
+ "theoretical_loss": 4.042740189217677,
+ "tokens_seen": 397934592
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106278836509529,
+ "loss": 1.4486,
+ "theoretical_loss": 4.042666167710273,
+ "tokens_seen": 398000128
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007106118355065195,
+ "loss": 1.5478,
+ "theoretical_loss": 4.042592161802662,
+ "tokens_seen": 398065664
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105957873620863,
+ "loss": 1.4975,
+ "theoretical_loss": 4.042518171488989,
+ "tokens_seen": 398131200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105797392176529,
+ "loss": 1.5275,
+ "theoretical_loss": 4.042444196763403,
+ "tokens_seen": 398196736
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105636910732197,
+ "loss": 1.5028,
+ "theoretical_loss": 4.042370237620053,
+ "tokens_seen": 398262272
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105476429287863,
+ "loss": 1.4962,
+ "theoretical_loss": 4.042296294053097,
+ "tokens_seen": 398327808
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105315947843531,
+ "loss": 1.6266,
+ "theoretical_loss": 4.0422223660566905,
+ "tokens_seen": 398393344
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007105155466399197,
+ "loss": 1.5788,
+ "theoretical_loss": 4.0421484536249945,
+ "tokens_seen": 398458880
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104994984954865,
+ "loss": 1.7934,
+ "theoretical_loss": 4.042074556752172,
+ "tokens_seen": 398524416
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104834503510531,
+ "loss": 1.629,
+ "theoretical_loss": 4.042000675432391,
+ "tokens_seen": 398589952
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104674022066199,
+ "loss": 1.4217,
+ "theoretical_loss": 4.041926809659821,
+ "tokens_seen": 398655488
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104513540621865,
+ "loss": 1.5271,
+ "theoretical_loss": 4.0418529594286365,
+ "tokens_seen": 398721024
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104353059177533,
+ "loss": 1.5935,
+ "theoretical_loss": 4.041779124733012,
+ "tokens_seen": 398786560
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104192577733199,
+ "loss": 1.4487,
+ "theoretical_loss": 4.041705305567129,
+ "tokens_seen": 398852096
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007104032096288867,
+ "loss": 1.7103,
+ "theoretical_loss": 4.041631501925169,
+ "tokens_seen": 398917632
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103871614844533,
+ "loss": 1.5326,
+ "theoretical_loss": 4.041557713801319,
+ "tokens_seen": 398983168
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103711133400201,
+ "loss": 1.5504,
+ "theoretical_loss": 4.041483941189766,
+ "tokens_seen": 399048704
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103550651955867,
+ "loss": 1.58,
+ "theoretical_loss": 4.041410184084703,
+ "tokens_seen": 399114240
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103390170511535,
+ "loss": 1.5606,
+ "theoretical_loss": 4.041336442480326,
+ "tokens_seen": 399179776
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103229689067201,
+ "loss": 1.6753,
+ "theoretical_loss": 4.041262716370833,
+ "tokens_seen": 399245312
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007103069207622869,
+ "loss": 1.6089,
+ "theoretical_loss": 4.041189005750424,
+ "tokens_seen": 399310848
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007102908726178535,
+ "loss": 1.4364,
+ "theoretical_loss": 4.041115310613307,
+ "tokens_seen": 399376384
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007102748244734203,
+ "loss": 1.6079,
+ "theoretical_loss": 4.0410416309536865,
+ "tokens_seen": 399441920
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000710258776328987,
+ "loss": 1.5491,
+ "theoretical_loss": 4.040967966765774,
+ "tokens_seen": 399507456
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007102427281845537,
+ "loss": 1.5805,
+ "theoretical_loss": 4.040894318043785,
+ "tokens_seen": 399572992
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007102266800401204,
+ "loss": 1.6833,
+ "theoretical_loss": 4.040820684781934,
+ "tokens_seen": 399638528
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007102106318956871,
+ "loss": 1.5103,
+ "theoretical_loss": 4.040747066974443,
+ "tokens_seen": 399704064
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 139589,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5951530933380127,
+ "objective/train/theoretical_loss": 4.040673464615535,
+ "objective/train/tokens_used": 420229600,
+ "theoretical_loss": 4.040673464615535,
+ "tokens_seen": 399769600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101945837512538,
+ "loss": 1.5524,
+ "theoretical_loss": 4.040673464615535,
+ "tokens_seen": 399769600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101785356068205,
+ "loss": 1.5063,
+ "theoretical_loss": 4.040599877699437,
+ "tokens_seen": 399835136
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101624874623872,
+ "loss": 1.5295,
+ "theoretical_loss": 4.040526306220377,
+ "tokens_seen": 399900672
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101464393179539,
+ "loss": 1.6562,
+ "theoretical_loss": 4.040452750172588,
+ "tokens_seen": 399966208
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101303911735206,
+ "loss": 1.5602,
+ "theoretical_loss": 4.040379209550307,
+ "tokens_seen": 400031744
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007101143430290873,
+ "loss": 1.5235,
+ "theoretical_loss": 4.0403056843477705,
+ "tokens_seen": 400097280
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000710098294884654,
+ "loss": 1.4961,
+ "theoretical_loss": 4.040232174559223,
+ "tokens_seen": 400162816
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100822467402207,
+ "loss": 1.5648,
+ "theoretical_loss": 4.040158680178907,
+ "tokens_seen": 400228352
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100661985957874,
+ "loss": 1.4735,
+ "theoretical_loss": 4.040085201201072,
+ "tokens_seen": 400293888
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100501504513541,
+ "loss": 1.5627,
+ "theoretical_loss": 4.0400117376199685,
+ "tokens_seen": 400359424
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100341023069208,
+ "loss": 1.6345,
+ "theoretical_loss": 4.039938289429852,
+ "tokens_seen": 400424960
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100180541624875,
+ "loss": 1.6676,
+ "theoretical_loss": 4.039864856624979,
+ "tokens_seen": 400490496
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007100020060180542,
+ "loss": 1.5924,
+ "theoretical_loss": 4.039791439199609,
+ "tokens_seen": 400556032
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007099859578736209,
+ "loss": 1.5713,
+ "theoretical_loss": 4.039718037148006,
+ "tokens_seen": 400621568
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007099699097291876,
+ "loss": 1.5793,
+ "theoretical_loss": 4.039644650464438,
+ "tokens_seen": 400687104
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007099538615847543,
+ "loss": 1.5998,
+ "theoretical_loss": 4.039571279143172,
+ "tokens_seen": 400752640
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709937813440321,
+ "loss": 1.541,
+ "theoretical_loss": 4.039497923178484,
+ "tokens_seen": 400818176
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007099217652958877,
+ "loss": 1.4307,
+ "theoretical_loss": 4.039424582564646,
+ "tokens_seen": 400883712
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007099057171514545,
+ "loss": 1.4884,
+ "theoretical_loss": 4.03935125729594,
+ "tokens_seen": 400949248
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098896690070211,
+ "loss": 1.6443,
+ "theoretical_loss": 4.039277947366647,
+ "tokens_seen": 401014784
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098736208625879,
+ "loss": 1.5543,
+ "theoretical_loss": 4.039204652771051,
+ "tokens_seen": 401080320
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098575727181545,
+ "loss": 1.4671,
+ "theoretical_loss": 4.039131373503441,
+ "tokens_seen": 401145856
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098415245737213,
+ "loss": 1.5812,
+ "theoretical_loss": 4.039058109558107,
+ "tokens_seen": 401211392
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098254764292879,
+ "loss": 1.6335,
+ "theoretical_loss": 4.038984860929346,
+ "tokens_seen": 401276928
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007098094282848547,
+ "loss": 1.5945,
+ "theoretical_loss": 4.038911627611451,
+ "tokens_seen": 401342464
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097933801404213,
+ "loss": 1.5385,
+ "theoretical_loss": 4.038838409598727,
+ "tokens_seen": 401408000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097773319959881,
+ "loss": 1.491,
+ "theoretical_loss": 4.038765206885472,
+ "tokens_seen": 401473536
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097612838515547,
+ "loss": 1.5328,
+ "theoretical_loss": 4.038692019465997,
+ "tokens_seen": 401539072
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097452357071215,
+ "loss": 1.573,
+ "theoretical_loss": 4.038618847334609,
+ "tokens_seen": 401604608
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097291875626881,
+ "loss": 1.7025,
+ "theoretical_loss": 4.038545690485622,
+ "tokens_seen": 401670144
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007097131394182549,
+ "loss": 1.5662,
+ "theoretical_loss": 4.038472548913349,
+ "tokens_seen": 401735680
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096970912738215,
+ "loss": 1.59,
+ "theoretical_loss": 4.038399422612112,
+ "tokens_seen": 401801216
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096810431293883,
+ "loss": 1.5199,
+ "theoretical_loss": 4.038326311576229,
+ "tokens_seen": 401866752
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096649949849549,
+ "loss": 1.542,
+ "theoretical_loss": 4.038253215800028,
+ "tokens_seen": 401932288
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096489468405217,
+ "loss": 1.5482,
+ "theoretical_loss": 4.038180135277834,
+ "tokens_seen": 401997824
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096328986960883,
+ "loss": 1.6522,
+ "theoretical_loss": 4.038107070003979,
+ "tokens_seen": 402063360
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096168505516551,
+ "loss": 1.5936,
+ "theoretical_loss": 4.038034019972796,
+ "tokens_seen": 402128896
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007096008024072217,
+ "loss": 1.6195,
+ "theoretical_loss": 4.037960985178623,
+ "tokens_seen": 402194432
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095847542627884,
+ "loss": 1.5161,
+ "theoretical_loss": 4.037887965615798,
+ "tokens_seen": 402259968
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095687061183551,
+ "loss": 1.5262,
+ "theoretical_loss": 4.0378149612786665,
+ "tokens_seen": 402325504
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095526579739218,
+ "loss": 1.6219,
+ "theoretical_loss": 4.037741972161571,
+ "tokens_seen": 402391040
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095366098294885,
+ "loss": 1.584,
+ "theoretical_loss": 4.037668998258863,
+ "tokens_seen": 402456576
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095205616850552,
+ "loss": 1.532,
+ "theoretical_loss": 4.037596039564892,
+ "tokens_seen": 402522112
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007095045135406219,
+ "loss": 1.6007,
+ "theoretical_loss": 4.0375230960740165,
+ "tokens_seen": 402587648
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007094884653961886,
+ "loss": 1.5508,
+ "theoretical_loss": 4.03745016778059,
+ "tokens_seen": 402653184
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007094724172517553,
+ "loss": 1.5275,
+ "theoretical_loss": 4.037377254678977,
+ "tokens_seen": 402718720
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709456369107322,
+ "loss": 1.587,
+ "theoretical_loss": 4.037304356763539,
+ "tokens_seen": 402784256
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007094403209628886,
+ "loss": 1.5588,
+ "theoretical_loss": 4.037231474028644,
+ "tokens_seen": 402849792
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007094242728184554,
+ "loss": 1.4966,
+ "theoretical_loss": 4.037158606468663,
+ "tokens_seen": 402915328
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709408224674022,
+ "loss": 1.5709,
+ "theoretical_loss": 4.037085754077966,
+ "tokens_seen": 402980864
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 140534,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5789742469787598,
+ "objective/train/theoretical_loss": 4.037012916850932,
+ "objective/train/tokens_used": 423506400,
+ "theoretical_loss": 4.037012916850932,
+ "tokens_seen": 403046400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093921765295888,
+ "loss": 1.5523,
+ "theoretical_loss": 4.037012916850932,
+ "tokens_seen": 403046400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093761283851554,
+ "loss": 1.5749,
+ "theoretical_loss": 4.036940094781937,
+ "tokens_seen": 403111936
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093600802407222,
+ "loss": 1.596,
+ "theoretical_loss": 4.036867287865366,
+ "tokens_seen": 403177472
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093440320962888,
+ "loss": 1.4619,
+ "theoretical_loss": 4.036794496095602,
+ "tokens_seen": 403243008
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093279839518556,
+ "loss": 1.587,
+ "theoretical_loss": 4.0367217194670335,
+ "tokens_seen": 403308544
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007093119358074222,
+ "loss": 1.5353,
+ "theoretical_loss": 4.036648957974052,
+ "tokens_seen": 403374080
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709295887662989,
+ "loss": 1.5422,
+ "theoretical_loss": 4.03657621161105,
+ "tokens_seen": 403439616
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007092798395185556,
+ "loss": 1.6412,
+ "theoretical_loss": 4.036503480372424,
+ "tokens_seen": 403505152
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007092637913741224,
+ "loss": 1.5548,
+ "theoretical_loss": 4.036430764252577,
+ "tokens_seen": 403570688
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709247743229689,
+ "loss": 1.5112,
+ "theoretical_loss": 4.036358063245908,
+ "tokens_seen": 403636224
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007092316950852558,
+ "loss": 1.515,
+ "theoretical_loss": 4.036285377346826,
+ "tokens_seen": 403701760
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007092156469408224,
+ "loss": 1.422,
+ "theoretical_loss": 4.036212706549738,
+ "tokens_seen": 403767296
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091995987963892,
+ "loss": 1.6223,
+ "theoretical_loss": 4.036140050849056,
+ "tokens_seen": 403832832
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091835506519558,
+ "loss": 1.4715,
+ "theoretical_loss": 4.036067410239196,
+ "tokens_seen": 403898368
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091675025075226,
+ "loss": 1.5133,
+ "theoretical_loss": 4.035994784714574,
+ "tokens_seen": 403963904
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091514543630892,
+ "loss": 1.4933,
+ "theoretical_loss": 4.035922174269611,
+ "tokens_seen": 404029440
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709135406218656,
+ "loss": 1.3653,
+ "theoretical_loss": 4.035849578898731,
+ "tokens_seen": 404094976
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091193580742226,
+ "loss": 1.5697,
+ "theoretical_loss": 4.035776998596361,
+ "tokens_seen": 404160512
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007091033099297894,
+ "loss": 1.5925,
+ "theoretical_loss": 4.035704433356932,
+ "tokens_seen": 404226048
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000709087261785356,
+ "loss": 1.5928,
+ "theoretical_loss": 4.035631883174873,
+ "tokens_seen": 404291584
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007090712136409228,
+ "loss": 1.51,
+ "theoretical_loss": 4.035559348044622,
+ "tokens_seen": 404357120
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007090551654964894,
+ "loss": 1.5758,
+ "theoretical_loss": 4.035486827960617,
+ "tokens_seen": 404422656
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007090391173520562,
+ "loss": 1.4794,
+ "theoretical_loss": 4.035414322917299,
+ "tokens_seen": 404488192
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007090230692076228,
+ "loss": 1.643,
+ "theoretical_loss": 4.035341832909114,
+ "tokens_seen": 404553728
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007090070210631896,
+ "loss": 1.5227,
+ "theoretical_loss": 4.035269357930507,
+ "tokens_seen": 404619264
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007089909729187562,
+ "loss": 1.5348,
+ "theoretical_loss": 4.03519689797593,
+ "tokens_seen": 404684800
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708974924774323,
+ "loss": 1.5649,
+ "theoretical_loss": 4.035124453039836,
+ "tokens_seen": 404750336
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007089588766298896,
+ "loss": 1.6123,
+ "theoretical_loss": 4.035052023116679,
+ "tokens_seen": 404815872
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007089428284854564,
+ "loss": 1.5052,
+ "theoretical_loss": 4.0349796082009215,
+ "tokens_seen": 404881408
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708926780341023,
+ "loss": 1.493,
+ "theoretical_loss": 4.034907208287024,
+ "tokens_seen": 404946944
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007089107321965898,
+ "loss": 1.6608,
+ "theoretical_loss": 4.034834823369451,
+ "tokens_seen": 405012480
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007088946840521565,
+ "loss": 1.5893,
+ "theoretical_loss": 4.034762453442672,
+ "tokens_seen": 405078016
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007088786359077232,
+ "loss": 1.4565,
+ "theoretical_loss": 4.034690098501155,
+ "tokens_seen": 405143552
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007088625877632899,
+ "loss": 1.5087,
+ "theoretical_loss": 4.034617758539377,
+ "tokens_seen": 405209088
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007088465396188566,
+ "loss": 1.5578,
+ "theoretical_loss": 4.034545433551814,
+ "tokens_seen": 405274624
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007088304914744233,
+ "loss": 1.4697,
+ "theoretical_loss": 4.034473123532944,
+ "tokens_seen": 405340160
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.00070881444332999,
+ "loss": 1.5152,
+ "theoretical_loss": 4.034400828477251,
+ "tokens_seen": 405405696
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087983951855567,
+ "loss": 1.5794,
+ "theoretical_loss": 4.03432854837922,
+ "tokens_seen": 405471232
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087823470411234,
+ "loss": 1.4242,
+ "theoretical_loss": 4.03425628323334,
+ "tokens_seen": 405536768
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087662988966901,
+ "loss": 1.5615,
+ "theoretical_loss": 4.034184033034102,
+ "tokens_seen": 405602304
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087502507522568,
+ "loss": 1.5607,
+ "theoretical_loss": 4.034111797776001,
+ "tokens_seen": 405667840
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087342026078235,
+ "loss": 1.5839,
+ "theoretical_loss": 4.034039577453533,
+ "tokens_seen": 405733376
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007087181544633902,
+ "loss": 1.3925,
+ "theoretical_loss": 4.033967372061199,
+ "tokens_seen": 405798912
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708702106318957,
+ "loss": 1.5731,
+ "theoretical_loss": 4.033895181593501,
+ "tokens_seen": 405864448
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007086860581745236,
+ "loss": 1.4833,
+ "theoretical_loss": 4.033823006044947,
+ "tokens_seen": 405929984
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007086700100300904,
+ "loss": 1.5742,
+ "theoretical_loss": 4.033750845410044,
+ "tokens_seen": 405995520
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708653961885657,
+ "loss": 1.5395,
+ "theoretical_loss": 4.033678699683303,
+ "tokens_seen": 406061056
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007086379137412238,
+ "loss": 1.5878,
+ "theoretical_loss": 4.033606568859241,
+ "tokens_seen": 406126592
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007086218655967904,
+ "loss": 1.4696,
+ "theoretical_loss": 4.033534452932376,
+ "tokens_seen": 406192128
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007086058174523572,
+ "loss": 1.5745,
+ "theoretical_loss": 4.033462351897225,
+ "tokens_seen": 406257664
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 141740,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.756189227104187,
+ "objective/train/theoretical_loss": 4.033390265748315,
+ "objective/train/tokens_used": 426783200,
+ "theoretical_loss": 4.033390265748315,
+ "tokens_seen": 406323200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007085897693079238,
+ "loss": 1.5844,
+ "theoretical_loss": 4.033390265748315,
+ "tokens_seen": 406323200
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007085737211634906,
+ "loss": 1.4169,
+ "theoretical_loss": 4.033318194480169,
+ "tokens_seen": 406388736
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007085576730190572,
+ "loss": 1.6124,
+ "theoretical_loss": 4.0332461380873195,
+ "tokens_seen": 406454272
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708541624874624,
+ "loss": 1.6184,
+ "theoretical_loss": 4.033174096564295,
+ "tokens_seen": 406519808
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007085255767301906,
+ "loss": 1.4786,
+ "theoretical_loss": 4.033102069905635,
+ "tokens_seen": 406585344
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007085095285857574,
+ "loss": 1.4864,
+ "theoretical_loss": 4.033030058105872,
+ "tokens_seen": 406650880
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708493480441324,
+ "loss": 1.5946,
+ "theoretical_loss": 4.032958061159551,
+ "tokens_seen": 406716416
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007084774322968908,
+ "loss": 1.5697,
+ "theoretical_loss": 4.032886079061214,
+ "tokens_seen": 406781952
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007084613841524574,
+ "loss": 1.6082,
+ "theoretical_loss": 4.032814111805407,
+ "tokens_seen": 406847488
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007084453360080242,
+ "loss": 1.6108,
+ "theoretical_loss": 4.03274215938668,
+ "tokens_seen": 406913024
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007084292878635908,
+ "loss": 1.5825,
+ "theoretical_loss": 4.0326702217995845,
+ "tokens_seen": 406978560
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007084132397191576,
+ "loss": 1.6002,
+ "theoretical_loss": 4.0325982990386775,
+ "tokens_seen": 407044096
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007083971915747242,
+ "loss": 1.5136,
+ "theoretical_loss": 4.032526391098515,
+ "tokens_seen": 407109632
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708381143430291,
+ "loss": 1.5522,
+ "theoretical_loss": 4.032454497973659,
+ "tokens_seen": 407175168
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007083650952858576,
+ "loss": 1.5036,
+ "theoretical_loss": 4.032382619658673,
+ "tokens_seen": 407240704
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007083490471414244,
+ "loss": 1.5827,
+ "theoretical_loss": 4.0323107561481235,
+ "tokens_seen": 407306240
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708332998996991,
+ "loss": 1.4544,
+ "theoretical_loss": 4.0322389074365805,
+ "tokens_seen": 407371776
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007083169508525578,
+ "loss": 1.6027,
+ "theoretical_loss": 4.032167073518615,
+ "tokens_seen": 407437312
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007083009027081244,
+ "loss": 1.5711,
+ "theoretical_loss": 4.032095254388803,
+ "tokens_seen": 407502848
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007082848545636912,
+ "loss": 1.5284,
+ "theoretical_loss": 4.032023450041724,
+ "tokens_seen": 407568384
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007082688064192578,
+ "loss": 1.5628,
+ "theoretical_loss": 4.031951660471956,
+ "tokens_seen": 407633920
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007082527582748246,
+ "loss": 1.5589,
+ "theoretical_loss": 4.031879885674085,
+ "tokens_seen": 407699456
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007082367101303912,
+ "loss": 1.6216,
+ "theoretical_loss": 4.031808125642697,
+ "tokens_seen": 407764992
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708220661985958,
+ "loss": 1.552,
+ "theoretical_loss": 4.031736380372382,
+ "tokens_seen": 407830528
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007082046138415246,
+ "loss": 1.4943,
+ "theoretical_loss": 4.031664649857731,
+ "tokens_seen": 407896064
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007081885656970914,
+ "loss": 1.5147,
+ "theoretical_loss": 4.031592934093341,
+ "tokens_seen": 407961600
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000708172517552658,
+ "loss": 1.4208,
+ "theoretical_loss": 4.031521233073809,
+ "tokens_seen": 408027136
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007081564694082247,
+ "loss": 1.5565,
+ "theoretical_loss": 4.031449546793736,
+ "tokens_seen": 408092672
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007081404212637914,
+ "loss": 1.5643,
+ "theoretical_loss": 4.031377875247726,
+ "tokens_seen": 408158208
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007081243731193581,
+ "loss": 1.6523,
+ "theoretical_loss": 4.031306218430385,
+ "tokens_seen": 408223744
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007081083249749248,
+ "loss": 1.5344,
+ "theoretical_loss": 4.031234576336322,
+ "tokens_seen": 408289280
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080922768304915,
+ "loss": 1.5772,
+ "theoretical_loss": 4.0311629489601515,
+ "tokens_seen": 408354816
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080762286860582,
+ "loss": 1.6145,
+ "theoretical_loss": 4.031091336296487,
+ "tokens_seen": 408420352
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080601805416249,
+ "loss": 1.5051,
+ "theoretical_loss": 4.031019738339946,
+ "tokens_seen": 408485888
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080441323971915,
+ "loss": 1.5164,
+ "theoretical_loss": 4.03094815508515,
+ "tokens_seen": 408551424
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080280842527583,
+ "loss": 1.5636,
+ "theoretical_loss": 4.030876586526722,
+ "tokens_seen": 408616960
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007080120361083249,
+ "loss": 1.3939,
+ "theoretical_loss": 4.0308050326592895,
+ "tokens_seen": 408682496
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079959879638917,
+ "loss": 1.4653,
+ "theoretical_loss": 4.030733493477481,
+ "tokens_seen": 408748032
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079799398194583,
+ "loss": 1.5727,
+ "theoretical_loss": 4.030661968975929,
+ "tokens_seen": 408813568
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079638916750251,
+ "loss": 1.5448,
+ "theoretical_loss": 4.030590459149267,
+ "tokens_seen": 408879104
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079478435305917,
+ "loss": 1.453,
+ "theoretical_loss": 4.030518963992135,
+ "tokens_seen": 408944640
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079317953861585,
+ "loss": 1.5661,
+ "theoretical_loss": 4.030447483499171,
+ "tokens_seen": 409010176
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007079157472417251,
+ "loss": 1.504,
+ "theoretical_loss": 4.03037601766502,
+ "tokens_seen": 409075712
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078996990972919,
+ "loss": 1.4467,
+ "theoretical_loss": 4.030304566484329,
+ "tokens_seen": 409141248
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078836509528585,
+ "loss": 1.5275,
+ "theoretical_loss": 4.030233129951744,
+ "tokens_seen": 409206784
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078676028084253,
+ "loss": 1.5733,
+ "theoretical_loss": 4.03016170806192,
+ "tokens_seen": 409272320
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078515546639919,
+ "loss": 1.4817,
+ "theoretical_loss": 4.030090300809509,
+ "tokens_seen": 409337856
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078355065195587,
+ "loss": 1.6054,
+ "theoretical_loss": 4.03001890818917,
+ "tokens_seen": 409403392
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078194583751253,
+ "loss": 1.4895,
+ "theoretical_loss": 4.029947530195563,
+ "tokens_seen": 409468928
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007078034102306921,
+ "loss": 1.547,
+ "theoretical_loss": 4.029876166823351,
+ "tokens_seen": 409534464
+ },
+ {
+ "epoch": 0.12,
+ "objective/train/docs_used": 142735,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6000092029571533,
+ "objective/train/theoretical_loss": 4.029804818067199,
+ "objective/train/tokens_used": 430060000,
+ "theoretical_loss": 4.029804818067199,
+ "tokens_seen": 409600000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077873620862587,
+ "loss": 1.5053,
+ "theoretical_loss": 4.029804818067199,
+ "tokens_seen": 409600000
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077713139418255,
+ "loss": 1.4704,
+ "theoretical_loss": 4.029733483921777,
+ "tokens_seen": 409665536
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077552657973921,
+ "loss": 1.5721,
+ "theoretical_loss": 4.0296621643817545,
+ "tokens_seen": 409731072
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077392176529589,
+ "loss": 1.6005,
+ "theoretical_loss": 4.029590859441807,
+ "tokens_seen": 409796608
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077231695085255,
+ "loss": 1.5829,
+ "theoretical_loss": 4.029519569096612,
+ "tokens_seen": 409862144
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007077071213640923,
+ "loss": 1.5834,
+ "theoretical_loss": 4.029448293340847,
+ "tokens_seen": 409927680
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076910732196589,
+ "loss": 1.4792,
+ "theoretical_loss": 4.029377032169197,
+ "tokens_seen": 409993216
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076750250752257,
+ "loss": 1.4813,
+ "theoretical_loss": 4.029305785576347,
+ "tokens_seen": 410058752
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076589769307923,
+ "loss": 1.597,
+ "theoretical_loss": 4.029234553556985,
+ "tokens_seen": 410124288
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076429287863591,
+ "loss": 1.5541,
+ "theoretical_loss": 4.029163336105801,
+ "tokens_seen": 410189824
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076268806419257,
+ "loss": 1.5452,
+ "theoretical_loss": 4.029092133217489,
+ "tokens_seen": 410255360
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007076108324974925,
+ "loss": 1.4583,
+ "theoretical_loss": 4.029020944886746,
+ "tokens_seen": 410320896
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007075947843530591,
+ "loss": 1.5313,
+ "theoretical_loss": 4.0289497711082705,
+ "tokens_seen": 410386432
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007075787362086259,
+ "loss": 1.5252,
+ "theoretical_loss": 4.028878611876765,
+ "tokens_seen": 410451968
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007075626880641925,
+ "loss": 1.5533,
+ "theoretical_loss": 4.028807467186935,
+ "tokens_seen": 410517504
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007075466399197593,
+ "loss": 1.5356,
+ "theoretical_loss": 4.028736337033488,
+ "tokens_seen": 410583040
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.000707530591775326,
+ "loss": 1.4687,
+ "theoretical_loss": 4.028665221411133,
+ "tokens_seen": 410648576
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007075145436308927,
+ "loss": 1.5098,
+ "theoretical_loss": 4.028594120314585,
+ "tokens_seen": 410714112
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074984954864595,
+ "loss": 1.5612,
+ "theoretical_loss": 4.028523033738558,
+ "tokens_seen": 410779648
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074824473420261,
+ "loss": 1.4409,
+ "theoretical_loss": 4.028451961677772,
+ "tokens_seen": 410845184
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074663991975929,
+ "loss": 1.3957,
+ "theoretical_loss": 4.028380904126948,
+ "tokens_seen": 410910720
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074503510531595,
+ "loss": 1.616,
+ "theoretical_loss": 4.0283098610808095,
+ "tokens_seen": 410976256
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074343029087263,
+ "loss": 1.5218,
+ "theoretical_loss": 4.028238832534084,
+ "tokens_seen": 411041792
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074182547642929,
+ "loss": 1.5817,
+ "theoretical_loss": 4.028167818481502,
+ "tokens_seen": 411107328
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007074022066198597,
+ "loss": 1.5122,
+ "theoretical_loss": 4.0280968189177955,
+ "tokens_seen": 411172864
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073861584754263,
+ "loss": 1.5176,
+ "theoretical_loss": 4.028025833837699,
+ "tokens_seen": 411238400
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073701103309931,
+ "loss": 1.5894,
+ "theoretical_loss": 4.027954863235951,
+ "tokens_seen": 411303936
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073540621865597,
+ "loss": 1.3881,
+ "theoretical_loss": 4.0278839071072925,
+ "tokens_seen": 411369472
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073380140421265,
+ "loss": 1.492,
+ "theoretical_loss": 4.027812965446467,
+ "tokens_seen": 411435008
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073219658976931,
+ "loss": 1.6257,
+ "theoretical_loss": 4.02774203824822,
+ "tokens_seen": 411500544
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007073059177532599,
+ "loss": 1.4058,
+ "theoretical_loss": 4.0276711255073,
+ "tokens_seen": 411566080
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072898696088265,
+ "loss": 1.4869,
+ "theoretical_loss": 4.027600227218461,
+ "tokens_seen": 411631616
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072738214643933,
+ "loss": 1.5341,
+ "theoretical_loss": 4.027529343376456,
+ "tokens_seen": 411697152
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072577733199599,
+ "loss": 1.541,
+ "theoretical_loss": 4.027458473976041,
+ "tokens_seen": 411762688
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072417251755267,
+ "loss": 1.5091,
+ "theoretical_loss": 4.027387619011977,
+ "tokens_seen": 411828224
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072256770310933,
+ "loss": 1.4396,
+ "theoretical_loss": 4.027316778479027,
+ "tokens_seen": 411893760
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007072096288866601,
+ "loss": 1.5064,
+ "theoretical_loss": 4.027245952371956,
+ "tokens_seen": 411959296
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071935807422267,
+ "loss": 1.4344,
+ "theoretical_loss": 4.027175140685532,
+ "tokens_seen": 412024832
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071775325977935,
+ "loss": 1.4299,
+ "theoretical_loss": 4.027104343414527,
+ "tokens_seen": 412090368
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071614844533601,
+ "loss": 1.509,
+ "theoretical_loss": 4.027033560553713,
+ "tokens_seen": 412155904
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071454363089269,
+ "loss": 1.4293,
+ "theoretical_loss": 4.026962792097867,
+ "tokens_seen": 412221440
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071293881644935,
+ "loss": 1.423,
+ "theoretical_loss": 4.026892038041767,
+ "tokens_seen": 412286976
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007071133400200603,
+ "loss": 1.6205,
+ "theoretical_loss": 4.026821298380197,
+ "tokens_seen": 412352512
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007070972918756269,
+ "loss": 1.5077,
+ "theoretical_loss": 4.026750573107939,
+ "tokens_seen": 412418048
+ },
+ {
+ "epoch": 0.12,
+ "learning_rate": 0.0007070812437311937,
+ "loss": 1.5667,
+ "theoretical_loss": 4.0266798622197815,
+ "tokens_seen": 412483584
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007070651955867603,
+ "loss": 1.5787,
+ "theoretical_loss": 4.026609165710514,
+ "tokens_seen": 412549120
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007070491474423271,
+ "loss": 1.5379,
+ "theoretical_loss": 4.02653848357493,
+ "tokens_seen": 412614656
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007070330992978937,
+ "loss": 1.4793,
+ "theoretical_loss": 4.026467815807823,
+ "tokens_seen": 412680192
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007070170511534605,
+ "loss": 1.5342,
+ "theoretical_loss": 4.026397162403994,
+ "tokens_seen": 412745728
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007070010030090271,
+ "loss": 1.5203,
+ "theoretical_loss": 4.0263265233582395,
+ "tokens_seen": 412811264
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 143929,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.583418369293213,
+ "objective/train/theoretical_loss": 4.026255898665367,
+ "objective/train/tokens_used": 433336800,
+ "theoretical_loss": 4.026255898665367,
+ "tokens_seen": 412876800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069849548645939,
+ "loss": 1.5316,
+ "theoretical_loss": 4.026255898665367,
+ "tokens_seen": 412876800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069689067201605,
+ "loss": 1.5474,
+ "theoretical_loss": 4.02618528832018,
+ "tokens_seen": 412942336
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069528585757273,
+ "loss": 1.5566,
+ "theoretical_loss": 4.026114692317489,
+ "tokens_seen": 413007872
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069368104312939,
+ "loss": 1.2872,
+ "theoretical_loss": 4.026044110652105,
+ "tokens_seen": 413073408
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069207622868607,
+ "loss": 1.5705,
+ "theoretical_loss": 4.025973543318841,
+ "tokens_seen": 413138944
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007069047141424273,
+ "loss": 1.5252,
+ "theoretical_loss": 4.025902990312517,
+ "tokens_seen": 413204480
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068886659979941,
+ "loss": 1.5701,
+ "theoretical_loss": 4.025832451627949,
+ "tokens_seen": 413270016
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068726178535607,
+ "loss": 1.5828,
+ "theoretical_loss": 4.025761927259961,
+ "tokens_seen": 413335552
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068565697091275,
+ "loss": 1.4946,
+ "theoretical_loss": 4.0256914172033795,
+ "tokens_seen": 413401088
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068405215646941,
+ "loss": 1.5515,
+ "theoretical_loss": 4.025620921453029,
+ "tokens_seen": 413466624
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068244734202609,
+ "loss": 1.639,
+ "theoretical_loss": 4.025550440003744,
+ "tokens_seen": 413532160
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007068084252758275,
+ "loss": 1.549,
+ "theoretical_loss": 4.025479972850354,
+ "tokens_seen": 413597696
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007067923771313943,
+ "loss": 1.4433,
+ "theoretical_loss": 4.025409519987696,
+ "tokens_seen": 413663232
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007067763289869609,
+ "loss": 1.4926,
+ "theoretical_loss": 4.025339081410609,
+ "tokens_seen": 413728768
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007067602808425276,
+ "loss": 1.5137,
+ "theoretical_loss": 4.025268657113934,
+ "tokens_seen": 413794304
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007067442326980943,
+ "loss": 1.6172,
+ "theoretical_loss": 4.025198247092514,
+ "tokens_seen": 413859840
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706728184553661,
+ "loss": 1.5152,
+ "theoretical_loss": 4.025127851341196,
+ "tokens_seen": 413925376
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007067121364092277,
+ "loss": 1.4704,
+ "theoretical_loss": 4.025057469854831,
+ "tokens_seen": 413990912
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066960882647944,
+ "loss": 1.454,
+ "theoretical_loss": 4.024987102628267,
+ "tokens_seen": 414056448
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066800401203611,
+ "loss": 1.482,
+ "theoretical_loss": 4.024916749656362,
+ "tokens_seen": 414121984
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066639919759278,
+ "loss": 1.4719,
+ "theoretical_loss": 4.024846410933971,
+ "tokens_seen": 414187520
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066479438314945,
+ "loss": 1.523,
+ "theoretical_loss": 4.024776086455955,
+ "tokens_seen": 414253056
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066318956870612,
+ "loss": 1.4781,
+ "theoretical_loss": 4.024705776217177,
+ "tokens_seen": 414318592
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007066158475426278,
+ "loss": 1.5754,
+ "theoretical_loss": 4.024635480212501,
+ "tokens_seen": 414384128
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007065997993981946,
+ "loss": 1.5037,
+ "theoretical_loss": 4.024565198436796,
+ "tokens_seen": 414449664
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007065837512537612,
+ "loss": 1.5883,
+ "theoretical_loss": 4.024494930884931,
+ "tokens_seen": 414515200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706567703109328,
+ "loss": 1.5003,
+ "theoretical_loss": 4.024424677551781,
+ "tokens_seen": 414580736
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007065516549648946,
+ "loss": 1.4816,
+ "theoretical_loss": 4.02435443843222,
+ "tokens_seen": 414646272
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007065356068204614,
+ "loss": 1.6653,
+ "theoretical_loss": 4.024284213521128,
+ "tokens_seen": 414711808
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706519558676028,
+ "loss": 1.5012,
+ "theoretical_loss": 4.024214002813386,
+ "tokens_seen": 414777344
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007065035105315948,
+ "loss": 1.4605,
+ "theoretical_loss": 4.024143806303877,
+ "tokens_seen": 414842880
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007064874623871614,
+ "loss": 1.4218,
+ "theoretical_loss": 4.024073623987487,
+ "tokens_seen": 414908416
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007064714142427282,
+ "loss": 1.5412,
+ "theoretical_loss": 4.024003455859107,
+ "tokens_seen": 414973952
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007064553660982948,
+ "loss": 1.4357,
+ "theoretical_loss": 4.023933301913628,
+ "tokens_seen": 415039488
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007064393179538616,
+ "loss": 1.5242,
+ "theoretical_loss": 4.023863162145943,
+ "tokens_seen": 415105024
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007064232698094282,
+ "loss": 1.6922,
+ "theoretical_loss": 4.023793036550952,
+ "tokens_seen": 415170560
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706407221664995,
+ "loss": 1.4891,
+ "theoretical_loss": 4.023722925123551,
+ "tokens_seen": 415236096
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007063911735205616,
+ "loss": 1.5164,
+ "theoretical_loss": 4.023652827858646,
+ "tokens_seen": 415301632
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007063751253761284,
+ "loss": 1.537,
+ "theoretical_loss": 4.023582744751138,
+ "tokens_seen": 415367168
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706359077231695,
+ "loss": 1.437,
+ "theoretical_loss": 4.0235126757959385,
+ "tokens_seen": 415432704
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007063430290872618,
+ "loss": 1.467,
+ "theoretical_loss": 4.0234426209879555,
+ "tokens_seen": 415498240
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007063269809428284,
+ "loss": 1.5664,
+ "theoretical_loss": 4.0233725803221025,
+ "tokens_seen": 415563776
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007063109327983952,
+ "loss": 1.5127,
+ "theoretical_loss": 4.023302553793295,
+ "tokens_seen": 415629312
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706294884653962,
+ "loss": 1.4854,
+ "theoretical_loss": 4.023232541396451,
+ "tokens_seen": 415694848
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007062788365095286,
+ "loss": 1.5801,
+ "theoretical_loss": 4.023162543126491,
+ "tokens_seen": 415760384
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007062627883650954,
+ "loss": 1.5972,
+ "theoretical_loss": 4.023092558978339,
+ "tokens_seen": 415825920
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706246740220662,
+ "loss": 1.5104,
+ "theoretical_loss": 4.023022588946921,
+ "tokens_seen": 415891456
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007062306920762288,
+ "loss": 1.3768,
+ "theoretical_loss": 4.022952633027164,
+ "tokens_seen": 415956992
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007062146439317954,
+ "loss": 1.5053,
+ "theoretical_loss": 4.022882691214002,
+ "tokens_seen": 416022528
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061985957873622,
+ "loss": 1.6381,
+ "theoretical_loss": 4.022812763502367,
+ "tokens_seen": 416088064
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 144846,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4289838075637817,
+ "objective/train/theoretical_loss": 4.022742849887195,
+ "objective/train/tokens_used": 436613600,
+ "theoretical_loss": 4.022742849887195,
+ "tokens_seen": 416153600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061825476429288,
+ "loss": 1.5006,
+ "theoretical_loss": 4.022742849887195,
+ "tokens_seen": 416153600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061664994984956,
+ "loss": 1.5972,
+ "theoretical_loss": 4.022672950363427,
+ "tokens_seen": 416219136
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061504513540622,
+ "loss": 1.5633,
+ "theoretical_loss": 4.022603064926004,
+ "tokens_seen": 416284672
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706134403209629,
+ "loss": 1.578,
+ "theoretical_loss": 4.02253319356987,
+ "tokens_seen": 416350208
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061183550651956,
+ "loss": 1.475,
+ "theoretical_loss": 4.022463336289971,
+ "tokens_seen": 416415744
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007061023069207624,
+ "loss": 1.4692,
+ "theoretical_loss": 4.022393493081259,
+ "tokens_seen": 416481280
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000706086258776329,
+ "loss": 1.4357,
+ "theoretical_loss": 4.022323663938684,
+ "tokens_seen": 416546816
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007060702106318958,
+ "loss": 1.4955,
+ "theoretical_loss": 4.022253848857201,
+ "tokens_seen": 416612352
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007060541624874624,
+ "loss": 1.544,
+ "theoretical_loss": 4.022184047831768,
+ "tokens_seen": 416677888
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007060381143430292,
+ "loss": 1.5028,
+ "theoretical_loss": 4.022114260857345,
+ "tokens_seen": 416743424
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007060220661985958,
+ "loss": 1.523,
+ "theoretical_loss": 4.022044487928895,
+ "tokens_seen": 416808960
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007060060180541626,
+ "loss": 1.5272,
+ "theoretical_loss": 4.021974729041381,
+ "tokens_seen": 416874496
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007059899699097292,
+ "loss": 1.5397,
+ "theoretical_loss": 4.021904984189773,
+ "tokens_seen": 416940032
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705973921765296,
+ "loss": 1.5114,
+ "theoretical_loss": 4.021835253369041,
+ "tokens_seen": 417005568
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007059578736208626,
+ "loss": 1.6011,
+ "theoretical_loss": 4.021765536574157,
+ "tokens_seen": 417071104
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007059418254764294,
+ "loss": 1.5084,
+ "theoretical_loss": 4.021695833800098,
+ "tokens_seen": 417136640
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705925777331996,
+ "loss": 1.5491,
+ "theoretical_loss": 4.02162614504184,
+ "tokens_seen": 417202176
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007059097291875628,
+ "loss": 1.4945,
+ "theoretical_loss": 4.021556470294366,
+ "tokens_seen": 417267712
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007058936810431294,
+ "loss": 1.5515,
+ "theoretical_loss": 4.021486809552657,
+ "tokens_seen": 417333248
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007058776328986962,
+ "loss": 1.4881,
+ "theoretical_loss": 4.021417162811701,
+ "tokens_seen": 417398784
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007058615847542628,
+ "loss": 1.6362,
+ "theoretical_loss": 4.021347530066485,
+ "tokens_seen": 417464320
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007058455366098296,
+ "loss": 1.5221,
+ "theoretical_loss": 4.0212779113120005,
+ "tokens_seen": 417529856
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007058294884653962,
+ "loss": 1.5595,
+ "theoretical_loss": 4.021208306543242,
+ "tokens_seen": 417595392
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705813440320963,
+ "loss": 1.5748,
+ "theoretical_loss": 4.021138715755204,
+ "tokens_seen": 417660928
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057973921765296,
+ "loss": 1.5799,
+ "theoretical_loss": 4.021069138942886,
+ "tokens_seen": 417726464
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057813440320964,
+ "loss": 1.5296,
+ "theoretical_loss": 4.020999576101291,
+ "tokens_seen": 417792000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705765295887663,
+ "loss": 1.4946,
+ "theoretical_loss": 4.02093002722542,
+ "tokens_seen": 417857536
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057492477432298,
+ "loss": 1.583,
+ "theoretical_loss": 4.020860492310281,
+ "tokens_seen": 417923072
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057331995987964,
+ "loss": 1.5426,
+ "theoretical_loss": 4.020790971350884,
+ "tokens_seen": 417988608
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057171514543632,
+ "loss": 1.5689,
+ "theoretical_loss": 4.020721464342238,
+ "tokens_seen": 418054144
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007057011033099298,
+ "loss": 1.4474,
+ "theoretical_loss": 4.02065197127936,
+ "tokens_seen": 418119680
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007056850551654966,
+ "loss": 1.5951,
+ "theoretical_loss": 4.020582492157265,
+ "tokens_seen": 418185216
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007056690070210632,
+ "loss": 1.5839,
+ "theoretical_loss": 4.020513026970973,
+ "tokens_seen": 418250752
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.00070565295887663,
+ "loss": 1.5332,
+ "theoretical_loss": 4.020443575715506,
+ "tokens_seen": 418316288
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007056369107321966,
+ "loss": 1.5696,
+ "theoretical_loss": 4.020374138385887,
+ "tokens_seen": 418381824
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007056208625877634,
+ "loss": 1.6332,
+ "theoretical_loss": 4.020304714977145,
+ "tokens_seen": 418447360
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.00070560481444333,
+ "loss": 1.5723,
+ "theoretical_loss": 4.020235305484308,
+ "tokens_seen": 418512896
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055887662988968,
+ "loss": 1.5005,
+ "theoretical_loss": 4.020165909902409,
+ "tokens_seen": 418578432
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055727181544634,
+ "loss": 1.4726,
+ "theoretical_loss": 4.020096528226483,
+ "tokens_seen": 418643968
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055566700100302,
+ "loss": 1.59,
+ "theoretical_loss": 4.020027160451566,
+ "tokens_seen": 418709504
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055406218655968,
+ "loss": 1.4871,
+ "theoretical_loss": 4.019957806572697,
+ "tokens_seen": 418775040
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055245737211636,
+ "loss": 1.54,
+ "theoretical_loss": 4.019888466584921,
+ "tokens_seen": 418840576
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007055085255767302,
+ "loss": 1.4329,
+ "theoretical_loss": 4.019819140483281,
+ "tokens_seen": 418906112
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705492477432297,
+ "loss": 1.613,
+ "theoretical_loss": 4.019749828262825,
+ "tokens_seen": 418971648
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007054764292878636,
+ "loss": 1.501,
+ "theoretical_loss": 4.0196805299186025,
+ "tokens_seen": 419037184
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007054603811434304,
+ "loss": 1.3925,
+ "theoretical_loss": 4.019611245445666,
+ "tokens_seen": 419102720
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705444332998997,
+ "loss": 1.4554,
+ "theoretical_loss": 4.01954197483907,
+ "tokens_seen": 419168256
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007054282848545638,
+ "loss": 1.4464,
+ "theoretical_loss": 4.019472718093874,
+ "tokens_seen": 419233792
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007054122367101304,
+ "loss": 1.5303,
+ "theoretical_loss": 4.019403475205135,
+ "tokens_seen": 419299328
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053961885656972,
+ "loss": 1.5512,
+ "theoretical_loss": 4.019334246167919,
+ "tokens_seen": 419364864
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 145965,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.599278211593628,
+ "objective/train/theoretical_loss": 4.019265030977288,
+ "objective/train/tokens_used": 439890400,
+ "theoretical_loss": 4.019265030977288,
+ "tokens_seen": 419430400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053801404212638,
+ "loss": 1.615,
+ "theoretical_loss": 4.019265030977288,
+ "tokens_seen": 419430400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053640922768305,
+ "loss": 1.4472,
+ "theoretical_loss": 4.019195829628311,
+ "tokens_seen": 419495936
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053480441323972,
+ "loss": 1.5333,
+ "theoretical_loss": 4.019126642116059,
+ "tokens_seen": 419561472
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053319959879639,
+ "loss": 1.4841,
+ "theoretical_loss": 4.019057468435603,
+ "tokens_seen": 419627008
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007053159478435306,
+ "loss": 1.4736,
+ "theoretical_loss": 4.01898830858202,
+ "tokens_seen": 419692544
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052998996990973,
+ "loss": 1.5241,
+ "theoretical_loss": 4.018919162550386,
+ "tokens_seen": 419758080
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000705283851554664,
+ "loss": 1.5164,
+ "theoretical_loss": 4.018850030335783,
+ "tokens_seen": 419823616
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052678034102307,
+ "loss": 1.4566,
+ "theoretical_loss": 4.018780911933293,
+ "tokens_seen": 419889152
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052517552657974,
+ "loss": 1.629,
+ "theoretical_loss": 4.018711807338001,
+ "tokens_seen": 419954688
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052357071213641,
+ "loss": 1.5485,
+ "theoretical_loss": 4.018642716544994,
+ "tokens_seen": 420020224
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052196589769307,
+ "loss": 1.5464,
+ "theoretical_loss": 4.018573639549364,
+ "tokens_seen": 420085760
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007052036108324975,
+ "loss": 1.5059,
+ "theoretical_loss": 4.018504576346203,
+ "tokens_seen": 420151296
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051875626880641,
+ "loss": 1.5752,
+ "theoretical_loss": 4.018435526930607,
+ "tokens_seen": 420216832
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051715145436309,
+ "loss": 1.6088,
+ "theoretical_loss": 4.018366491297673,
+ "tokens_seen": 420282368
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051554663991975,
+ "loss": 1.4821,
+ "theoretical_loss": 4.018297469442502,
+ "tokens_seen": 420347904
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051394182547643,
+ "loss": 1.4243,
+ "theoretical_loss": 4.018228461360197,
+ "tokens_seen": 420413440
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051233701103309,
+ "loss": 1.538,
+ "theoretical_loss": 4.018159467045862,
+ "tokens_seen": 420478976
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007051073219658977,
+ "loss": 1.4714,
+ "theoretical_loss": 4.018090486494607,
+ "tokens_seen": 420544512
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050912738214644,
+ "loss": 1.4927,
+ "theoretical_loss": 4.0180215197015405,
+ "tokens_seen": 420610048
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050752256770311,
+ "loss": 1.4405,
+ "theoretical_loss": 4.017952566661776,
+ "tokens_seen": 420675584
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050591775325978,
+ "loss": 1.6563,
+ "theoretical_loss": 4.017883627370431,
+ "tokens_seen": 420741120
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050431293881645,
+ "loss": 1.5569,
+ "theoretical_loss": 4.017814701822619,
+ "tokens_seen": 420806656
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050270812437312,
+ "loss": 1.5939,
+ "theoretical_loss": 4.017745790013464,
+ "tokens_seen": 420872192
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007050110330992979,
+ "loss": 1.6454,
+ "theoretical_loss": 4.017676891938088,
+ "tokens_seen": 420937728
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007049949849548647,
+ "loss": 1.5453,
+ "theoretical_loss": 4.0176080075916145,
+ "tokens_seen": 421003264
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007049789368104313,
+ "loss": 1.5243,
+ "theoretical_loss": 4.017539136969174,
+ "tokens_seen": 421068800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000704962888665998,
+ "loss": 1.6324,
+ "theoretical_loss": 4.017470280065895,
+ "tokens_seen": 421134336
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007049468405215647,
+ "loss": 1.4929,
+ "theoretical_loss": 4.017401436876911,
+ "tokens_seen": 421199872
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007049307923771315,
+ "loss": 1.4403,
+ "theoretical_loss": 4.017332607397358,
+ "tokens_seen": 421265408
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007049147442326981,
+ "loss": 1.4566,
+ "theoretical_loss": 4.017263791622373,
+ "tokens_seen": 421330944
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048986960882649,
+ "loss": 1.5641,
+ "theoretical_loss": 4.017194989547095,
+ "tokens_seen": 421396480
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048826479438315,
+ "loss": 1.4466,
+ "theoretical_loss": 4.017126201166668,
+ "tokens_seen": 421462016
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048665997993983,
+ "loss": 1.6451,
+ "theoretical_loss": 4.017057426476239,
+ "tokens_seen": 421527552
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048505516549649,
+ "loss": 1.5085,
+ "theoretical_loss": 4.016988665470953,
+ "tokens_seen": 421593088
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048345035105317,
+ "loss": 1.528,
+ "theoretical_loss": 4.016919918145961,
+ "tokens_seen": 421658624
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048184553660983,
+ "loss": 1.4832,
+ "theoretical_loss": 4.016851184496415,
+ "tokens_seen": 421724160
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007048024072216651,
+ "loss": 1.338,
+ "theoretical_loss": 4.016782464517472,
+ "tokens_seen": 421789696
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047863590772317,
+ "loss": 1.4554,
+ "theoretical_loss": 4.016713758204288,
+ "tokens_seen": 421855232
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047703109327985,
+ "loss": 1.5311,
+ "theoretical_loss": 4.016645065552023,
+ "tokens_seen": 421920768
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047542627883651,
+ "loss": 1.4613,
+ "theoretical_loss": 4.0165763865558395,
+ "tokens_seen": 421986304
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047382146439319,
+ "loss": 1.3784,
+ "theoretical_loss": 4.016507721210904,
+ "tokens_seen": 422051840
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047221664994985,
+ "loss": 1.5047,
+ "theoretical_loss": 4.016439069512382,
+ "tokens_seen": 422117376
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007047061183550653,
+ "loss": 1.5729,
+ "theoretical_loss": 4.016370431455444,
+ "tokens_seen": 422182912
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046900702106319,
+ "loss": 1.5015,
+ "theoretical_loss": 4.016301807035264,
+ "tokens_seen": 422248448
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046740220661987,
+ "loss": 1.4857,
+ "theoretical_loss": 4.016233196247014,
+ "tokens_seen": 422313984
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046579739217653,
+ "loss": 1.4861,
+ "theoretical_loss": 4.016164599085872,
+ "tokens_seen": 422379520
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046419257773321,
+ "loss": 1.3585,
+ "theoretical_loss": 4.016096015547021,
+ "tokens_seen": 422445056
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046258776328987,
+ "loss": 1.7261,
+ "theoretical_loss": 4.016027445625638,
+ "tokens_seen": 422510592
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007046098294884655,
+ "loss": 1.4888,
+ "theoretical_loss": 4.015958889316911,
+ "tokens_seen": 422576128
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045937813440321,
+ "loss": 1.505,
+ "theoretical_loss": 4.015890346616026,
+ "tokens_seen": 422641664
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 147195,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.464314579963684,
+ "objective/train/theoretical_loss": 4.015821817518173,
+ "objective/train/tokens_used": 443167200,
+ "theoretical_loss": 4.015821817518173,
+ "tokens_seen": 422707200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045777331995989,
+ "loss": 1.4434,
+ "theoretical_loss": 4.015821817518173,
+ "tokens_seen": 422707200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045616850551655,
+ "loss": 1.5134,
+ "theoretical_loss": 4.015753302018544,
+ "tokens_seen": 422772736
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045456369107323,
+ "loss": 1.3918,
+ "theoretical_loss": 4.015684800112332,
+ "tokens_seen": 422838272
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045295887662989,
+ "loss": 1.4096,
+ "theoretical_loss": 4.015616311794735,
+ "tokens_seen": 422903808
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007045135406218657,
+ "loss": 1.5184,
+ "theoretical_loss": 4.015547837060953,
+ "tokens_seen": 422969344
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044974924774323,
+ "loss": 1.4947,
+ "theoretical_loss": 4.015479375906187,
+ "tokens_seen": 423034880
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044814443329991,
+ "loss": 1.5867,
+ "theoretical_loss": 4.0154109283256405,
+ "tokens_seen": 423100416
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044653961885657,
+ "loss": 1.5139,
+ "theoretical_loss": 4.015342494314521,
+ "tokens_seen": 423165952
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044493480441325,
+ "loss": 1.5439,
+ "theoretical_loss": 4.015274073868037,
+ "tokens_seen": 423231488
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044332998996991,
+ "loss": 1.5311,
+ "theoretical_loss": 4.015205666981401,
+ "tokens_seen": 423297024
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044172517552659,
+ "loss": 1.5146,
+ "theoretical_loss": 4.015137273649826,
+ "tokens_seen": 423362560
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007044012036108325,
+ "loss": 1.6298,
+ "theoretical_loss": 4.015068893868528,
+ "tokens_seen": 423428096
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043851554663993,
+ "loss": 1.3893,
+ "theoretical_loss": 4.015000527632726,
+ "tokens_seen": 423493632
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043691073219659,
+ "loss": 1.452,
+ "theoretical_loss": 4.014932174937642,
+ "tokens_seen": 423559168
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043530591775327,
+ "loss": 1.4128,
+ "theoretical_loss": 4.014863835778499,
+ "tokens_seen": 423624704
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043370110330993,
+ "loss": 1.5542,
+ "theoretical_loss": 4.014795510150522,
+ "tokens_seen": 423690240
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043209628886661,
+ "loss": 1.6039,
+ "theoretical_loss": 4.014727198048942,
+ "tokens_seen": 423755776
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007043049147442327,
+ "loss": 1.5696,
+ "theoretical_loss": 4.014658899468988,
+ "tokens_seen": 423821312
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042888665997995,
+ "loss": 1.566,
+ "theoretical_loss": 4.014590614405893,
+ "tokens_seen": 423886848
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042728184553661,
+ "loss": 1.5941,
+ "theoretical_loss": 4.014522342854895,
+ "tokens_seen": 423952384
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042567703109329,
+ "loss": 1.4703,
+ "theoretical_loss": 4.01445408481123,
+ "tokens_seen": 424017920
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042407221664995,
+ "loss": 1.5825,
+ "theoretical_loss": 4.014385840270139,
+ "tokens_seen": 424083456
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042246740220663,
+ "loss": 1.5623,
+ "theoretical_loss": 4.014317609226865,
+ "tokens_seen": 424148992
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007042086258776329,
+ "loss": 1.466,
+ "theoretical_loss": 4.014249391676652,
+ "tokens_seen": 424214528
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041925777331997,
+ "loss": 1.4422,
+ "theoretical_loss": 4.014181187614751,
+ "tokens_seen": 424280064
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041765295887663,
+ "loss": 1.4518,
+ "theoretical_loss": 4.01411299703641,
+ "tokens_seen": 424345600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041604814443331,
+ "loss": 1.5206,
+ "theoretical_loss": 4.014044819936882,
+ "tokens_seen": 424411136
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041444332998997,
+ "loss": 1.4531,
+ "theoretical_loss": 4.013976656311422,
+ "tokens_seen": 424476672
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041283851554665,
+ "loss": 1.5651,
+ "theoretical_loss": 4.013908506155287,
+ "tokens_seen": 424542208
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007041123370110331,
+ "loss": 1.5244,
+ "theoretical_loss": 4.013840369463737,
+ "tokens_seen": 424607744
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040962888665999,
+ "loss": 1.5536,
+ "theoretical_loss": 4.013772246232035,
+ "tokens_seen": 424673280
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040802407221665,
+ "loss": 1.4733,
+ "theoretical_loss": 4.013704136455445,
+ "tokens_seen": 424738816
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040641925777333,
+ "loss": 1.4762,
+ "theoretical_loss": 4.013636040129233,
+ "tokens_seen": 424804352
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040481444332999,
+ "loss": 1.4481,
+ "theoretical_loss": 4.0135679572486715,
+ "tokens_seen": 424869888
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040320962888667,
+ "loss": 1.5199,
+ "theoretical_loss": 4.013499887809028,
+ "tokens_seen": 424935424
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040160481444333,
+ "loss": 1.5524,
+ "theoretical_loss": 4.01343183180558,
+ "tokens_seen": 425000960
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007040000000000001,
+ "loss": 1.5354,
+ "theoretical_loss": 4.013363789233602,
+ "tokens_seen": 425066496
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039839518555667,
+ "loss": 1.4442,
+ "theoretical_loss": 4.013295760088374,
+ "tokens_seen": 425132032
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039679037111334,
+ "loss": 1.4176,
+ "theoretical_loss": 4.013227744365178,
+ "tokens_seen": 425197568
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039518555667001,
+ "loss": 1.4971,
+ "theoretical_loss": 4.013159742059296,
+ "tokens_seen": 425263104
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039358074222668,
+ "loss": 1.4715,
+ "theoretical_loss": 4.013091753166015,
+ "tokens_seen": 425328640
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039197592778335,
+ "loss": 1.5929,
+ "theoretical_loss": 4.013023777680624,
+ "tokens_seen": 425394176
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007039037111334002,
+ "loss": 1.5164,
+ "theoretical_loss": 4.0129558155984135,
+ "tokens_seen": 425459712
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007038876629889669,
+ "loss": 1.395,
+ "theoretical_loss": 4.012887866914675,
+ "tokens_seen": 425525248
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007038716148445336,
+ "loss": 1.4983,
+ "theoretical_loss": 4.0128199316247075,
+ "tokens_seen": 425590784
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007038555667001003,
+ "loss": 1.5055,
+ "theoretical_loss": 4.0127520097238065,
+ "tokens_seen": 425656320
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703839518555667,
+ "loss": 1.5522,
+ "theoretical_loss": 4.012684101207273,
+ "tokens_seen": 425721856
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007038234704112337,
+ "loss": 1.3876,
+ "theoretical_loss": 4.01261620607041,
+ "tokens_seen": 425787392
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007038074222668004,
+ "loss": 1.5193,
+ "theoretical_loss": 4.012548324308522,
+ "tokens_seen": 425852928
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007037913741223671,
+ "loss": 1.5764,
+ "theoretical_loss": 4.012480455916918,
+ "tokens_seen": 425918464
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 148172,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.435992956161499,
+ "objective/train/theoretical_loss": 4.012412600890906,
+ "objective/train/tokens_used": 446444000,
+ "theoretical_loss": 4.012412600890906,
+ "tokens_seen": 425984000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007037753259779338,
+ "loss": 1.5679,
+ "theoretical_loss": 4.012412600890906,
+ "tokens_seen": 425984000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007037592778335005,
+ "loss": 1.5018,
+ "theoretical_loss": 4.012344759225799,
+ "tokens_seen": 426049536
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007037432296890672,
+ "loss": 1.5631,
+ "theoretical_loss": 4.0122769309169115,
+ "tokens_seen": 426115072
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703727181544634,
+ "loss": 1.5465,
+ "theoretical_loss": 4.0122091159595605,
+ "tokens_seen": 426180608
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007037111334002006,
+ "loss": 1.4451,
+ "theoretical_loss": 4.0121413143490665,
+ "tokens_seen": 426246144
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007036950852557673,
+ "loss": 1.479,
+ "theoretical_loss": 4.012073526080749,
+ "tokens_seen": 426311680
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703679037111334,
+ "loss": 1.5421,
+ "theoretical_loss": 4.012005751149934,
+ "tokens_seen": 426377216
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007036629889669007,
+ "loss": 1.416,
+ "theoretical_loss": 4.011937989551946,
+ "tokens_seen": 426442752
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007036469408224674,
+ "loss": 1.5782,
+ "theoretical_loss": 4.011870241282114,
+ "tokens_seen": 426508288
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007036308926780342,
+ "loss": 1.4749,
+ "theoretical_loss": 4.011802506335771,
+ "tokens_seen": 426573824
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007036148445336008,
+ "loss": 1.515,
+ "theoretical_loss": 4.011734784708249,
+ "tokens_seen": 426639360
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007035987963891676,
+ "loss": 1.5876,
+ "theoretical_loss": 4.011667076394884,
+ "tokens_seen": 426704896
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007035827482447342,
+ "loss": 1.4823,
+ "theoretical_loss": 4.011599381391014,
+ "tokens_seen": 426770432
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703566700100301,
+ "loss": 1.5256,
+ "theoretical_loss": 4.0115316996919805,
+ "tokens_seen": 426835968
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007035506519558676,
+ "loss": 1.4832,
+ "theoretical_loss": 4.011464031293125,
+ "tokens_seen": 426901504
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007035346038114344,
+ "loss": 1.4891,
+ "theoretical_loss": 4.0113963761897935,
+ "tokens_seen": 426967040
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703518555667001,
+ "loss": 1.4984,
+ "theoretical_loss": 4.011328734377333,
+ "tokens_seen": 427032576
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007035025075225678,
+ "loss": 1.4975,
+ "theoretical_loss": 4.011261105851093,
+ "tokens_seen": 427098112
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007034864593781344,
+ "loss": 1.51,
+ "theoretical_loss": 4.011193490606427,
+ "tokens_seen": 427163648
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007034704112337012,
+ "loss": 1.5087,
+ "theoretical_loss": 4.01112588863869,
+ "tokens_seen": 427229184
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007034543630892678,
+ "loss": 1.4949,
+ "theoretical_loss": 4.011058299943237,
+ "tokens_seen": 427294720
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007034383149448346,
+ "loss": 1.501,
+ "theoretical_loss": 4.010990724515427,
+ "tokens_seen": 427360256
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007034222668004012,
+ "loss": 1.4712,
+ "theoretical_loss": 4.010923162350624,
+ "tokens_seen": 427425792
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703406218655968,
+ "loss": 1.5337,
+ "theoretical_loss": 4.01085561344419,
+ "tokens_seen": 427491328
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007033901705115346,
+ "loss": 1.5073,
+ "theoretical_loss": 4.010788077791491,
+ "tokens_seen": 427556864
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007033741223671014,
+ "loss": 1.4828,
+ "theoretical_loss": 4.0107205553878975,
+ "tokens_seen": 427622400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703358074222668,
+ "loss": 1.5587,
+ "theoretical_loss": 4.010653046228779,
+ "tokens_seen": 427687936
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007033420260782348,
+ "loss": 1.4948,
+ "theoretical_loss": 4.010585550309508,
+ "tokens_seen": 427753472
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007033259779338014,
+ "loss": 1.4606,
+ "theoretical_loss": 4.010518067625461,
+ "tokens_seen": 427819008
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007033099297893682,
+ "loss": 1.5048,
+ "theoretical_loss": 4.010450598172016,
+ "tokens_seen": 427884544
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007032938816449348,
+ "loss": 1.6048,
+ "theoretical_loss": 4.010383141944553,
+ "tokens_seen": 427950080
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007032778335005016,
+ "loss": 1.567,
+ "theoretical_loss": 4.010315698938455,
+ "tokens_seen": 428015616
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007032617853560682,
+ "loss": 1.4507,
+ "theoretical_loss": 4.0102482691491055,
+ "tokens_seen": 428081152
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703245737211635,
+ "loss": 1.5643,
+ "theoretical_loss": 4.010180852571893,
+ "tokens_seen": 428146688
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007032296890672016,
+ "loss": 1.5148,
+ "theoretical_loss": 4.010113449202206,
+ "tokens_seen": 428212224
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007032136409227684,
+ "loss": 1.4841,
+ "theoretical_loss": 4.010046059035437,
+ "tokens_seen": 428277760
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703197592778335,
+ "loss": 1.438,
+ "theoretical_loss": 4.009978682066979,
+ "tokens_seen": 428343296
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031815446339018,
+ "loss": 1.5299,
+ "theoretical_loss": 4.00991131829223,
+ "tokens_seen": 428408832
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031654964894684,
+ "loss": 1.5179,
+ "theoretical_loss": 4.009843967706586,
+ "tokens_seen": 428474368
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031494483450352,
+ "loss": 1.4384,
+ "theoretical_loss": 4.009776630305451,
+ "tokens_seen": 428539904
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031334002006018,
+ "loss": 1.4572,
+ "theoretical_loss": 4.009709306084227,
+ "tokens_seen": 428605440
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031173520561686,
+ "loss": 1.5282,
+ "theoretical_loss": 4.00964199503832,
+ "tokens_seen": 428670976
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007031013039117352,
+ "loss": 1.476,
+ "theoretical_loss": 4.009574697163137,
+ "tokens_seen": 428736512
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703085255767302,
+ "loss": 1.6019,
+ "theoretical_loss": 4.00950741245409,
+ "tokens_seen": 428802048
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007030692076228686,
+ "loss": 1.4104,
+ "theoretical_loss": 4.009440140906589,
+ "tokens_seen": 428867584
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007030531594784354,
+ "loss": 1.6,
+ "theoretical_loss": 4.00937288251605,
+ "tokens_seen": 428933120
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000703037111334002,
+ "loss": 1.5577,
+ "theoretical_loss": 4.009305637277892,
+ "tokens_seen": 428998656
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007030210631895688,
+ "loss": 1.5977,
+ "theoretical_loss": 4.009238405187531,
+ "tokens_seen": 429064192
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007030050150451354,
+ "loss": 1.57,
+ "theoretical_loss": 4.009171186240391,
+ "tokens_seen": 429129728
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007029889669007022,
+ "loss": 1.478,
+ "theoretical_loss": 4.009103980431896,
+ "tokens_seen": 429195264
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 149293,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.394667625427246,
+ "objective/train/theoretical_loss": 4.009036787757472,
+ "objective/train/tokens_used": 449720800,
+ "theoretical_loss": 4.009036787757472,
+ "tokens_seen": 429260800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007029729187562688,
+ "loss": 1.4451,
+ "theoretical_loss": 4.009036787757472,
+ "tokens_seen": 429260800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007029568706118356,
+ "loss": 1.6264,
+ "theoretical_loss": 4.0089696082125466,
+ "tokens_seen": 429326336
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007029408224674022,
+ "loss": 1.6049,
+ "theoretical_loss": 4.008902441792552,
+ "tokens_seen": 429391872
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702924774322969,
+ "loss": 1.6117,
+ "theoretical_loss": 4.00883528849292,
+ "tokens_seen": 429457408
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007029087261785356,
+ "loss": 1.6004,
+ "theoretical_loss": 4.008768148309087,
+ "tokens_seen": 429522944
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007028926780341024,
+ "loss": 1.5711,
+ "theoretical_loss": 4.00870102123649,
+ "tokens_seen": 429588480
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702876629889669,
+ "loss": 1.4821,
+ "theoretical_loss": 4.008633907270571,
+ "tokens_seen": 429654016
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007028605817452358,
+ "loss": 1.5774,
+ "theoretical_loss": 4.008566806406771,
+ "tokens_seen": 429719552
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007028445336008024,
+ "loss": 1.3801,
+ "theoretical_loss": 4.0084997186405324,
+ "tokens_seen": 429785088
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007028284854563692,
+ "loss": 1.5135,
+ "theoretical_loss": 4.008432643967305,
+ "tokens_seen": 429850624
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007028124373119358,
+ "loss": 1.4067,
+ "theoretical_loss": 4.008365582382538,
+ "tokens_seen": 429916160
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007027963891675026,
+ "loss": 1.6318,
+ "theoretical_loss": 4.008298533881681,
+ "tokens_seen": 429981696
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007027803410230692,
+ "loss": 1.5392,
+ "theoretical_loss": 4.008231498460189,
+ "tokens_seen": 430047232
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702764292878636,
+ "loss": 1.4467,
+ "theoretical_loss": 4.008164476113517,
+ "tokens_seen": 430112768
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007027482447342026,
+ "loss": 1.5168,
+ "theoretical_loss": 4.008097466837124,
+ "tokens_seen": 430178304
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007027321965897694,
+ "loss": 1.5886,
+ "theoretical_loss": 4.00803047062647,
+ "tokens_seen": 430243840
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702716148445336,
+ "loss": 1.5773,
+ "theoretical_loss": 4.007963487477018,
+ "tokens_seen": 430309376
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007027001003009028,
+ "loss": 1.5376,
+ "theoretical_loss": 4.007896517384233,
+ "tokens_seen": 430374912
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007026840521564694,
+ "loss": 1.5965,
+ "theoretical_loss": 4.007829560343582,
+ "tokens_seen": 430440448
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007026680040120362,
+ "loss": 1.5795,
+ "theoretical_loss": 4.007762616350535,
+ "tokens_seen": 430505984
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007026519558676028,
+ "loss": 1.4647,
+ "theoretical_loss": 4.007695685400563,
+ "tokens_seen": 430571520
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007026359077231696,
+ "loss": 1.552,
+ "theoretical_loss": 4.007628767489142,
+ "tokens_seen": 430637056
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007026198595787362,
+ "loss": 1.4869,
+ "theoretical_loss": 4.007561862611746,
+ "tokens_seen": 430702592
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702603811434303,
+ "loss": 1.4319,
+ "theoretical_loss": 4.007494970763855,
+ "tokens_seen": 430768128
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007025877632898696,
+ "loss": 1.5502,
+ "theoretical_loss": 4.00742809194095,
+ "tokens_seen": 430833664
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007025717151454363,
+ "loss": 1.5765,
+ "theoretical_loss": 4.007361226138513,
+ "tokens_seen": 430899200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702555667001003,
+ "loss": 1.4739,
+ "theoretical_loss": 4.00729437335203,
+ "tokens_seen": 430964736
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007025396188565697,
+ "loss": 1.3935,
+ "theoretical_loss": 4.007227533576988,
+ "tokens_seen": 431030272
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007025235707121364,
+ "loss": 1.5863,
+ "theoretical_loss": 4.007160706808879,
+ "tokens_seen": 431095808
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007025075225677031,
+ "loss": 1.4851,
+ "theoretical_loss": 4.007093893043192,
+ "tokens_seen": 431161344
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024914744232698,
+ "loss": 1.5307,
+ "theoretical_loss": 4.007027092275424,
+ "tokens_seen": 431226880
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024754262788365,
+ "loss": 1.5869,
+ "theoretical_loss": 4.006960304501069,
+ "tokens_seen": 431292416
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024593781344032,
+ "loss": 1.5409,
+ "theoretical_loss": 4.0068935297156285,
+ "tokens_seen": 431357952
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024433299899699,
+ "loss": 1.572,
+ "theoretical_loss": 4.0068267679146015,
+ "tokens_seen": 431423488
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024272818455366,
+ "loss": 1.4711,
+ "theoretical_loss": 4.006760019093493,
+ "tokens_seen": 431489024
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007024112337011033,
+ "loss": 1.4946,
+ "theoretical_loss": 4.006693283247808,
+ "tokens_seen": 431554560
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.00070239518555667,
+ "loss": 1.4717,
+ "theoretical_loss": 4.0066265603730535,
+ "tokens_seen": 431620096
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007023791374122367,
+ "loss": 1.5803,
+ "theoretical_loss": 4.006559850464741,
+ "tokens_seen": 431685632
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007023630892678034,
+ "loss": 1.652,
+ "theoretical_loss": 4.006493153518381,
+ "tokens_seen": 431751168
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007023470411233701,
+ "loss": 1.5243,
+ "theoretical_loss": 4.006426469529488,
+ "tokens_seen": 431816704
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007023309929789368,
+ "loss": 1.4888,
+ "theoretical_loss": 4.006359798493582,
+ "tokens_seen": 431882240
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007023149448345035,
+ "loss": 1.5452,
+ "theoretical_loss": 4.006293140406179,
+ "tokens_seen": 431947776
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022988966900702,
+ "loss": 1.5458,
+ "theoretical_loss": 4.0062264952627995,
+ "tokens_seen": 432013312
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022828485456369,
+ "loss": 1.6067,
+ "theoretical_loss": 4.00615986305897,
+ "tokens_seen": 432078848
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022668004012037,
+ "loss": 1.5937,
+ "theoretical_loss": 4.006093243790214,
+ "tokens_seen": 432144384
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022507522567703,
+ "loss": 1.6446,
+ "theoretical_loss": 4.0060266374520594,
+ "tokens_seen": 432209920
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000702234704112337,
+ "loss": 1.5289,
+ "theoretical_loss": 4.005960044040037,
+ "tokens_seen": 432275456
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022186559679037,
+ "loss": 1.5373,
+ "theoretical_loss": 4.005893463549679,
+ "tokens_seen": 432340992
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007022026078234705,
+ "loss": 1.502,
+ "theoretical_loss": 4.0058268959765195,
+ "tokens_seen": 432406528
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021865596790371,
+ "loss": 1.4267,
+ "theoretical_loss": 4.005760341316096,
+ "tokens_seen": 432472064
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 150155,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2991299629211426,
+ "objective/train/theoretical_loss": 4.0056937995639466,
+ "objective/train/tokens_used": 452997600,
+ "theoretical_loss": 4.0056937995639466,
+ "tokens_seen": 432537600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021705115346039,
+ "loss": 1.4196,
+ "theoretical_loss": 4.0056937995639466,
+ "tokens_seen": 432537600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021544633901705,
+ "loss": 1.5659,
+ "theoretical_loss": 4.005627270715614,
+ "tokens_seen": 432603136
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021384152457373,
+ "loss": 1.5304,
+ "theoretical_loss": 4.00556075476664,
+ "tokens_seen": 432668672
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021223671013039,
+ "loss": 1.5103,
+ "theoretical_loss": 4.005494251712571,
+ "tokens_seen": 432734208
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007021063189568707,
+ "loss": 1.4877,
+ "theoretical_loss": 4.005427761548955,
+ "tokens_seen": 432799744
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020902708124373,
+ "loss": 1.5152,
+ "theoretical_loss": 4.005361284271341,
+ "tokens_seen": 432865280
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020742226680041,
+ "loss": 1.3662,
+ "theoretical_loss": 4.005294819875283,
+ "tokens_seen": 432930816
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020581745235707,
+ "loss": 1.5065,
+ "theoretical_loss": 4.005228368356335,
+ "tokens_seen": 432996352
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020421263791375,
+ "loss": 1.5918,
+ "theoretical_loss": 4.005161929710052,
+ "tokens_seen": 433061888
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020260782347041,
+ "loss": 1.5184,
+ "theoretical_loss": 4.0050955039319955,
+ "tokens_seen": 433127424
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007020100300902709,
+ "loss": 1.4619,
+ "theoretical_loss": 4.0050290910177235,
+ "tokens_seen": 433192960
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019939819458375,
+ "loss": 1.5408,
+ "theoretical_loss": 4.004962690962802,
+ "tokens_seen": 433258496
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019779338014043,
+ "loss": 1.4712,
+ "theoretical_loss": 4.004896303762797,
+ "tokens_seen": 433324032
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019618856569709,
+ "loss": 1.4964,
+ "theoretical_loss": 4.0048299294132725,
+ "tokens_seen": 433389568
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019458375125377,
+ "loss": 1.5561,
+ "theoretical_loss": 4.0047635679098015,
+ "tokens_seen": 433455104
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019297893681043,
+ "loss": 1.4142,
+ "theoretical_loss": 4.004697219247955,
+ "tokens_seen": 433520640
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007019137412236711,
+ "loss": 1.5248,
+ "theoretical_loss": 4.004630883423308,
+ "tokens_seen": 433586176
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018976930792377,
+ "loss": 1.5179,
+ "theoretical_loss": 4.004564560431437,
+ "tokens_seen": 433651712
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018816449348045,
+ "loss": 1.6083,
+ "theoretical_loss": 4.004498250267919,
+ "tokens_seen": 433717248
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018655967903711,
+ "loss": 1.4272,
+ "theoretical_loss": 4.004431952928337,
+ "tokens_seen": 433782784
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018495486459379,
+ "loss": 1.5259,
+ "theoretical_loss": 4.004365668408273,
+ "tokens_seen": 433848320
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018335005015045,
+ "loss": 1.4605,
+ "theoretical_loss": 4.004299396703312,
+ "tokens_seen": 433913856
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018174523570713,
+ "loss": 1.5143,
+ "theoretical_loss": 4.004233137809042,
+ "tokens_seen": 433979392
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007018014042126379,
+ "loss": 1.5833,
+ "theoretical_loss": 4.004166891721052,
+ "tokens_seen": 434044928
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017853560682047,
+ "loss": 1.4719,
+ "theoretical_loss": 4.004100658434934,
+ "tokens_seen": 434110464
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017693079237713,
+ "loss": 1.5535,
+ "theoretical_loss": 4.004034437946283,
+ "tokens_seen": 434176000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017532597793381,
+ "loss": 1.4578,
+ "theoretical_loss": 4.003968230250694,
+ "tokens_seen": 434241536
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017372116349047,
+ "loss": 1.5173,
+ "theoretical_loss": 4.003902035343766,
+ "tokens_seen": 434307072
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017211634904715,
+ "loss": 1.5396,
+ "theoretical_loss": 4.003835853221099,
+ "tokens_seen": 434372608
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007017051153460381,
+ "loss": 1.3807,
+ "theoretical_loss": 4.003769683878296,
+ "tokens_seen": 434438144
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016890672016049,
+ "loss": 1.4859,
+ "theoretical_loss": 4.003703527310962,
+ "tokens_seen": 434503680
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016730190571715,
+ "loss": 1.4641,
+ "theoretical_loss": 4.003637383514704,
+ "tokens_seen": 434569216
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016569709127383,
+ "loss": 1.5701,
+ "theoretical_loss": 4.00357125248513,
+ "tokens_seen": 434634752
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016409227683049,
+ "loss": 1.5586,
+ "theoretical_loss": 4.003505134217853,
+ "tokens_seen": 434700288
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016248746238717,
+ "loss": 1.5719,
+ "theoretical_loss": 4.0034390287084864,
+ "tokens_seen": 434765824
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007016088264794383,
+ "loss": 1.6704,
+ "theoretical_loss": 4.003372935952646,
+ "tokens_seen": 434831360
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015927783350051,
+ "loss": 1.5484,
+ "theoretical_loss": 4.003306855945948,
+ "tokens_seen": 434896896
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015767301905717,
+ "loss": 1.4919,
+ "theoretical_loss": 4.003240788684015,
+ "tokens_seen": 434962432
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015606820461385,
+ "loss": 1.5402,
+ "theoretical_loss": 4.003174734162467,
+ "tokens_seen": 435027968
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015446339017051,
+ "loss": 1.515,
+ "theoretical_loss": 4.00310869237693,
+ "tokens_seen": 435093504
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015285857572719,
+ "loss": 1.5989,
+ "theoretical_loss": 4.003042663323029,
+ "tokens_seen": 435159040
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007015125376128385,
+ "loss": 1.4633,
+ "theoretical_loss": 4.0029766469963945,
+ "tokens_seen": 435224576
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014964894684053,
+ "loss": 1.4019,
+ "theoretical_loss": 4.0029106433926565,
+ "tokens_seen": 435290112
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000701480441323972,
+ "loss": 1.5274,
+ "theoretical_loss": 4.002844652507449,
+ "tokens_seen": 435355648
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014643931795387,
+ "loss": 1.4794,
+ "theoretical_loss": 4.002778674336405,
+ "tokens_seen": 435421184
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014483450351054,
+ "loss": 1.5274,
+ "theoretical_loss": 4.002712708875162,
+ "tokens_seen": 435486720
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014322968906721,
+ "loss": 1.5422,
+ "theoretical_loss": 4.002646756119362,
+ "tokens_seen": 435552256
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014162487462388,
+ "loss": 1.5547,
+ "theoretical_loss": 4.002580816064645,
+ "tokens_seen": 435617792
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007014002006018055,
+ "loss": 1.5411,
+ "theoretical_loss": 4.002514888706655,
+ "tokens_seen": 435683328
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007013841524573722,
+ "loss": 1.374,
+ "theoretical_loss": 4.002448974041037,
+ "tokens_seen": 435748864
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 151381,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.41636323928833,
+ "objective/train/theoretical_loss": 4.0023830720634415,
+ "objective/train/tokens_used": 456274400,
+ "theoretical_loss": 4.0023830720634415,
+ "tokens_seen": 435814400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007013681043129389,
+ "loss": 1.3912,
+ "theoretical_loss": 4.0023830720634415,
+ "tokens_seen": 435814400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007013520561685056,
+ "loss": 1.4124,
+ "theoretical_loss": 4.0023171827695165,
+ "tokens_seen": 435879936
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007013360080240723,
+ "loss": 1.4398,
+ "theoretical_loss": 4.002251306154916,
+ "tokens_seen": 435945472
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000701319959879639,
+ "loss": 1.5039,
+ "theoretical_loss": 4.002185442215293,
+ "tokens_seen": 436011008
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007013039117352057,
+ "loss": 1.4421,
+ "theoretical_loss": 4.002119590946305,
+ "tokens_seen": 436076544
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012878635907723,
+ "loss": 1.5099,
+ "theoretical_loss": 4.002053752343611,
+ "tokens_seen": 436142080
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012718154463391,
+ "loss": 1.523,
+ "theoretical_loss": 4.001987926402871,
+ "tokens_seen": 436207616
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012557673019057,
+ "loss": 1.46,
+ "theoretical_loss": 4.001922113119749,
+ "tokens_seen": 436273152
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012397191574725,
+ "loss": 1.542,
+ "theoretical_loss": 4.00185631248991,
+ "tokens_seen": 436338688
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012236710130391,
+ "loss": 1.4672,
+ "theoretical_loss": 4.001790524509021,
+ "tokens_seen": 436404224
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007012076228686059,
+ "loss": 1.5367,
+ "theoretical_loss": 4.001724749172752,
+ "tokens_seen": 436469760
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007011915747241725,
+ "loss": 1.5853,
+ "theoretical_loss": 4.001658986476775,
+ "tokens_seen": 436535296
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007011755265797393,
+ "loss": 1.4906,
+ "theoretical_loss": 4.0015932364167615,
+ "tokens_seen": 436600832
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007011594784353059,
+ "loss": 1.5542,
+ "theoretical_loss": 4.00152749898839,
+ "tokens_seen": 436666368
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007011434302908726,
+ "loss": 1.3937,
+ "theoretical_loss": 4.001461774187338,
+ "tokens_seen": 436731904
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007011273821464393,
+ "loss": 1.6308,
+ "theoretical_loss": 4.001396062009285,
+ "tokens_seen": 436797440
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000701111334002006,
+ "loss": 1.5693,
+ "theoretical_loss": 4.001330362449914,
+ "tokens_seen": 436862976
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010952858575727,
+ "loss": 1.4551,
+ "theoretical_loss": 4.001264675504908,
+ "tokens_seen": 436928512
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010792377131394,
+ "loss": 1.5887,
+ "theoretical_loss": 4.001199001169955,
+ "tokens_seen": 436994048
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010631895687061,
+ "loss": 1.5951,
+ "theoretical_loss": 4.001133339440743,
+ "tokens_seen": 437059584
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010471414242728,
+ "loss": 1.448,
+ "theoretical_loss": 4.001067690312963,
+ "tokens_seen": 437125120
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010310932798395,
+ "loss": 1.4664,
+ "theoretical_loss": 4.001002053782307,
+ "tokens_seen": 437190656
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007010150451354062,
+ "loss": 1.4245,
+ "theoretical_loss": 4.0009364298444705,
+ "tokens_seen": 437256192
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700998996990973,
+ "loss": 1.464,
+ "theoretical_loss": 4.000870818495152,
+ "tokens_seen": 437321728
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007009829488465396,
+ "loss": 1.5072,
+ "theoretical_loss": 4.000805219730049,
+ "tokens_seen": 437387264
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007009669007021063,
+ "loss": 1.3711,
+ "theoretical_loss": 4.000739633544863,
+ "tokens_seen": 437452800
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700950852557673,
+ "loss": 1.5206,
+ "theoretical_loss": 4.000674059935298,
+ "tokens_seen": 437518336
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007009348044132397,
+ "loss": 1.5586,
+ "theoretical_loss": 4.000608498897059,
+ "tokens_seen": 437583872
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007009187562688064,
+ "loss": 1.6071,
+ "theoretical_loss": 4.000542950425855,
+ "tokens_seen": 437649408
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007009027081243731,
+ "loss": 1.4701,
+ "theoretical_loss": 4.000477414517394,
+ "tokens_seen": 437714944
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007008866599799398,
+ "loss": 1.4597,
+ "theoretical_loss": 4.000411891167389,
+ "tokens_seen": 437780480
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007008706118355066,
+ "loss": 1.5,
+ "theoretical_loss": 4.000346380371554,
+ "tokens_seen": 437846016
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007008545636910732,
+ "loss": 1.4292,
+ "theoretical_loss": 4.000280882125605,
+ "tokens_seen": 437911552
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.00070083851554664,
+ "loss": 1.45,
+ "theoretical_loss": 4.000215396425259,
+ "tokens_seen": 437977088
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007008224674022066,
+ "loss": 1.5011,
+ "theoretical_loss": 4.000149923266237,
+ "tokens_seen": 438042624
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007008064192577734,
+ "loss": 1.4882,
+ "theoretical_loss": 4.0000844626442635,
+ "tokens_seen": 438108160
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.00070079037111334,
+ "loss": 1.5182,
+ "theoretical_loss": 4.000019014555059,
+ "tokens_seen": 438173696
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007007743229689068,
+ "loss": 1.402,
+ "theoretical_loss": 3.9999535789943543,
+ "tokens_seen": 438239232
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007007582748244734,
+ "loss": 1.591,
+ "theoretical_loss": 3.9998881559578754,
+ "tokens_seen": 438304768
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007007422266800402,
+ "loss": 1.4926,
+ "theoretical_loss": 3.999822745441353,
+ "tokens_seen": 438370304
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007007261785356068,
+ "loss": 1.504,
+ "theoretical_loss": 3.9997573474405215,
+ "tokens_seen": 438435840
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007007101303911736,
+ "loss": 1.4095,
+ "theoretical_loss": 3.9996919619511146,
+ "tokens_seen": 438501376
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007006940822467402,
+ "loss": 1.5043,
+ "theoretical_loss": 3.9996265889688702,
+ "tokens_seen": 438566912
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700678034102307,
+ "loss": 1.4652,
+ "theoretical_loss": 3.999561228489527,
+ "tokens_seen": 438632448
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007006619859578736,
+ "loss": 1.444,
+ "theoretical_loss": 3.9994958805088268,
+ "tokens_seen": 438697984
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007006459378134404,
+ "loss": 1.5337,
+ "theoretical_loss": 3.999430545022512,
+ "tokens_seen": 438763520
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700629889669007,
+ "loss": 1.5445,
+ "theoretical_loss": 3.999365222026329,
+ "tokens_seen": 438829056
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007006138415245738,
+ "loss": 1.689,
+ "theoretical_loss": 3.999299911516024,
+ "tokens_seen": 438894592
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005977933801404,
+ "loss": 1.5859,
+ "theoretical_loss": 3.9992346134873484,
+ "tokens_seen": 438960128
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005817452357072,
+ "loss": 1.4005,
+ "theoretical_loss": 3.999169327936052,
+ "tokens_seen": 439025664
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 152335,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6528295278549194,
+ "objective/train/theoretical_loss": 3.99910405485789,
+ "objective/train/tokens_used": 459551200,
+ "theoretical_loss": 3.99910405485789,
+ "tokens_seen": 439091200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005656970912738,
+ "loss": 1.5898,
+ "theoretical_loss": 3.99910405485789,
+ "tokens_seen": 439091200
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005496489468406,
+ "loss": 1.601,
+ "theoretical_loss": 3.9990387942486167,
+ "tokens_seen": 439156736
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005336008024072,
+ "loss": 1.5703,
+ "theoretical_loss": 3.9989735461039917,
+ "tokens_seen": 439222272
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700517552657974,
+ "loss": 1.5155,
+ "theoretical_loss": 3.9989083104197736,
+ "tokens_seen": 439287808
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007005015045135406,
+ "loss": 1.5189,
+ "theoretical_loss": 3.9988430871917253,
+ "tokens_seen": 439353344
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007004854563691074,
+ "loss": 1.4124,
+ "theoretical_loss": 3.9987778764156108,
+ "tokens_seen": 439418880
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700469408224674,
+ "loss": 1.4934,
+ "theoretical_loss": 3.9987126780871955,
+ "tokens_seen": 439484416
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007004533600802408,
+ "loss": 1.4822,
+ "theoretical_loss": 3.998647492202249,
+ "tokens_seen": 439549952
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007004373119358074,
+ "loss": 1.4562,
+ "theoretical_loss": 3.99858231875654,
+ "tokens_seen": 439615488
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007004212637913742,
+ "loss": 1.6713,
+ "theoretical_loss": 3.9985171577458427,
+ "tokens_seen": 439681024
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007004052156469408,
+ "loss": 1.4871,
+ "theoretical_loss": 3.99845200916593,
+ "tokens_seen": 439746560
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007003891675025076,
+ "loss": 1.5559,
+ "theoretical_loss": 3.9983868730125796,
+ "tokens_seen": 439812096
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007003731193580742,
+ "loss": 1.4817,
+ "theoretical_loss": 3.998321749281569,
+ "tokens_seen": 439877632
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700357071213641,
+ "loss": 1.6303,
+ "theoretical_loss": 3.99825663796868,
+ "tokens_seen": 439943168
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007003410230692076,
+ "loss": 1.5342,
+ "theoretical_loss": 3.998191539069694,
+ "tokens_seen": 440008704
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007003249749247744,
+ "loss": 1.4689,
+ "theoretical_loss": 3.9981264525803972,
+ "tokens_seen": 440074240
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700308926780341,
+ "loss": 1.636,
+ "theoretical_loss": 3.9980613784965753,
+ "tokens_seen": 440139776
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002928786359078,
+ "loss": 1.4691,
+ "theoretical_loss": 3.9979963168140182,
+ "tokens_seen": 440205312
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002768304914745,
+ "loss": 1.5015,
+ "theoretical_loss": 3.997931267528516,
+ "tokens_seen": 440270848
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002607823470412,
+ "loss": 1.5555,
+ "theoretical_loss": 3.997866230635862,
+ "tokens_seen": 440336384
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002447342026079,
+ "loss": 1.4922,
+ "theoretical_loss": 3.9978012061318515,
+ "tokens_seen": 440401920
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002286860581746,
+ "loss": 1.5052,
+ "theoretical_loss": 3.997736194012281,
+ "tokens_seen": 440467456
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007002126379137413,
+ "loss": 1.4865,
+ "theoretical_loss": 3.99767119427295,
+ "tokens_seen": 440532992
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700196589769308,
+ "loss": 1.5147,
+ "theoretical_loss": 3.9976062069096603,
+ "tokens_seen": 440598528
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001805416248747,
+ "loss": 1.5291,
+ "theoretical_loss": 3.9975412319182144,
+ "tokens_seen": 440664064
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001644934804414,
+ "loss": 1.4955,
+ "theoretical_loss": 3.9974762692944172,
+ "tokens_seen": 440729600
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001484453360081,
+ "loss": 1.5683,
+ "theoretical_loss": 3.997411319034077,
+ "tokens_seen": 440795136
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001323971915748,
+ "loss": 1.4643,
+ "theoretical_loss": 3.9973463811330037,
+ "tokens_seen": 440860672
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001163490471415,
+ "loss": 1.436,
+ "theoretical_loss": 3.997281455587007,
+ "tokens_seen": 440926208
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007001003009027082,
+ "loss": 1.6007,
+ "theoretical_loss": 3.9972165423919015,
+ "tokens_seen": 440991744
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007000842527582749,
+ "loss": 1.4685,
+ "theoretical_loss": 3.9971516415435024,
+ "tokens_seen": 441057280
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007000682046138416,
+ "loss": 1.5206,
+ "theoretical_loss": 3.9970867530376273,
+ "tokens_seen": 441122816
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007000521564694083,
+ "loss": 1.4993,
+ "theoretical_loss": 3.9970218768700967,
+ "tokens_seen": 441188352
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000700036108324975,
+ "loss": 1.48,
+ "theoretical_loss": 3.99695701303673,
+ "tokens_seen": 441253888
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007000200601805417,
+ "loss": 1.4757,
+ "theoretical_loss": 3.9968921615333532,
+ "tokens_seen": 441319424
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0007000040120361084,
+ "loss": 1.5006,
+ "theoretical_loss": 3.996827322355791,
+ "tokens_seen": 441384960
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999879638916751,
+ "loss": 1.432,
+ "theoretical_loss": 3.9967624954998713,
+ "tokens_seen": 441450496
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999719157472418,
+ "loss": 1.7472,
+ "theoretical_loss": 3.9966976809614234,
+ "tokens_seen": 441516032
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999558676028085,
+ "loss": 1.4991,
+ "theoretical_loss": 3.99663287873628,
+ "tokens_seen": 441581568
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999398194583752,
+ "loss": 1.5962,
+ "theoretical_loss": 3.9965680888202737,
+ "tokens_seen": 441647104
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999237713139419,
+ "loss": 1.4266,
+ "theoretical_loss": 3.996503311209242,
+ "tokens_seen": 441712640
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006999077231695086,
+ "loss": 1.587,
+ "theoretical_loss": 3.9964385458990206,
+ "tokens_seen": 441778176
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006998916750250752,
+ "loss": 1.5794,
+ "theoretical_loss": 3.9963737928854517,
+ "tokens_seen": 441843712
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000699875626880642,
+ "loss": 1.4391,
+ "theoretical_loss": 3.996309052164376,
+ "tokens_seen": 441909248
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006998595787362086,
+ "loss": 1.5746,
+ "theoretical_loss": 3.996244323731638,
+ "tokens_seen": 441974784
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006998435305917754,
+ "loss": 1.5798,
+ "theoretical_loss": 3.996179607583082,
+ "tokens_seen": 442040320
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000699827482447342,
+ "loss": 1.3749,
+ "theoretical_loss": 3.9961149037145587,
+ "tokens_seen": 442105856
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006998114343029088,
+ "loss": 1.554,
+ "theoretical_loss": 3.996050212121916,
+ "tokens_seen": 442171392
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997953861584754,
+ "loss": 1.4281,
+ "theoretical_loss": 3.995985532801007,
+ "tokens_seen": 442236928
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997793380140422,
+ "loss": 1.5208,
+ "theoretical_loss": 3.995920865747685,
+ "tokens_seen": 442302464
+ },
+ {
+ "epoch": 0.13,
+ "objective/train/docs_used": 153589,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5176739692687988,
+ "objective/train/theoretical_loss": 3.9958562109578066,
+ "objective/train/tokens_used": 462828000,
+ "theoretical_loss": 3.9958562109578066,
+ "tokens_seen": 442368000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997632898696088,
+ "loss": 1.4968,
+ "theoretical_loss": 3.9958562109578066,
+ "tokens_seen": 442368000
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997472417251755,
+ "loss": 1.5038,
+ "theoretical_loss": 3.9957915684272294,
+ "tokens_seen": 442433536
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997311935807422,
+ "loss": 1.4439,
+ "theoretical_loss": 3.995726938151815,
+ "tokens_seen": 442499072
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006997151454363089,
+ "loss": 1.644,
+ "theoretical_loss": 3.995662320127423,
+ "tokens_seen": 442564608
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996990972918756,
+ "loss": 1.4451,
+ "theoretical_loss": 3.9955977143499197,
+ "tokens_seen": 442630144
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996830491474423,
+ "loss": 1.612,
+ "theoretical_loss": 3.99553312081517,
+ "tokens_seen": 442695680
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000699667001003009,
+ "loss": 1.6093,
+ "theoretical_loss": 3.995468539519042,
+ "tokens_seen": 442761216
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996509528585757,
+ "loss": 1.4835,
+ "theoretical_loss": 3.9954039704574065,
+ "tokens_seen": 442826752
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996349047141424,
+ "loss": 1.6179,
+ "theoretical_loss": 3.9953394136261355,
+ "tokens_seen": 442892288
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996188565697091,
+ "loss": 1.5285,
+ "theoretical_loss": 3.995274869021103,
+ "tokens_seen": 442957824
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006996028084252758,
+ "loss": 1.5937,
+ "theoretical_loss": 3.995210336638185,
+ "tokens_seen": 443023360
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006995867602808425,
+ "loss": 1.6425,
+ "theoretical_loss": 3.9951458164732596,
+ "tokens_seen": 443088896
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006995707121364092,
+ "loss": 1.468,
+ "theoretical_loss": 3.9950813085222068,
+ "tokens_seen": 443154432
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006995546639919759,
+ "loss": 1.4978,
+ "theoretical_loss": 3.99501681278091,
+ "tokens_seen": 443219968
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006995386158475426,
+ "loss": 1.4505,
+ "theoretical_loss": 3.994952329245251,
+ "tokens_seen": 443285504
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006995225677031093,
+ "loss": 1.5477,
+ "theoretical_loss": 3.994887857911118,
+ "tokens_seen": 443351040
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000699506519558676,
+ "loss": 1.5128,
+ "theoretical_loss": 3.9948233987743986,
+ "tokens_seen": 443416576
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994904714142427,
+ "loss": 1.529,
+ "theoretical_loss": 3.994758951830982,
+ "tokens_seen": 443482112
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994744232698095,
+ "loss": 1.5579,
+ "theoretical_loss": 3.9946945170767614,
+ "tokens_seen": 443547648
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994583751253761,
+ "loss": 1.4624,
+ "theoretical_loss": 3.994630094507631,
+ "tokens_seen": 443613184
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994423269809429,
+ "loss": 1.5406,
+ "theoretical_loss": 3.994565684119486,
+ "tokens_seen": 443678720
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994262788365095,
+ "loss": 1.5907,
+ "theoretical_loss": 3.9945012859082247,
+ "tokens_seen": 443744256
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006994102306920763,
+ "loss": 1.4722,
+ "theoretical_loss": 3.9944368998697475,
+ "tokens_seen": 443809792
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993941825476429,
+ "loss": 1.4417,
+ "theoretical_loss": 3.9943725259999563,
+ "tokens_seen": 443875328
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993781344032097,
+ "loss": 1.5681,
+ "theoretical_loss": 3.994308164294755,
+ "tokens_seen": 443940864
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993620862587763,
+ "loss": 1.3864,
+ "theoretical_loss": 3.9942438147500496,
+ "tokens_seen": 444006400
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993460381143431,
+ "loss": 1.5363,
+ "theoretical_loss": 3.9941794773617487,
+ "tokens_seen": 444071936
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993299899699097,
+ "loss": 1.572,
+ "theoretical_loss": 3.9941151521257616,
+ "tokens_seen": 444137472
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006993139418254765,
+ "loss": 1.4912,
+ "theoretical_loss": 3.9940508390380005,
+ "tokens_seen": 444203008
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992978936810431,
+ "loss": 1.6135,
+ "theoretical_loss": 3.9939865380943793,
+ "tokens_seen": 444268544
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992818455366099,
+ "loss": 1.5042,
+ "theoretical_loss": 3.993922249290814,
+ "tokens_seen": 444334080
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992657973921765,
+ "loss": 1.4777,
+ "theoretical_loss": 3.993857972623222,
+ "tokens_seen": 444399616
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992497492477433,
+ "loss": 1.5358,
+ "theoretical_loss": 3.993793708087524,
+ "tokens_seen": 444465152
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992337011033099,
+ "loss": 1.4613,
+ "theoretical_loss": 3.9937294556796417,
+ "tokens_seen": 444530688
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992176529588767,
+ "loss": 1.6225,
+ "theoretical_loss": 3.993665215395498,
+ "tokens_seen": 444596224
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006992016048144433,
+ "loss": 1.5545,
+ "theoretical_loss": 3.9936009872310194,
+ "tokens_seen": 444661760
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991855566700101,
+ "loss": 1.5691,
+ "theoretical_loss": 3.993536771182134,
+ "tokens_seen": 444727296
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991695085255767,
+ "loss": 1.4441,
+ "theoretical_loss": 3.993472567244771,
+ "tokens_seen": 444792832
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991534603811435,
+ "loss": 1.6013,
+ "theoretical_loss": 3.9934083754148615,
+ "tokens_seen": 444858368
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991374122367101,
+ "loss": 1.4713,
+ "theoretical_loss": 3.9933441956883398,
+ "tokens_seen": 444923904
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991213640922769,
+ "loss": 1.6447,
+ "theoretical_loss": 3.9932800280611422,
+ "tokens_seen": 444989440
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006991053159478435,
+ "loss": 1.4422,
+ "theoretical_loss": 3.9932158725292055,
+ "tokens_seen": 445054976
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006990892678034103,
+ "loss": 1.4835,
+ "theoretical_loss": 3.993151729088469,
+ "tokens_seen": 445120512
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.000699073219658977,
+ "loss": 1.4059,
+ "theoretical_loss": 3.993087597734875,
+ "tokens_seen": 445186048
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006990571715145437,
+ "loss": 1.4937,
+ "theoretical_loss": 3.993023478464366,
+ "tokens_seen": 445251584
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006990411233701104,
+ "loss": 1.4638,
+ "theoretical_loss": 3.992959371272888,
+ "tokens_seen": 445317120
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006990250752256771,
+ "loss": 1.57,
+ "theoretical_loss": 3.992895276156388,
+ "tokens_seen": 445382656
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 0.0006990090270812438,
+ "loss": 1.4905,
+ "theoretical_loss": 3.992831193110816,
+ "tokens_seen": 445448192
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006989929789368105,
+ "loss": 1.4754,
+ "theoretical_loss": 3.992767122132123,
+ "tokens_seen": 445513728
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006989769307923772,
+ "loss": 1.5195,
+ "theoretical_loss": 3.9927030632162612,
+ "tokens_seen": 445579264
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 154520,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.673126220703125,
+ "objective/train/theoretical_loss": 3.9926390163591874,
+ "objective/train/tokens_used": 466104800,
+ "theoretical_loss": 3.9926390163591874,
+ "tokens_seen": 445644800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006989608826479439,
+ "loss": 1.5465,
+ "theoretical_loss": 3.9926390163591874,
+ "tokens_seen": 445644800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006989448345035106,
+ "loss": 1.4801,
+ "theoretical_loss": 3.9925749815568583,
+ "tokens_seen": 445710336
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006989287863590773,
+ "loss": 1.4372,
+ "theoretical_loss": 3.9925109588052328,
+ "tokens_seen": 445775872
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698912738214644,
+ "loss": 1.5426,
+ "theoretical_loss": 3.9924469481002713,
+ "tokens_seen": 445841408
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988966900702107,
+ "loss": 1.5802,
+ "theoretical_loss": 3.9923829494379377,
+ "tokens_seen": 445906944
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988806419257774,
+ "loss": 1.5393,
+ "theoretical_loss": 3.9923189628141964,
+ "tokens_seen": 445972480
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988645937813441,
+ "loss": 1.4943,
+ "theoretical_loss": 3.9922549882250147,
+ "tokens_seen": 446038016
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988485456369108,
+ "loss": 1.4797,
+ "theoretical_loss": 3.9921910256663606,
+ "tokens_seen": 446103552
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988324974924775,
+ "loss": 1.4582,
+ "theoretical_loss": 3.9921270751342064,
+ "tokens_seen": 446169088
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988164493480442,
+ "loss": 1.3838,
+ "theoretical_loss": 3.992063136624523,
+ "tokens_seen": 446234624
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006988004012036109,
+ "loss": 1.4467,
+ "theoretical_loss": 3.9919992101332866,
+ "tokens_seen": 446300160
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006987843530591776,
+ "loss": 1.5887,
+ "theoretical_loss": 3.991935295656473,
+ "tokens_seen": 446365696
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006987683049147443,
+ "loss": 1.4229,
+ "theoretical_loss": 3.9918713931900602,
+ "tokens_seen": 446431232
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698752256770311,
+ "loss": 1.5085,
+ "theoretical_loss": 3.9918075027300297,
+ "tokens_seen": 446496768
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006987362086258777,
+ "loss": 1.5026,
+ "theoretical_loss": 3.991743624272363,
+ "tokens_seen": 446562304
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006987201604814444,
+ "loss": 1.5529,
+ "theoretical_loss": 3.9916797578130456,
+ "tokens_seen": 446627840
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006987041123370111,
+ "loss": 1.4314,
+ "theoretical_loss": 3.9916159033480625,
+ "tokens_seen": 446693376
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986880641925778,
+ "loss": 1.5042,
+ "theoretical_loss": 3.9915520608734028,
+ "tokens_seen": 446758912
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986720160481445,
+ "loss": 1.5427,
+ "theoretical_loss": 3.991488230385056,
+ "tokens_seen": 446824448
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986559679037112,
+ "loss": 1.5715,
+ "theoretical_loss": 3.991424411879014,
+ "tokens_seen": 446889984
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986399197592779,
+ "loss": 1.532,
+ "theoretical_loss": 3.9913606053512716,
+ "tokens_seen": 446955520
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986238716148446,
+ "loss": 1.4223,
+ "theoretical_loss": 3.9912968107978237,
+ "tokens_seen": 447021056
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006986078234704113,
+ "loss": 1.5344,
+ "theoretical_loss": 3.9912330282146695,
+ "tokens_seen": 447086592
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698591775325978,
+ "loss": 1.606,
+ "theoretical_loss": 3.9911692575978077,
+ "tokens_seen": 447152128
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006985757271815447,
+ "loss": 1.4802,
+ "theoretical_loss": 3.9911054989432397,
+ "tokens_seen": 447217664
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006985596790371114,
+ "loss": 1.4954,
+ "theoretical_loss": 3.9910417522469706,
+ "tokens_seen": 447283200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006985436308926781,
+ "loss": 1.4643,
+ "theoretical_loss": 3.990978017505004,
+ "tokens_seen": 447348736
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006985275827482448,
+ "loss": 1.4879,
+ "theoretical_loss": 3.990914294713348,
+ "tokens_seen": 447414272
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006985115346038115,
+ "loss": 1.3527,
+ "theoretical_loss": 3.9908505838680126,
+ "tokens_seen": 447479808
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984954864593781,
+ "loss": 1.5504,
+ "theoretical_loss": 3.9907868849650088,
+ "tokens_seen": 447545344
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984794383149449,
+ "loss": 1.4939,
+ "theoretical_loss": 3.9907231980003495,
+ "tokens_seen": 447610880
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984633901705115,
+ "loss": 1.4516,
+ "theoretical_loss": 3.99065952297005,
+ "tokens_seen": 447676416
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984473420260783,
+ "loss": 1.4114,
+ "theoretical_loss": 3.990595859870127,
+ "tokens_seen": 447741952
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984312938816449,
+ "loss": 1.6211,
+ "theoretical_loss": 3.9905322086966004,
+ "tokens_seen": 447807488
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006984152457372117,
+ "loss": 1.5145,
+ "theoretical_loss": 3.990468569445489,
+ "tokens_seen": 447873024
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983991975927783,
+ "loss": 1.5833,
+ "theoretical_loss": 3.990404942112818,
+ "tokens_seen": 447938560
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983831494483451,
+ "loss": 1.4963,
+ "theoretical_loss": 3.9903413266946104,
+ "tokens_seen": 448004096
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983671013039117,
+ "loss": 1.4331,
+ "theoretical_loss": 3.9902777231868933,
+ "tokens_seen": 448069632
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983510531594784,
+ "loss": 1.4788,
+ "theoretical_loss": 3.990214131585695,
+ "tokens_seen": 448135168
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983350050150451,
+ "loss": 1.4973,
+ "theoretical_loss": 3.990150551887047,
+ "tokens_seen": 448200704
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983189568706118,
+ "loss": 1.4562,
+ "theoretical_loss": 3.9900869840869793,
+ "tokens_seen": 448266240
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006983029087261785,
+ "loss": 1.6024,
+ "theoretical_loss": 3.9900234281815283,
+ "tokens_seen": 448331776
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006982868605817452,
+ "loss": 1.4377,
+ "theoretical_loss": 3.989959884166728,
+ "tokens_seen": 448397312
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006982708124373119,
+ "loss": 1.5387,
+ "theoretical_loss": 3.9898963520386186,
+ "tokens_seen": 448462848
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006982547642928786,
+ "loss": 1.5395,
+ "theoretical_loss": 3.9898328317932386,
+ "tokens_seen": 448528384
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006982387161484453,
+ "loss": 1.5434,
+ "theoretical_loss": 3.98976932342663,
+ "tokens_seen": 448593920
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698222668004012,
+ "loss": 1.6005,
+ "theoretical_loss": 3.9897058269348364,
+ "tokens_seen": 448659456
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006982066198595787,
+ "loss": 1.5173,
+ "theoretical_loss": 3.9896423423139034,
+ "tokens_seen": 448724992
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006981905717151454,
+ "loss": 1.4321,
+ "theoretical_loss": 3.989578869559879,
+ "tokens_seen": 448790528
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006981745235707121,
+ "loss": 1.3811,
+ "theoretical_loss": 3.989515408668812,
+ "tokens_seen": 448856064
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 155690,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5394407510757446,
+ "objective/train/theoretical_loss": 3.989451959636754,
+ "objective/train/tokens_used": 469381600,
+ "theoretical_loss": 3.989451959636754,
+ "tokens_seen": 448921600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006981584754262788,
+ "loss": 1.4618,
+ "theoretical_loss": 3.989451959636754,
+ "tokens_seen": 448921600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006981424272818456,
+ "loss": 1.5767,
+ "theoretical_loss": 3.989388522459757,
+ "tokens_seen": 448987136
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006981263791374122,
+ "loss": 1.5553,
+ "theoretical_loss": 3.989325097133878,
+ "tokens_seen": 449052672
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698110330992979,
+ "loss": 1.59,
+ "theoretical_loss": 3.989261683655172,
+ "tokens_seen": 449118208
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006980942828485456,
+ "loss": 1.5345,
+ "theoretical_loss": 3.989198282019699,
+ "tokens_seen": 449183744
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006980782347041124,
+ "loss": 1.4384,
+ "theoretical_loss": 3.989134892223519,
+ "tokens_seen": 449249280
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000698062186559679,
+ "loss": 1.5084,
+ "theoretical_loss": 3.9890715142626956,
+ "tokens_seen": 449314816
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006980461384152458,
+ "loss": 1.4859,
+ "theoretical_loss": 3.9890081481332924,
+ "tokens_seen": 449380352
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006980300902708124,
+ "loss": 1.5314,
+ "theoretical_loss": 3.988944793831376,
+ "tokens_seen": 449445888
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006980140421263792,
+ "loss": 1.4309,
+ "theoretical_loss": 3.9888814513530138,
+ "tokens_seen": 449511424
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006979979939819458,
+ "loss": 1.5783,
+ "theoretical_loss": 3.988818120694277,
+ "tokens_seen": 449576960
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006979819458375126,
+ "loss": 1.423,
+ "theoretical_loss": 3.988754801851237,
+ "tokens_seen": 449642496
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006979658976930792,
+ "loss": 1.517,
+ "theoretical_loss": 3.9886914948199683,
+ "tokens_seen": 449708032
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697949849548646,
+ "loss": 1.5909,
+ "theoretical_loss": 3.988628199596546,
+ "tokens_seen": 449773568
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006979338014042126,
+ "loss": 1.4381,
+ "theoretical_loss": 3.9885649161770473,
+ "tokens_seen": 449839104
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006979177532597794,
+ "loss": 1.4802,
+ "theoretical_loss": 3.988501644557553,
+ "tokens_seen": 449904640
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697901705115346,
+ "loss": 1.4035,
+ "theoretical_loss": 3.9884383847341436,
+ "tokens_seen": 449970176
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978856569709128,
+ "loss": 1.5362,
+ "theoretical_loss": 3.988375136702902,
+ "tokens_seen": 450035712
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978696088264795,
+ "loss": 1.4519,
+ "theoretical_loss": 3.988311900459914,
+ "tokens_seen": 450101248
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978535606820462,
+ "loss": 1.5602,
+ "theoretical_loss": 3.988248676001267,
+ "tokens_seen": 450166784
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978375125376129,
+ "loss": 1.4926,
+ "theoretical_loss": 3.9881854633230485,
+ "tokens_seen": 450232320
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978214643931796,
+ "loss": 1.5247,
+ "theoretical_loss": 3.9881222624213497,
+ "tokens_seen": 450297856
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006978054162487463,
+ "loss": 1.5003,
+ "theoretical_loss": 3.9880590732922636,
+ "tokens_seen": 450363392
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697789368104313,
+ "loss": 1.7043,
+ "theoretical_loss": 3.9879958959318844,
+ "tokens_seen": 450428928
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006977733199598797,
+ "loss": 1.583,
+ "theoretical_loss": 3.9879327303363086,
+ "tokens_seen": 450494464
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006977572718154464,
+ "loss": 1.5782,
+ "theoretical_loss": 3.9878695765016334,
+ "tokens_seen": 450560000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006977412236710131,
+ "loss": 1.4689,
+ "theoretical_loss": 3.98780643442396,
+ "tokens_seen": 450625536
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006977251755265798,
+ "loss": 1.4843,
+ "theoretical_loss": 3.9877433040993897,
+ "tokens_seen": 450691072
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006977091273821465,
+ "loss": 1.54,
+ "theoretical_loss": 3.9876801855240265,
+ "tokens_seen": 450756608
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006976930792377132,
+ "loss": 1.4755,
+ "theoretical_loss": 3.987617078693975,
+ "tokens_seen": 450822144
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006976770310932799,
+ "loss": 1.4973,
+ "theoretical_loss": 3.987553983605345,
+ "tokens_seen": 450887680
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006976609829488466,
+ "loss": 1.6195,
+ "theoretical_loss": 3.9874909002542434,
+ "tokens_seen": 450953216
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006976449348044133,
+ "loss": 1.4774,
+ "theoretical_loss": 3.987427828636782,
+ "tokens_seen": 451018752
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.00069762888665998,
+ "loss": 1.5001,
+ "theoretical_loss": 3.9873647687490745,
+ "tokens_seen": 451084288
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006976128385155467,
+ "loss": 1.6324,
+ "theoretical_loss": 3.9873017205872356,
+ "tokens_seen": 451149824
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975967903711134,
+ "loss": 1.432,
+ "theoretical_loss": 3.9872386841473815,
+ "tokens_seen": 451215360
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975807422266801,
+ "loss": 1.5045,
+ "theoretical_loss": 3.987175659425631,
+ "tokens_seen": 451280896
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975646940822468,
+ "loss": 1.5344,
+ "theoretical_loss": 3.9871126464181046,
+ "tokens_seen": 451346432
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975486459378135,
+ "loss": 1.5424,
+ "theoretical_loss": 3.9870496451209245,
+ "tokens_seen": 451411968
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975325977933802,
+ "loss": 1.605,
+ "theoretical_loss": 3.986986655530215,
+ "tokens_seen": 451477504
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975165496489469,
+ "loss": 1.5406,
+ "theoretical_loss": 3.9869236776421015,
+ "tokens_seen": 451543040
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006975005015045136,
+ "loss": 1.4135,
+ "theoretical_loss": 3.986860711452713,
+ "tokens_seen": 451608576
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006974844533600803,
+ "loss": 1.5881,
+ "theoretical_loss": 3.986797756958178,
+ "tokens_seen": 451674112
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697468405215647,
+ "loss": 1.5084,
+ "theoretical_loss": 3.9867348141546284,
+ "tokens_seen": 451739648
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006974523570712137,
+ "loss": 1.5528,
+ "theoretical_loss": 3.9866718830381975,
+ "tokens_seen": 451805184
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006974363089267804,
+ "loss": 1.4579,
+ "theoretical_loss": 3.98660896360502,
+ "tokens_seen": 451870720
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006974202607823471,
+ "loss": 1.5445,
+ "theoretical_loss": 3.9865460558512336,
+ "tokens_seen": 451936256
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006974042126379138,
+ "loss": 1.3889,
+ "theoretical_loss": 3.986483159772977,
+ "tokens_seen": 452001792
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006973881644934805,
+ "loss": 1.4828,
+ "theoretical_loss": 3.986420275366391,
+ "tokens_seen": 452067328
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006973721163490472,
+ "loss": 1.5615,
+ "theoretical_loss": 3.9863574026276174,
+ "tokens_seen": 452132864
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 156834,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4055612087249756,
+ "objective/train/theoretical_loss": 3.986294541552801,
+ "objective/train/tokens_used": 472658400,
+ "theoretical_loss": 3.986294541552801,
+ "tokens_seen": 452198400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006973560682046139,
+ "loss": 1.4389,
+ "theoretical_loss": 3.986294541552801,
+ "tokens_seen": 452198400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006973400200601806,
+ "loss": 1.5344,
+ "theoretical_loss": 3.9862316921380883,
+ "tokens_seen": 452263936
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006973239719157473,
+ "loss": 1.5517,
+ "theoretical_loss": 3.9861688543796268,
+ "tokens_seen": 452329472
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697307923771314,
+ "loss": 1.4465,
+ "theoretical_loss": 3.986106028273567,
+ "tokens_seen": 452395008
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972918756268807,
+ "loss": 1.4825,
+ "theoretical_loss": 3.9860432138160595,
+ "tokens_seen": 452460544
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972758274824474,
+ "loss": 1.3684,
+ "theoretical_loss": 3.985980411003259,
+ "tokens_seen": 452526080
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972597793380141,
+ "loss": 1.4558,
+ "theoretical_loss": 3.98591761983132,
+ "tokens_seen": 452591616
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972437311935808,
+ "loss": 1.5356,
+ "theoretical_loss": 3.9858548402964,
+ "tokens_seen": 452657152
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972276830491475,
+ "loss": 1.4826,
+ "theoretical_loss": 3.985792072394658,
+ "tokens_seen": 452722688
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006972116349047142,
+ "loss": 1.4988,
+ "theoretical_loss": 3.9857293161222542,
+ "tokens_seen": 452788224
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006971955867602809,
+ "loss": 1.5111,
+ "theoretical_loss": 3.985666571475352,
+ "tokens_seen": 452853760
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006971795386158476,
+ "loss": 1.5831,
+ "theoretical_loss": 3.985603838450116,
+ "tokens_seen": 452919296
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006971634904714143,
+ "loss": 1.5667,
+ "theoretical_loss": 3.9855411170427115,
+ "tokens_seen": 452984832
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697147442326981,
+ "loss": 1.4883,
+ "theoretical_loss": 3.985478407249307,
+ "tokens_seen": 453050368
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006971313941825477,
+ "loss": 1.5357,
+ "theoretical_loss": 3.985415709066073,
+ "tokens_seen": 453115904
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006971153460381144,
+ "loss": 1.4585,
+ "theoretical_loss": 3.98535302248918,
+ "tokens_seen": 453181440
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000697099297893681,
+ "loss": 1.4994,
+ "theoretical_loss": 3.9852903475148027,
+ "tokens_seen": 453246976
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970832497492478,
+ "loss": 1.5574,
+ "theoretical_loss": 3.985227684139116,
+ "tokens_seen": 453312512
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970672016048144,
+ "loss": 1.5326,
+ "theoretical_loss": 3.9851650323582977,
+ "tokens_seen": 453378048
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970511534603812,
+ "loss": 1.4839,
+ "theoretical_loss": 3.9851023921685256,
+ "tokens_seen": 453443584
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970351053159478,
+ "loss": 1.471,
+ "theoretical_loss": 3.9850397635659807,
+ "tokens_seen": 453509120
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970190571715146,
+ "loss": 1.4931,
+ "theoretical_loss": 3.9849771465468464,
+ "tokens_seen": 453574656
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006970030090270812,
+ "loss": 1.5389,
+ "theoretical_loss": 3.9849145411073064,
+ "tokens_seen": 453640192
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696986960882648,
+ "loss": 1.6396,
+ "theoretical_loss": 3.984851947243547,
+ "tokens_seen": 453705728
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006969709127382146,
+ "loss": 1.4131,
+ "theoretical_loss": 3.984789364951757,
+ "tokens_seen": 453771264
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006969548645937813,
+ "loss": 1.534,
+ "theoretical_loss": 3.9847267942281253,
+ "tokens_seen": 453836800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696938816449348,
+ "loss": 1.4727,
+ "theoretical_loss": 3.984664235068843,
+ "tokens_seen": 453902336
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006969227683049147,
+ "loss": 1.4447,
+ "theoretical_loss": 3.9846016874701053,
+ "tokens_seen": 453967872
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006969067201604814,
+ "loss": 1.5291,
+ "theoretical_loss": 3.9845391514281063,
+ "tokens_seen": 454033408
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968906720160481,
+ "loss": 1.465,
+ "theoretical_loss": 3.9844766269390437,
+ "tokens_seen": 454098944
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968746238716148,
+ "loss": 1.5212,
+ "theoretical_loss": 3.984414113999115,
+ "tokens_seen": 454164480
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968585757271815,
+ "loss": 1.4242,
+ "theoretical_loss": 3.984351612604522,
+ "tokens_seen": 454230016
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968425275827482,
+ "loss": 1.4277,
+ "theoretical_loss": 3.984289122751467,
+ "tokens_seen": 454295552
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968264794383149,
+ "loss": 1.5631,
+ "theoretical_loss": 3.984226644436154,
+ "tokens_seen": 454361088
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006968104312938816,
+ "loss": 1.4569,
+ "theoretical_loss": 3.9841641776547885,
+ "tokens_seen": 454426624
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006967943831494483,
+ "loss": 1.4869,
+ "theoretical_loss": 3.98410172240358,
+ "tokens_seen": 454492160
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696778335005015,
+ "loss": 1.4457,
+ "theoretical_loss": 3.984039278678736,
+ "tokens_seen": 454557696
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006967622868605817,
+ "loss": 1.5251,
+ "theoretical_loss": 3.9839768464764695,
+ "tokens_seen": 454623232
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006967462387161485,
+ "loss": 1.6093,
+ "theoretical_loss": 3.9839144257929933,
+ "tokens_seen": 454688768
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006967301905717151,
+ "loss": 1.409,
+ "theoretical_loss": 3.983852016624522,
+ "tokens_seen": 454754304
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006967141424272819,
+ "loss": 1.5454,
+ "theoretical_loss": 3.9837896189672724,
+ "tokens_seen": 454819840
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966980942828485,
+ "loss": 1.5249,
+ "theoretical_loss": 3.9837272328174635,
+ "tokens_seen": 454885376
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966820461384153,
+ "loss": 1.4779,
+ "theoretical_loss": 3.9836648581713154,
+ "tokens_seen": 454950912
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696665997993982,
+ "loss": 1.5053,
+ "theoretical_loss": 3.9836024950250497,
+ "tokens_seen": 455016448
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966499498495487,
+ "loss": 1.5391,
+ "theoretical_loss": 3.983540143374891,
+ "tokens_seen": 455081984
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966339017051154,
+ "loss": 1.5406,
+ "theoretical_loss": 3.983477803217066,
+ "tokens_seen": 455147520
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966178535606821,
+ "loss": 1.6182,
+ "theoretical_loss": 3.9834154745478,
+ "tokens_seen": 455213056
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006966018054162488,
+ "loss": 1.3834,
+ "theoretical_loss": 3.9833531573633234,
+ "tokens_seen": 455278592
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006965857572718155,
+ "loss": 1.5062,
+ "theoretical_loss": 3.9832908516598673,
+ "tokens_seen": 455344128
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006965697091273822,
+ "loss": 1.509,
+ "theoretical_loss": 3.983228557433664,
+ "tokens_seen": 455409664
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 157689,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5005377531051636,
+ "objective/train/theoretical_loss": 3.9831662746809493,
+ "objective/train/tokens_used": 475935200,
+ "theoretical_loss": 3.9831662746809493,
+ "tokens_seen": 455475200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006965536609829489,
+ "loss": 1.5773,
+ "theoretical_loss": 3.9831662746809493,
+ "tokens_seen": 455475200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006965376128385156,
+ "loss": 1.516,
+ "theoretical_loss": 3.9831040033979583,
+ "tokens_seen": 455540736
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006965215646940823,
+ "loss": 1.3727,
+ "theoretical_loss": 3.9830417435809298,
+ "tokens_seen": 455606272
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696505516549649,
+ "loss": 1.4682,
+ "theoretical_loss": 3.9829794952261035,
+ "tokens_seen": 455671808
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964894684052157,
+ "loss": 1.5151,
+ "theoretical_loss": 3.9829172583297208,
+ "tokens_seen": 455737344
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964734202607824,
+ "loss": 1.4777,
+ "theoretical_loss": 3.9828550328880263,
+ "tokens_seen": 455802880
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964573721163491,
+ "loss": 1.6122,
+ "theoretical_loss": 3.982792818897264,
+ "tokens_seen": 455868416
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964413239719158,
+ "loss": 1.5951,
+ "theoretical_loss": 3.9827306163536815,
+ "tokens_seen": 455933952
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964252758274825,
+ "loss": 1.5475,
+ "theoretical_loss": 3.982668425253528,
+ "tokens_seen": 455999488
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006964092276830492,
+ "loss": 1.4892,
+ "theoretical_loss": 3.982606245593053,
+ "tokens_seen": 456065024
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006963931795386159,
+ "loss": 1.5035,
+ "theoretical_loss": 3.9825440773685092,
+ "tokens_seen": 456130560
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006963771313941826,
+ "loss": 1.493,
+ "theoretical_loss": 3.9824819205761512,
+ "tokens_seen": 456196096
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006963610832497493,
+ "loss": 1.5549,
+ "theoretical_loss": 3.982419775212235,
+ "tokens_seen": 456261632
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696345035105316,
+ "loss": 1.6392,
+ "theoretical_loss": 3.9823576412730173,
+ "tokens_seen": 456327168
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006963289869608827,
+ "loss": 1.5156,
+ "theoretical_loss": 3.982295518754758,
+ "tokens_seen": 456392704
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006963129388164494,
+ "loss": 1.5393,
+ "theoretical_loss": 3.982233407653718,
+ "tokens_seen": 456458240
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962968906720161,
+ "loss": 1.5114,
+ "theoretical_loss": 3.982171307966161,
+ "tokens_seen": 456523776
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962808425275828,
+ "loss": 1.5031,
+ "theoretical_loss": 3.9821092196883505,
+ "tokens_seen": 456589312
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962647943831495,
+ "loss": 1.5498,
+ "theoretical_loss": 3.982047142816554,
+ "tokens_seen": 456654848
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962487462387162,
+ "loss": 1.5781,
+ "theoretical_loss": 3.981985077347039,
+ "tokens_seen": 456720384
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962326980942829,
+ "loss": 1.5119,
+ "theoretical_loss": 3.981923023276075,
+ "tokens_seen": 456785920
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962166499498496,
+ "loss": 1.5188,
+ "theoretical_loss": 3.9818609805999348,
+ "tokens_seen": 456851456
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006962006018054163,
+ "loss": 1.4882,
+ "theoretical_loss": 3.981798949314891,
+ "tokens_seen": 456916992
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000696184553660983,
+ "loss": 1.6035,
+ "theoretical_loss": 3.98173692941722,
+ "tokens_seen": 456982528
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006961685055165497,
+ "loss": 1.5139,
+ "theoretical_loss": 3.981674920903197,
+ "tokens_seen": 457048064
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006961524573721164,
+ "loss": 1.5006,
+ "theoretical_loss": 3.9816129237691023,
+ "tokens_seen": 457113600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006961364092276831,
+ "loss": 1.5368,
+ "theoretical_loss": 3.981550938011215,
+ "tokens_seen": 457179136
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006961203610832498,
+ "loss": 1.4274,
+ "theoretical_loss": 3.981488963625818,
+ "tokens_seen": 457244672
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006961043129388165,
+ "loss": 1.3543,
+ "theoretical_loss": 3.9814270006091954,
+ "tokens_seen": 457310208
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006960882647943832,
+ "loss": 1.5751,
+ "theoretical_loss": 3.981365048957633,
+ "tokens_seen": 457375744
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006960722166499499,
+ "loss": 1.5665,
+ "theoretical_loss": 3.9813031086674173,
+ "tokens_seen": 457441280
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006960561685055166,
+ "loss": 1.4666,
+ "theoretical_loss": 3.9812411797348384,
+ "tokens_seen": 457506816
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006960401203610833,
+ "loss": 1.4307,
+ "theoretical_loss": 3.9811792621561874,
+ "tokens_seen": 457572352
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.00069602407221665,
+ "loss": 1.4939,
+ "theoretical_loss": 3.981117355927756,
+ "tokens_seen": 457637888
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006960080240722167,
+ "loss": 1.541,
+ "theoretical_loss": 3.981055461045839,
+ "tokens_seen": 457703424
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959919759277834,
+ "loss": 1.5948,
+ "theoretical_loss": 3.980993577506733,
+ "tokens_seen": 457768960
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959759277833501,
+ "loss": 1.5795,
+ "theoretical_loss": 3.980931705306735,
+ "tokens_seen": 457834496
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959598796389168,
+ "loss": 1.519,
+ "theoretical_loss": 3.9808698444421458,
+ "tokens_seen": 457900032
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959438314944835,
+ "loss": 1.5537,
+ "theoretical_loss": 3.9808079949092656,
+ "tokens_seen": 457965568
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959277833500502,
+ "loss": 1.5433,
+ "theoretical_loss": 3.9807461567043987,
+ "tokens_seen": 458031104
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006959117352056169,
+ "loss": 1.4325,
+ "theoretical_loss": 3.980684329823849,
+ "tokens_seen": 458096640
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006958956870611836,
+ "loss": 1.5263,
+ "theoretical_loss": 3.980622514263923,
+ "tokens_seen": 458162176
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006958796389167503,
+ "loss": 1.5642,
+ "theoretical_loss": 3.9805607100209297,
+ "tokens_seen": 458227712
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000695863590772317,
+ "loss": 1.4736,
+ "theoretical_loss": 3.980498917091179,
+ "tokens_seen": 458293248
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006958475426278837,
+ "loss": 1.5423,
+ "theoretical_loss": 3.980437135470982,
+ "tokens_seen": 458358784
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006958314944834504,
+ "loss": 1.5497,
+ "theoretical_loss": 3.980375365156653,
+ "tokens_seen": 458424320
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006958154463390171,
+ "loss": 1.4201,
+ "theoretical_loss": 3.9803136061445064,
+ "tokens_seen": 458489856
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957993981945838,
+ "loss": 1.5075,
+ "theoretical_loss": 3.9802518584308597,
+ "tokens_seen": 458555392
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957833500501505,
+ "loss": 1.407,
+ "theoretical_loss": 3.980190122012032,
+ "tokens_seen": 458620928
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957673019057172,
+ "loss": 1.4852,
+ "theoretical_loss": 3.980128396884343,
+ "tokens_seen": 458686464
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 158992,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4671831130981445,
+ "objective/train/theoretical_loss": 3.9800666830441145,
+ "objective/train/tokens_used": 479212000,
+ "theoretical_loss": 3.9800666830441145,
+ "tokens_seen": 458752000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957512537612839,
+ "loss": 1.5057,
+ "theoretical_loss": 3.9800666830441145,
+ "tokens_seen": 458752000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957352056168506,
+ "loss": 1.3742,
+ "theoretical_loss": 3.9800049804876716,
+ "tokens_seen": 458817536
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957191574724173,
+ "loss": 1.6351,
+ "theoretical_loss": 3.9799432892113384,
+ "tokens_seen": 458883072
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006957031093279839,
+ "loss": 1.4647,
+ "theoretical_loss": 3.9798816092114437,
+ "tokens_seen": 458948608
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956870611835507,
+ "loss": 1.4541,
+ "theoretical_loss": 3.9798199404843153,
+ "tokens_seen": 459014144
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956710130391173,
+ "loss": 1.5378,
+ "theoretical_loss": 3.9797582830262845,
+ "tokens_seen": 459079680
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956549648946841,
+ "loss": 1.4761,
+ "theoretical_loss": 3.9796966368336832,
+ "tokens_seen": 459145216
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956389167502507,
+ "loss": 1.5889,
+ "theoretical_loss": 3.979635001902847,
+ "tokens_seen": 459210752
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956228686058175,
+ "loss": 1.6021,
+ "theoretical_loss": 3.9795733782301097,
+ "tokens_seen": 459276288
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006956068204613841,
+ "loss": 1.5399,
+ "theoretical_loss": 3.979511765811811,
+ "tokens_seen": 459341824
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955907723169509,
+ "loss": 1.5921,
+ "theoretical_loss": 3.9794501646442884,
+ "tokens_seen": 459407360
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955747241725175,
+ "loss": 1.5187,
+ "theoretical_loss": 3.9793885747238837,
+ "tokens_seen": 459472896
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955586760280842,
+ "loss": 1.5166,
+ "theoretical_loss": 3.97932699604694,
+ "tokens_seen": 459538432
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955426278836509,
+ "loss": 1.4833,
+ "theoretical_loss": 3.979265428609801,
+ "tokens_seen": 459603968
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955265797392176,
+ "loss": 1.5999,
+ "theoretical_loss": 3.9792038724088137,
+ "tokens_seen": 459669504
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006955105315947843,
+ "loss": 1.5948,
+ "theoretical_loss": 3.9791423274403255,
+ "tokens_seen": 459735040
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954944834503511,
+ "loss": 1.4881,
+ "theoretical_loss": 3.9790807937006853,
+ "tokens_seen": 459800576
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954784353059177,
+ "loss": 1.488,
+ "theoretical_loss": 3.979019271186246,
+ "tokens_seen": 459866112
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954623871614845,
+ "loss": 1.506,
+ "theoretical_loss": 3.9789577598933588,
+ "tokens_seen": 459931648
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954463390170511,
+ "loss": 1.4603,
+ "theoretical_loss": 3.9788962598183795,
+ "tokens_seen": 459997184
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954302908726179,
+ "loss": 1.5106,
+ "theoretical_loss": 3.978834770957665,
+ "tokens_seen": 460062720
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006954142427281846,
+ "loss": 1.5574,
+ "theoretical_loss": 3.9787732933075715,
+ "tokens_seen": 460128256
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953981945837513,
+ "loss": 1.6217,
+ "theoretical_loss": 3.97871182686446,
+ "tokens_seen": 460193792
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000695382146439318,
+ "loss": 1.4491,
+ "theoretical_loss": 3.978650371624692,
+ "tokens_seen": 460259328
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953660982948847,
+ "loss": 1.5272,
+ "theoretical_loss": 3.9785889275846307,
+ "tokens_seen": 460324864
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953500501504514,
+ "loss": 1.5238,
+ "theoretical_loss": 3.978527494740641,
+ "tokens_seen": 460390400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953340020060181,
+ "loss": 1.5595,
+ "theoretical_loss": 3.9784660730890895,
+ "tokens_seen": 460455936
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953179538615848,
+ "loss": 1.5279,
+ "theoretical_loss": 3.978404662626344,
+ "tokens_seen": 460521472
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006953019057171515,
+ "loss": 1.5194,
+ "theoretical_loss": 3.9783432633487745,
+ "tokens_seen": 460587008
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006952858575727182,
+ "loss": 1.5553,
+ "theoretical_loss": 3.9782818752527533,
+ "tokens_seen": 460652544
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006952698094282849,
+ "loss": 1.481,
+ "theoretical_loss": 3.978220498334654,
+ "tokens_seen": 460718080
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006952537612838516,
+ "loss": 1.5989,
+ "theoretical_loss": 3.97815913259085,
+ "tokens_seen": 460783616
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006952377131394183,
+ "loss": 1.4864,
+ "theoretical_loss": 3.9780977780177205,
+ "tokens_seen": 460849152
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000695221664994985,
+ "loss": 1.535,
+ "theoretical_loss": 3.978036434611642,
+ "tokens_seen": 460914688
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006952056168505517,
+ "loss": 1.4559,
+ "theoretical_loss": 3.977975102368995,
+ "tokens_seen": 460980224
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951895687061184,
+ "loss": 1.5792,
+ "theoretical_loss": 3.977913781286162,
+ "tokens_seen": 461045760
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951735205616851,
+ "loss": 1.5258,
+ "theoretical_loss": 3.9778524713595256,
+ "tokens_seen": 461111296
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951574724172518,
+ "loss": 1.4682,
+ "theoretical_loss": 3.977791172585472,
+ "tokens_seen": 461176832
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951414242728185,
+ "loss": 1.4718,
+ "theoretical_loss": 3.977729884960387,
+ "tokens_seen": 461242368
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951253761283852,
+ "loss": 1.5455,
+ "theoretical_loss": 3.9776686084806596,
+ "tokens_seen": 461307904
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006951093279839519,
+ "loss": 1.4778,
+ "theoretical_loss": 3.9776073431426804,
+ "tokens_seen": 461373440
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006950932798395186,
+ "loss": 1.5595,
+ "theoretical_loss": 3.977546088942841,
+ "tokens_seen": 461438976
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006950772316950853,
+ "loss": 1.5265,
+ "theoretical_loss": 3.9774848458775347,
+ "tokens_seen": 461504512
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000695061183550652,
+ "loss": 1.5641,
+ "theoretical_loss": 3.9774236139431576,
+ "tokens_seen": 461570048
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006950451354062187,
+ "loss": 1.4548,
+ "theoretical_loss": 3.977362393136106,
+ "tokens_seen": 461635584
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006950290872617854,
+ "loss": 1.4131,
+ "theoretical_loss": 3.977301183452779,
+ "tokens_seen": 461701120
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006950130391173521,
+ "loss": 1.4352,
+ "theoretical_loss": 3.977239984889576,
+ "tokens_seen": 461766656
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949969909729188,
+ "loss": 1.4112,
+ "theoretical_loss": 3.9771787974429,
+ "tokens_seen": 461832192
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949809428284855,
+ "loss": 1.443,
+ "theoretical_loss": 3.977117621109154,
+ "tokens_seen": 461897728
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949648946840522,
+ "loss": 1.4848,
+ "theoretical_loss": 3.977056455884744,
+ "tokens_seen": 461963264
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 159827,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.441860318183899,
+ "objective/train/theoretical_loss": 3.9769953017660766,
+ "objective/train/tokens_used": 482488800,
+ "theoretical_loss": 3.9769953017660766,
+ "tokens_seen": 462028800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949488465396189,
+ "loss": 1.4389,
+ "theoretical_loss": 3.9769953017660766,
+ "tokens_seen": 462028800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949327983951856,
+ "loss": 1.4872,
+ "theoretical_loss": 3.97693415874956,
+ "tokens_seen": 462094336
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006949167502507523,
+ "loss": 1.4321,
+ "theoretical_loss": 3.9768730268316057,
+ "tokens_seen": 462159872
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694900702106319,
+ "loss": 1.4834,
+ "theoretical_loss": 3.976811906008625,
+ "tokens_seen": 462225408
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948846539618857,
+ "loss": 1.4452,
+ "theoretical_loss": 3.976750796277031,
+ "tokens_seen": 462290944
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948686058174524,
+ "loss": 1.6206,
+ "theoretical_loss": 3.9766896976332404,
+ "tokens_seen": 462356480
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948525576730191,
+ "loss": 1.493,
+ "theoretical_loss": 3.9766286100736696,
+ "tokens_seen": 462422016
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948365095285858,
+ "loss": 1.479,
+ "theoretical_loss": 3.976567533594737,
+ "tokens_seen": 462487552
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948204613841525,
+ "loss": 1.5357,
+ "theoretical_loss": 3.9765064681928637,
+ "tokens_seen": 462553088
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006948044132397192,
+ "loss": 1.464,
+ "theoretical_loss": 3.9764454138644703,
+ "tokens_seen": 462618624
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006947883650952859,
+ "loss": 1.469,
+ "theoretical_loss": 3.976384370605982,
+ "tokens_seen": 462684160
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006947723169508526,
+ "loss": 1.4872,
+ "theoretical_loss": 3.9763233384138235,
+ "tokens_seen": 462749696
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006947562688064193,
+ "loss": 1.5855,
+ "theoretical_loss": 3.976262317284421,
+ "tokens_seen": 462815232
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694740220661986,
+ "loss": 1.5168,
+ "theoretical_loss": 3.976201307214205,
+ "tokens_seen": 462880768
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006947241725175527,
+ "loss": 1.5409,
+ "theoretical_loss": 3.9761403081996045,
+ "tokens_seen": 462946304
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006947081243731194,
+ "loss": 1.5067,
+ "theoretical_loss": 3.9760793202370524,
+ "tokens_seen": 463011840
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946920762286861,
+ "loss": 1.6068,
+ "theoretical_loss": 3.976018343322981,
+ "tokens_seen": 463077376
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946760280842528,
+ "loss": 1.4944,
+ "theoretical_loss": 3.975957377453826,
+ "tokens_seen": 463142912
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946599799398195,
+ "loss": 1.4191,
+ "theoretical_loss": 3.9758964226260254,
+ "tokens_seen": 463208448
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946439317953862,
+ "loss": 1.5305,
+ "theoretical_loss": 3.975835478836016,
+ "tokens_seen": 463273984
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946278836509529,
+ "loss": 1.581,
+ "theoretical_loss": 3.97577454608024,
+ "tokens_seen": 463339520
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006946118355065196,
+ "loss": 1.5766,
+ "theoretical_loss": 3.975713624355139,
+ "tokens_seen": 463405056
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006945957873620863,
+ "loss": 1.4439,
+ "theoretical_loss": 3.975652713657155,
+ "tokens_seen": 463470592
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694579739217653,
+ "loss": 1.5426,
+ "theoretical_loss": 3.975591813982734,
+ "tokens_seen": 463536128
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006945636910732197,
+ "loss": 1.4779,
+ "theoretical_loss": 3.9755309253283233,
+ "tokens_seen": 463601664
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006945476429287864,
+ "loss": 1.5188,
+ "theoretical_loss": 3.9754700476903713,
+ "tokens_seen": 463667200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006945315947843531,
+ "loss": 1.4893,
+ "theoretical_loss": 3.975409181065328,
+ "tokens_seen": 463732736
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006945155466399198,
+ "loss": 1.5916,
+ "theoretical_loss": 3.9753483254496444,
+ "tokens_seen": 463798272
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944994984954865,
+ "loss": 1.5098,
+ "theoretical_loss": 3.9752874808397753,
+ "tokens_seen": 463863808
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944834503510532,
+ "loss": 1.5938,
+ "theoretical_loss": 3.975226647232175,
+ "tokens_seen": 463929344
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944674022066199,
+ "loss": 1.5668,
+ "theoretical_loss": 3.9751658246233,
+ "tokens_seen": 463994880
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944513540621866,
+ "loss": 1.5402,
+ "theoretical_loss": 3.9751050130096095,
+ "tokens_seen": 464060416
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944353059177533,
+ "loss": 1.521,
+ "theoretical_loss": 3.9750442123875622,
+ "tokens_seen": 464125952
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.00069441925777332,
+ "loss": 1.5605,
+ "theoretical_loss": 3.9749834227536214,
+ "tokens_seen": 464191488
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006944032096288867,
+ "loss": 1.441,
+ "theoretical_loss": 3.9749226441042493,
+ "tokens_seen": 464257024
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006943871614844534,
+ "loss": 1.5675,
+ "theoretical_loss": 3.9748618764359107,
+ "tokens_seen": 464322560
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006943711133400201,
+ "loss": 1.5599,
+ "theoretical_loss": 3.9748011197450728,
+ "tokens_seen": 464388096
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006943550651955868,
+ "loss": 1.6316,
+ "theoretical_loss": 3.974740374028203,
+ "tokens_seen": 464453632
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006943390170511535,
+ "loss": 1.4587,
+ "theoretical_loss": 3.9746796392817716,
+ "tokens_seen": 464519168
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006943229689067202,
+ "loss": 1.4971,
+ "theoretical_loss": 3.9746189155022504,
+ "tokens_seen": 464584704
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694306920762287,
+ "loss": 1.5041,
+ "theoretical_loss": 3.9745582026861124,
+ "tokens_seen": 464650240
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006942908726178536,
+ "loss": 1.6098,
+ "theoretical_loss": 3.9744975008298313,
+ "tokens_seen": 464715776
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006942748244734202,
+ "loss": 1.5333,
+ "theoretical_loss": 3.974436809929885,
+ "tokens_seen": 464781312
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694258776328987,
+ "loss": 1.5918,
+ "theoretical_loss": 3.9743761299827494,
+ "tokens_seen": 464846848
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006942427281845536,
+ "loss": 1.4116,
+ "theoretical_loss": 3.974315460984906,
+ "tokens_seen": 464912384
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006942266800401204,
+ "loss": 1.5304,
+ "theoretical_loss": 3.974254802932836,
+ "tokens_seen": 464977920
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694210631895687,
+ "loss": 1.5563,
+ "theoretical_loss": 3.974194155823021,
+ "tokens_seen": 465043456
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941945837512538,
+ "loss": 1.5092,
+ "theoretical_loss": 3.9741335196519465,
+ "tokens_seen": 465108992
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941785356068204,
+ "loss": 1.5468,
+ "theoretical_loss": 3.9740728944160977,
+ "tokens_seen": 465174528
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941624874623872,
+ "loss": 1.4083,
+ "theoretical_loss": 3.974012280111963,
+ "tokens_seen": 465240064
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 161129,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5128990411758423,
+ "objective/train/theoretical_loss": 3.973951676736032,
+ "objective/train/tokens_used": 485765600,
+ "theoretical_loss": 3.973951676736032,
+ "tokens_seen": 465305600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941464393179538,
+ "loss": 1.4757,
+ "theoretical_loss": 3.973951676736032,
+ "tokens_seen": 465305600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941303911735206,
+ "loss": 1.6992,
+ "theoretical_loss": 3.973891084284795,
+ "tokens_seen": 465371136
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006941143430290872,
+ "loss": 1.4481,
+ "theoretical_loss": 3.973830502754745,
+ "tokens_seen": 465436672
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694098294884654,
+ "loss": 1.3695,
+ "theoretical_loss": 3.973769932142376,
+ "tokens_seen": 465502208
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006940822467402206,
+ "loss": 1.4866,
+ "theoretical_loss": 3.9737093724441843,
+ "tokens_seen": 465567744
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006940661985957874,
+ "loss": 1.6144,
+ "theoretical_loss": 3.9736488236566667,
+ "tokens_seen": 465633280
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000694050150451354,
+ "loss": 1.494,
+ "theoretical_loss": 3.973588285776323,
+ "tokens_seen": 465698816
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006940341023069208,
+ "loss": 1.3829,
+ "theoretical_loss": 3.9735277587996527,
+ "tokens_seen": 465764352
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006940180541624875,
+ "loss": 1.511,
+ "theoretical_loss": 3.973467242723159,
+ "tokens_seen": 465829888
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006940020060180542,
+ "loss": 1.5361,
+ "theoretical_loss": 3.9734067375433466,
+ "tokens_seen": 465895424
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006939859578736209,
+ "loss": 1.4227,
+ "theoretical_loss": 3.9733462432567195,
+ "tokens_seen": 465960960
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006939699097291876,
+ "loss": 1.5223,
+ "theoretical_loss": 3.9732857598597855,
+ "tokens_seen": 466026496
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006939538615847543,
+ "loss": 1.587,
+ "theoretical_loss": 3.9732252873490532,
+ "tokens_seen": 466092032
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693937813440321,
+ "loss": 1.4361,
+ "theoretical_loss": 3.9731648257210335,
+ "tokens_seen": 466157568
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006939217652958877,
+ "loss": 1.5038,
+ "theoretical_loss": 3.9731043749722375,
+ "tokens_seen": 466223104
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006939057171514544,
+ "loss": 1.5343,
+ "theoretical_loss": 3.9730439350991795,
+ "tokens_seen": 466288640
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938896690070211,
+ "loss": 1.5162,
+ "theoretical_loss": 3.9729835060983745,
+ "tokens_seen": 466354176
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938736208625878,
+ "loss": 1.4384,
+ "theoretical_loss": 3.972923087966339,
+ "tokens_seen": 466419712
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938575727181545,
+ "loss": 1.5003,
+ "theoretical_loss": 3.9728626806995915,
+ "tokens_seen": 466485248
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938415245737212,
+ "loss": 1.5118,
+ "theoretical_loss": 3.9728022842946524,
+ "tokens_seen": 466550784
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938254764292879,
+ "loss": 1.518,
+ "theoretical_loss": 3.972741898748043,
+ "tokens_seen": 466616320
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006938094282848546,
+ "loss": 1.5349,
+ "theoretical_loss": 3.972681524056286,
+ "tokens_seen": 466681856
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006937933801404213,
+ "loss": 1.6516,
+ "theoretical_loss": 3.9726211602159074,
+ "tokens_seen": 466747392
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693777331995988,
+ "loss": 1.5143,
+ "theoretical_loss": 3.9725608072234326,
+ "tokens_seen": 466812928
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006937612838515547,
+ "loss": 1.5186,
+ "theoretical_loss": 3.97250046507539,
+ "tokens_seen": 466878464
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006937452357071214,
+ "loss": 1.4036,
+ "theoretical_loss": 3.972440133768309,
+ "tokens_seen": 466944000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006937291875626881,
+ "loss": 1.4624,
+ "theoretical_loss": 3.972379813298721,
+ "tokens_seen": 467009536
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006937131394182548,
+ "loss": 1.4537,
+ "theoretical_loss": 3.972319503663159,
+ "tokens_seen": 467075072
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936970912738215,
+ "loss": 1.4609,
+ "theoretical_loss": 3.972259204858157,
+ "tokens_seen": 467140608
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936810431293882,
+ "loss": 1.5882,
+ "theoretical_loss": 3.9721989168802505,
+ "tokens_seen": 467206144
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936649949849549,
+ "loss": 1.5135,
+ "theoretical_loss": 3.9721386397259786,
+ "tokens_seen": 467271680
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936489468405216,
+ "loss": 1.6007,
+ "theoretical_loss": 3.972078373391879,
+ "tokens_seen": 467337216
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936328986960883,
+ "loss": 1.3934,
+ "theoretical_loss": 3.9720181178744935,
+ "tokens_seen": 467402752
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693616850551655,
+ "loss": 1.4687,
+ "theoretical_loss": 3.971957873170364,
+ "tokens_seen": 467468288
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006936008024072217,
+ "loss": 1.4979,
+ "theoretical_loss": 3.971897639276035,
+ "tokens_seen": 467533824
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935847542627884,
+ "loss": 1.4511,
+ "theoretical_loss": 3.9718374161880505,
+ "tokens_seen": 467599360
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935687061183551,
+ "loss": 1.396,
+ "theoretical_loss": 3.971777203902959,
+ "tokens_seen": 467664896
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935526579739218,
+ "loss": 1.5844,
+ "theoretical_loss": 3.971717002417309,
+ "tokens_seen": 467730432
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935366098294885,
+ "loss": 1.6607,
+ "theoretical_loss": 3.9716568117276503,
+ "tokens_seen": 467795968
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935205616850552,
+ "loss": 1.5169,
+ "theoretical_loss": 3.9715966318305354,
+ "tokens_seen": 467861504
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006935045135406219,
+ "loss": 1.4717,
+ "theoretical_loss": 3.9715364627225176,
+ "tokens_seen": 467927040
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006934884653961886,
+ "loss": 1.3652,
+ "theoretical_loss": 3.9714763044001513,
+ "tokens_seen": 467992576
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006934724172517553,
+ "loss": 1.5736,
+ "theoretical_loss": 3.971416156859995,
+ "tokens_seen": 468058112
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693456369107322,
+ "loss": 1.4701,
+ "theoretical_loss": 3.9713560200986042,
+ "tokens_seen": 468123648
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006934403209628887,
+ "loss": 1.4428,
+ "theoretical_loss": 3.9712958941125405,
+ "tokens_seen": 468189184
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006934242728184554,
+ "loss": 1.6038,
+ "theoretical_loss": 3.971235778898365,
+ "tokens_seen": 468254720
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006934082246740221,
+ "loss": 1.5205,
+ "theoretical_loss": 3.971175674452641,
+ "tokens_seen": 468320256
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933921765295888,
+ "loss": 1.5535,
+ "theoretical_loss": 3.9711155807719316,
+ "tokens_seen": 468385792
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933761283851555,
+ "loss": 1.5537,
+ "theoretical_loss": 3.971055497852804,
+ "tokens_seen": 468451328
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933600802407222,
+ "loss": 1.4948,
+ "theoretical_loss": 3.9709954256918265,
+ "tokens_seen": 468516864
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 161996,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3592880964279175,
+ "objective/train/theoretical_loss": 3.970935364285567,
+ "objective/train/tokens_used": 489042400,
+ "theoretical_loss": 3.970935364285567,
+ "tokens_seen": 468582400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933440320962889,
+ "loss": 1.4138,
+ "theoretical_loss": 3.970935364285567,
+ "tokens_seen": 468582400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933279839518556,
+ "loss": 1.438,
+ "theoretical_loss": 3.9708753136305974,
+ "tokens_seen": 468647936
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006933119358074223,
+ "loss": 1.4692,
+ "theoretical_loss": 3.9708152737234887,
+ "tokens_seen": 468713472
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693295887662989,
+ "loss": 1.4952,
+ "theoretical_loss": 3.9707552445608165,
+ "tokens_seen": 468779008
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006932798395185557,
+ "loss": 1.482,
+ "theoretical_loss": 3.9706952261391555,
+ "tokens_seen": 468844544
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006932637913741224,
+ "loss": 1.4899,
+ "theoretical_loss": 3.970635218455083,
+ "tokens_seen": 468910080
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006932477432296891,
+ "loss": 1.4871,
+ "theoretical_loss": 3.970575221505178,
+ "tokens_seen": 468975616
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006932316950852558,
+ "loss": 1.5286,
+ "theoretical_loss": 3.97051523528602,
+ "tokens_seen": 469041152
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006932156469408225,
+ "loss": 1.5471,
+ "theoretical_loss": 3.970455259794191,
+ "tokens_seen": 469106688
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931995987963892,
+ "loss": 1.5182,
+ "theoretical_loss": 3.970395295026275,
+ "tokens_seen": 469172224
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931835506519559,
+ "loss": 1.6327,
+ "theoretical_loss": 3.970335340978856,
+ "tokens_seen": 469237760
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931675025075226,
+ "loss": 1.4887,
+ "theoretical_loss": 3.970275397648522,
+ "tokens_seen": 469303296
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931514543630893,
+ "loss": 1.5198,
+ "theoretical_loss": 3.9702154650318597,
+ "tokens_seen": 469368832
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000693135406218656,
+ "loss": 1.5937,
+ "theoretical_loss": 3.9701555431254594,
+ "tokens_seen": 469434368
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931193580742227,
+ "loss": 1.4071,
+ "theoretical_loss": 3.9700956319259113,
+ "tokens_seen": 469499904
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006931033099297894,
+ "loss": 1.4972,
+ "theoretical_loss": 3.97003573142981,
+ "tokens_seen": 469565440
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930872617853562,
+ "loss": 1.5,
+ "theoretical_loss": 3.9699758416337483,
+ "tokens_seen": 469630976
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930712136409228,
+ "loss": 1.561,
+ "theoretical_loss": 3.9699159625343223,
+ "tokens_seen": 469696512
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930551654964896,
+ "loss": 1.3793,
+ "theoretical_loss": 3.96985609412813,
+ "tokens_seen": 469762048
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930391173520562,
+ "loss": 1.534,
+ "theoretical_loss": 3.96979623641177,
+ "tokens_seen": 469827584
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930230692076229,
+ "loss": 1.5069,
+ "theoretical_loss": 3.9697363893818434,
+ "tokens_seen": 469893120
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006930070210631896,
+ "loss": 1.503,
+ "theoretical_loss": 3.969676553034951,
+ "tokens_seen": 469958656
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006929909729187563,
+ "loss": 1.5206,
+ "theoretical_loss": 3.969616727367698,
+ "tokens_seen": 470024192
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692974924774323,
+ "loss": 1.5285,
+ "theoretical_loss": 3.9695569123766887,
+ "tokens_seen": 470089728
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006929588766298897,
+ "loss": 1.4659,
+ "theoretical_loss": 3.9694971080585297,
+ "tokens_seen": 470155264
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006929428284854564,
+ "loss": 1.4967,
+ "theoretical_loss": 3.9694373144098307,
+ "tokens_seen": 470220800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006929267803410231,
+ "loss": 1.514,
+ "theoretical_loss": 3.9693775314271997,
+ "tokens_seen": 470286336
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006929107321965898,
+ "loss": 1.4917,
+ "theoretical_loss": 3.9693177591072493,
+ "tokens_seen": 470351872
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928946840521565,
+ "loss": 1.4453,
+ "theoretical_loss": 3.9692579974465922,
+ "tokens_seen": 470417408
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928786359077231,
+ "loss": 1.4894,
+ "theoretical_loss": 3.9691982464418434,
+ "tokens_seen": 470482944
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928625877632899,
+ "loss": 1.5014,
+ "theoretical_loss": 3.9691385060896174,
+ "tokens_seen": 470548480
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928465396188565,
+ "loss": 1.5743,
+ "theoretical_loss": 3.9690787763865334,
+ "tokens_seen": 470614016
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928304914744233,
+ "loss": 1.5716,
+ "theoretical_loss": 3.96901905732921,
+ "tokens_seen": 470679552
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006928144433299899,
+ "loss": 1.5425,
+ "theoretical_loss": 3.968959348914268,
+ "tokens_seen": 470745088
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927983951855567,
+ "loss": 1.482,
+ "theoretical_loss": 3.9688996511383294,
+ "tokens_seen": 470810624
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927823470411233,
+ "loss": 1.4986,
+ "theoretical_loss": 3.968839963998018,
+ "tokens_seen": 470876160
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927662988966901,
+ "loss": 1.4271,
+ "theoretical_loss": 3.9687802874899596,
+ "tokens_seen": 470941696
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927502507522567,
+ "loss": 1.4772,
+ "theoretical_loss": 3.9687206216107804,
+ "tokens_seen": 471007232
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927342026078235,
+ "loss": 1.5334,
+ "theoretical_loss": 3.968660966357109,
+ "tokens_seen": 471072768
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927181544633901,
+ "loss": 1.5458,
+ "theoretical_loss": 3.9686013217255756,
+ "tokens_seen": 471138304
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006927021063189569,
+ "loss": 1.4409,
+ "theoretical_loss": 3.9685416877128112,
+ "tokens_seen": 471203840
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006926860581745235,
+ "loss": 1.5034,
+ "theoretical_loss": 3.9684820643154497,
+ "tokens_seen": 471269376
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006926700100300903,
+ "loss": 1.4848,
+ "theoretical_loss": 3.9684224515301247,
+ "tokens_seen": 471334912
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692653961885657,
+ "loss": 1.4413,
+ "theoretical_loss": 3.968362849353473,
+ "tokens_seen": 471400448
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006926379137412237,
+ "loss": 1.543,
+ "theoretical_loss": 3.9683032577821313,
+ "tokens_seen": 471465984
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006926218655967904,
+ "loss": 1.4091,
+ "theoretical_loss": 3.9682436768127394,
+ "tokens_seen": 471531520
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006926058174523571,
+ "loss": 1.5432,
+ "theoretical_loss": 3.968184106441938,
+ "tokens_seen": 471597056
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925897693079238,
+ "loss": 1.5376,
+ "theoretical_loss": 3.9681245466663695,
+ "tokens_seen": 471662592
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925737211634905,
+ "loss": 1.4423,
+ "theoretical_loss": 3.9680649974826774,
+ "tokens_seen": 471728128
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925576730190572,
+ "loss": 1.4998,
+ "theoretical_loss": 3.9680054588875064,
+ "tokens_seen": 471793664
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 163162,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4037538766860962,
+ "objective/train/theoretical_loss": 3.9679459308775042,
+ "objective/train/tokens_used": 492319200,
+ "theoretical_loss": 3.9679459308775042,
+ "tokens_seen": 471859200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925416248746239,
+ "loss": 1.4952,
+ "theoretical_loss": 3.9679459308775042,
+ "tokens_seen": 471859200
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925255767301906,
+ "loss": 1.5488,
+ "theoretical_loss": 3.9678864134493184,
+ "tokens_seen": 471924736
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006925095285857573,
+ "loss": 1.5842,
+ "theoretical_loss": 3.9678269065996,
+ "tokens_seen": 471990272
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692493480441324,
+ "loss": 1.5198,
+ "theoretical_loss": 3.967767410324999,
+ "tokens_seen": 472055808
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006924774322968907,
+ "loss": 1.4667,
+ "theoretical_loss": 3.967707924622169,
+ "tokens_seen": 472121344
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006924613841524574,
+ "loss": 1.5051,
+ "theoretical_loss": 3.967648449487764,
+ "tokens_seen": 472186880
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006924453360080241,
+ "loss": 1.5073,
+ "theoretical_loss": 3.9675889849184407,
+ "tokens_seen": 472252416
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006924292878635908,
+ "loss": 1.5508,
+ "theoretical_loss": 3.9675295309108556,
+ "tokens_seen": 472317952
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006924132397191575,
+ "loss": 1.4599,
+ "theoretical_loss": 3.9674700874616686,
+ "tokens_seen": 472383488
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923971915747242,
+ "loss": 1.6051,
+ "theoretical_loss": 3.9674106545675403,
+ "tokens_seen": 472449024
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923811434302909,
+ "loss": 1.4511,
+ "theoretical_loss": 3.967351232225132,
+ "tokens_seen": 472514560
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923650952858576,
+ "loss": 1.4974,
+ "theoretical_loss": 3.9672918204311074,
+ "tokens_seen": 472580096
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923490471414243,
+ "loss": 1.5511,
+ "theoretical_loss": 3.9672324191821318,
+ "tokens_seen": 472645632
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692332998996991,
+ "loss": 1.6074,
+ "theoretical_loss": 3.967173028474871,
+ "tokens_seen": 472711168
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923169508525577,
+ "loss": 1.6054,
+ "theoretical_loss": 3.967113648305995,
+ "tokens_seen": 472776704
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006923009027081244,
+ "loss": 1.5523,
+ "theoretical_loss": 3.9670542786721716,
+ "tokens_seen": 472842240
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922848545636911,
+ "loss": 1.4466,
+ "theoretical_loss": 3.9669949195700727,
+ "tokens_seen": 472907776
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922688064192578,
+ "loss": 1.5414,
+ "theoretical_loss": 3.9669355709963705,
+ "tokens_seen": 472973312
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922527582748245,
+ "loss": 1.4645,
+ "theoretical_loss": 3.9668762329477403,
+ "tokens_seen": 473038848
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922367101303912,
+ "loss": 1.4129,
+ "theoretical_loss": 3.966816905420856,
+ "tokens_seen": 473104384
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922206619859579,
+ "loss": 1.5528,
+ "theoretical_loss": 3.966757588412396,
+ "tokens_seen": 473169920
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006922046138415246,
+ "loss": 1.6118,
+ "theoretical_loss": 3.9666982819190384,
+ "tokens_seen": 473235456
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006921885656970913,
+ "loss": 1.4257,
+ "theoretical_loss": 3.9666389859374642,
+ "tokens_seen": 473300992
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692172517552658,
+ "loss": 1.5467,
+ "theoretical_loss": 3.9665797004643544,
+ "tokens_seen": 473366528
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006921564694082247,
+ "loss": 1.5235,
+ "theoretical_loss": 3.966520425496392,
+ "tokens_seen": 473432064
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006921404212637914,
+ "loss": 1.4722,
+ "theoretical_loss": 3.9664611610302627,
+ "tokens_seen": 473497600
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006921243731193581,
+ "loss": 1.3878,
+ "theoretical_loss": 3.966401907062652,
+ "tokens_seen": 473563136
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006921083249749248,
+ "loss": 1.5362,
+ "theoretical_loss": 3.9663426635902477,
+ "tokens_seen": 473628672
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006920922768304915,
+ "loss": 1.6144,
+ "theoretical_loss": 3.9662834306097388,
+ "tokens_seen": 473694208
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006920762286860582,
+ "loss": 1.4815,
+ "theoretical_loss": 3.9662242081178167,
+ "tokens_seen": 473759744
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006920601805416249,
+ "loss": 1.508,
+ "theoretical_loss": 3.9661649961111727,
+ "tokens_seen": 473825280
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006920441323971916,
+ "loss": 1.4545,
+ "theoretical_loss": 3.9661057945865017,
+ "tokens_seen": 473890816
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006920280842527583,
+ "loss": 1.565,
+ "theoretical_loss": 3.966046603540498,
+ "tokens_seen": 473956352
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000692012036108325,
+ "loss": 1.4851,
+ "theoretical_loss": 3.9659874229698584,
+ "tokens_seen": 474021888
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919959879638917,
+ "loss": 1.4032,
+ "theoretical_loss": 3.9659282528712816,
+ "tokens_seen": 474087424
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919799398194584,
+ "loss": 1.4323,
+ "theoretical_loss": 3.965869093241467,
+ "tokens_seen": 474152960
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919638916750251,
+ "loss": 1.519,
+ "theoretical_loss": 3.965809944077116,
+ "tokens_seen": 474218496
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919478435305918,
+ "loss": 1.4708,
+ "theoretical_loss": 3.965750805374931,
+ "tokens_seen": 474284032
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919317953861585,
+ "loss": 1.4264,
+ "theoretical_loss": 3.965691677131617,
+ "tokens_seen": 474349568
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006919157472417252,
+ "loss": 1.5769,
+ "theoretical_loss": 3.9656325593438786,
+ "tokens_seen": 474415104
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918996990972919,
+ "loss": 1.4914,
+ "theoretical_loss": 3.9655734520084236,
+ "tokens_seen": 474480640
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918836509528587,
+ "loss": 1.5013,
+ "theoretical_loss": 3.965514355121961,
+ "tokens_seen": 474546176
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918676028084253,
+ "loss": 1.5206,
+ "theoretical_loss": 3.9654552686812004,
+ "tokens_seen": 474611712
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918515546639921,
+ "loss": 1.452,
+ "theoretical_loss": 3.9653961926828534,
+ "tokens_seen": 474677248
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918355065195587,
+ "loss": 1.5568,
+ "theoretical_loss": 3.9653371271236337,
+ "tokens_seen": 474742784
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918194583751255,
+ "loss": 1.576,
+ "theoretical_loss": 3.9652780720002556,
+ "tokens_seen": 474808320
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006918034102306921,
+ "loss": 1.544,
+ "theoretical_loss": 3.965219027309435,
+ "tokens_seen": 474873856
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917873620862589,
+ "loss": 1.5643,
+ "theoretical_loss": 3.9651599930478905,
+ "tokens_seen": 474939392
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917713139418255,
+ "loss": 1.4463,
+ "theoretical_loss": 3.9651009692123402,
+ "tokens_seen": 475004928
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917552657973923,
+ "loss": 1.5332,
+ "theoretical_loss": 3.9650419557995047,
+ "tokens_seen": 475070464
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 164144,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6853489875793457,
+ "objective/train/theoretical_loss": 3.964982952806107,
+ "objective/train/tokens_used": 495596000,
+ "theoretical_loss": 3.964982952806107,
+ "tokens_seen": 475136000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917392176529589,
+ "loss": 1.5972,
+ "theoretical_loss": 3.964982952806107,
+ "tokens_seen": 475136000
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917231695085257,
+ "loss": 1.4829,
+ "theoretical_loss": 3.9649239602288695,
+ "tokens_seen": 475201536
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006917071213640923,
+ "loss": 1.3827,
+ "theoretical_loss": 3.9648649780645178,
+ "tokens_seen": 475267072
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916910732196591,
+ "loss": 1.4239,
+ "theoretical_loss": 3.9648060063097788,
+ "tokens_seen": 475332608
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916750250752257,
+ "loss": 1.4822,
+ "theoretical_loss": 3.9647470449613795,
+ "tokens_seen": 475398144
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916589769307925,
+ "loss": 1.4893,
+ "theoretical_loss": 3.9646880940160507,
+ "tokens_seen": 475463680
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916429287863591,
+ "loss": 1.4965,
+ "theoretical_loss": 3.9646291534705225,
+ "tokens_seen": 475529216
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916268806419258,
+ "loss": 1.5566,
+ "theoretical_loss": 3.9645702233215268,
+ "tokens_seen": 475594752
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006916108324974925,
+ "loss": 1.4053,
+ "theoretical_loss": 3.964511303565799,
+ "tokens_seen": 475660288
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006915947843530592,
+ "loss": 1.5627,
+ "theoretical_loss": 3.964452394200073,
+ "tokens_seen": 475725824
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691578736208626,
+ "loss": 1.4794,
+ "theoretical_loss": 3.9643934952210866,
+ "tokens_seen": 475791360
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006915626880641926,
+ "loss": 1.484,
+ "theoretical_loss": 3.9643346066255782,
+ "tokens_seen": 475856896
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006915466399197593,
+ "loss": 1.5084,
+ "theoretical_loss": 3.964275728410287,
+ "tokens_seen": 475922432
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691530591775326,
+ "loss": 1.4504,
+ "theoretical_loss": 3.9642168605719545,
+ "tokens_seen": 475987968
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006915145436308928,
+ "loss": 1.5379,
+ "theoretical_loss": 3.9641580031073236,
+ "tokens_seen": 476053504
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914984954864594,
+ "loss": 1.5919,
+ "theoretical_loss": 3.9640991560131384,
+ "tokens_seen": 476119040
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691482447342026,
+ "loss": 1.4496,
+ "theoretical_loss": 3.964040319286145,
+ "tokens_seen": 476184576
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914663991975928,
+ "loss": 1.4655,
+ "theoretical_loss": 3.9639814929230894,
+ "tokens_seen": 476250112
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914503510531594,
+ "loss": 1.5148,
+ "theoretical_loss": 3.9639226769207214,
+ "tokens_seen": 476315648
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914343029087262,
+ "loss": 1.4979,
+ "theoretical_loss": 3.963863871275791,
+ "tokens_seen": 476381184
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914182547642928,
+ "loss": 1.5029,
+ "theoretical_loss": 3.963805075985049,
+ "tokens_seen": 476446720
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006914022066198596,
+ "loss": 1.5035,
+ "theoretical_loss": 3.9637462910452497,
+ "tokens_seen": 476512256
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006913861584754262,
+ "loss": 1.6175,
+ "theoretical_loss": 3.963687516453146,
+ "tokens_seen": 476577792
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691370110330993,
+ "loss": 1.4935,
+ "theoretical_loss": 3.9636287522054947,
+ "tokens_seen": 476643328
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006913540621865596,
+ "loss": 1.5319,
+ "theoretical_loss": 3.9635699982990538,
+ "tokens_seen": 476708864
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006913380140421264,
+ "loss": 1.3645,
+ "theoretical_loss": 3.963511254730581,
+ "tokens_seen": 476774400
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691321965897693,
+ "loss": 1.4714,
+ "theoretical_loss": 3.9634525214968375,
+ "tokens_seen": 476839936
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006913059177532598,
+ "loss": 1.4609,
+ "theoretical_loss": 3.963393798594585,
+ "tokens_seen": 476905472
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006912898696088265,
+ "loss": 1.5017,
+ "theoretical_loss": 3.963335086020586,
+ "tokens_seen": 476971008
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006912738214643932,
+ "loss": 1.5675,
+ "theoretical_loss": 3.9632763837716065,
+ "tokens_seen": 477036544
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006912577733199599,
+ "loss": 1.5274,
+ "theoretical_loss": 3.9632176918444113,
+ "tokens_seen": 477102080
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006912417251755266,
+ "loss": 1.4453,
+ "theoretical_loss": 3.9631590102357697,
+ "tokens_seen": 477167616
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006912256770310933,
+ "loss": 1.491,
+ "theoretical_loss": 3.963100338942449,
+ "tokens_seen": 477233152
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.00069120962888666,
+ "loss": 1.4311,
+ "theoretical_loss": 3.963041677961221,
+ "tokens_seen": 477298688
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911935807422267,
+ "loss": 1.5531,
+ "theoretical_loss": 3.962983027288857,
+ "tokens_seen": 477364224
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911775325977934,
+ "loss": 1.4859,
+ "theoretical_loss": 3.9629243869221304,
+ "tokens_seen": 477429760
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911614844533601,
+ "loss": 1.5019,
+ "theoretical_loss": 3.962865756857817,
+ "tokens_seen": 477495296
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911454363089268,
+ "loss": 1.5433,
+ "theoretical_loss": 3.9628071370926925,
+ "tokens_seen": 477560832
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911293881644935,
+ "loss": 1.5089,
+ "theoretical_loss": 3.9627485276235346,
+ "tokens_seen": 477626368
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006911133400200602,
+ "loss": 1.5039,
+ "theoretical_loss": 3.962689928447123,
+ "tokens_seen": 477691904
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910972918756269,
+ "loss": 1.6092,
+ "theoretical_loss": 3.962631339560238,
+ "tokens_seen": 477757440
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910812437311936,
+ "loss": 1.5068,
+ "theoretical_loss": 3.962572760959662,
+ "tokens_seen": 477822976
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910651955867603,
+ "loss": 1.4194,
+ "theoretical_loss": 3.9625141926421783,
+ "tokens_seen": 477888512
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.000691049147442327,
+ "loss": 1.5435,
+ "theoretical_loss": 3.9624556346045727,
+ "tokens_seen": 477954048
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910330992978937,
+ "loss": 1.4924,
+ "theoretical_loss": 3.962397086843631,
+ "tokens_seen": 478019584
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910170511534604,
+ "loss": 1.5266,
+ "theoretical_loss": 3.962338549356141,
+ "tokens_seen": 478085120
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006910010030090271,
+ "loss": 1.5045,
+ "theoretical_loss": 3.962280022138893,
+ "tokens_seen": 478150656
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006909849548645938,
+ "loss": 1.5396,
+ "theoretical_loss": 3.962221505188677,
+ "tokens_seen": 478216192
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006909689067201605,
+ "loss": 1.4155,
+ "theoretical_loss": 3.9621629985022855,
+ "tokens_seen": 478281728
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006909528585757272,
+ "loss": 1.482,
+ "theoretical_loss": 3.9621045020765124,
+ "tokens_seen": 478347264
+ },
+ {
+ "epoch": 0.14,
+ "objective/train/docs_used": 165388,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.461626648902893,
+ "objective/train/theoretical_loss": 3.9620460159081525,
+ "objective/train/tokens_used": 498872800,
+ "theoretical_loss": 3.9620460159081525,
+ "tokens_seen": 478412800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006909368104312939,
+ "loss": 1.4965,
+ "theoretical_loss": 3.9620460159081525,
+ "tokens_seen": 478412800
+ },
+ {
+ "epoch": 0.14,
+ "learning_rate": 0.0006909207622868606,
+ "loss": 1.5332,
+ "theoretical_loss": 3.9619875399940025,
+ "tokens_seen": 478478336
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006909047141424273,
+ "loss": 1.4555,
+ "theoretical_loss": 3.9619290743308606,
+ "tokens_seen": 478543872
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690888665997994,
+ "loss": 1.5502,
+ "theoretical_loss": 3.961870618915526,
+ "tokens_seen": 478609408
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006908726178535607,
+ "loss": 1.4432,
+ "theoretical_loss": 3.961812173744801,
+ "tokens_seen": 478674944
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006908565697091274,
+ "loss": 1.4627,
+ "theoretical_loss": 3.961753738815486,
+ "tokens_seen": 478740480
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006908405215646941,
+ "loss": 1.4945,
+ "theoretical_loss": 3.961695314124385,
+ "tokens_seen": 478806016
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006908244734202608,
+ "loss": 1.5489,
+ "theoretical_loss": 3.961636899668304,
+ "tokens_seen": 478871552
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006908084252758275,
+ "loss": 1.5541,
+ "theoretical_loss": 3.96157849544405,
+ "tokens_seen": 478937088
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006907923771313942,
+ "loss": 1.5218,
+ "theoretical_loss": 3.96152010144843,
+ "tokens_seen": 479002624
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006907763289869609,
+ "loss": 1.476,
+ "theoretical_loss": 3.961461717678254,
+ "tokens_seen": 479068160
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006907602808425276,
+ "loss": 1.3915,
+ "theoretical_loss": 3.961403344130333,
+ "tokens_seen": 479133696
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006907442326980943,
+ "loss": 1.5185,
+ "theoretical_loss": 3.9613449808014796,
+ "tokens_seen": 479199232
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690728184553661,
+ "loss": 1.6288,
+ "theoretical_loss": 3.9612866276885073,
+ "tokens_seen": 479264768
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006907121364092277,
+ "loss": 1.5618,
+ "theoretical_loss": 3.961228284788231,
+ "tokens_seen": 479330304
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006906960882647944,
+ "loss": 1.403,
+ "theoretical_loss": 3.9611699520974684,
+ "tokens_seen": 479395840
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006906800401203612,
+ "loss": 1.5271,
+ "theoretical_loss": 3.961111629613036,
+ "tokens_seen": 479461376
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006906639919759278,
+ "loss": 1.627,
+ "theoretical_loss": 3.9610533173317553,
+ "tokens_seen": 479526912
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006906479438314946,
+ "loss": 1.4974,
+ "theoretical_loss": 3.960995015250446,
+ "tokens_seen": 479592448
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006906318956870612,
+ "loss": 1.4971,
+ "theoretical_loss": 3.96093672336593,
+ "tokens_seen": 479657984
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690615847542628,
+ "loss": 1.6243,
+ "theoretical_loss": 3.9608784416750322,
+ "tokens_seen": 479723520
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905997993981946,
+ "loss": 1.4766,
+ "theoretical_loss": 3.9608201701745775,
+ "tokens_seen": 479789056
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905837512537614,
+ "loss": 1.5663,
+ "theoretical_loss": 3.960761908861392,
+ "tokens_seen": 479854592
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690567703109328,
+ "loss": 1.4331,
+ "theoretical_loss": 3.9607036577323047,
+ "tokens_seen": 479920128
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905516549648948,
+ "loss": 1.3873,
+ "theoretical_loss": 3.9606454167841445,
+ "tokens_seen": 479985664
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905356068204614,
+ "loss": 1.5245,
+ "theoretical_loss": 3.960587186013742,
+ "tokens_seen": 480051200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905195586760282,
+ "loss": 1.5018,
+ "theoretical_loss": 3.96052896541793,
+ "tokens_seen": 480116736
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006905035105315948,
+ "loss": 1.4737,
+ "theoretical_loss": 3.9604707549935423,
+ "tokens_seen": 480182272
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006904874623871616,
+ "loss": 1.5611,
+ "theoretical_loss": 3.9604125547374136,
+ "tokens_seen": 480247808
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006904714142427282,
+ "loss": 1.4181,
+ "theoretical_loss": 3.9603543646463812,
+ "tokens_seen": 480313344
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690455366098295,
+ "loss": 1.5261,
+ "theoretical_loss": 3.9602961847172824,
+ "tokens_seen": 480378880
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006904393179538616,
+ "loss": 1.5351,
+ "theoretical_loss": 3.960238014946957,
+ "tokens_seen": 480444416
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006904232698094284,
+ "loss": 1.4876,
+ "theoretical_loss": 3.960179855332245,
+ "tokens_seen": 480509952
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690407221664995,
+ "loss": 1.4839,
+ "theoretical_loss": 3.96012170586999,
+ "tokens_seen": 480575488
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903911735205618,
+ "loss": 1.3866,
+ "theoretical_loss": 3.9600635665570345,
+ "tokens_seen": 480641024
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903751253761284,
+ "loss": 1.5976,
+ "theoretical_loss": 3.9600054373902243,
+ "tokens_seen": 480706560
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903590772316952,
+ "loss": 1.4092,
+ "theoretical_loss": 3.959947318366405,
+ "tokens_seen": 480772096
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903430290872618,
+ "loss": 1.4633,
+ "theoretical_loss": 3.9598892094824256,
+ "tokens_seen": 480837632
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903269809428286,
+ "loss": 1.5516,
+ "theoretical_loss": 3.9598311107351347,
+ "tokens_seen": 480903168
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006903109327983952,
+ "loss": 1.5714,
+ "theoretical_loss": 3.959773022121383,
+ "tokens_seen": 480968704
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690294884653962,
+ "loss": 1.4482,
+ "theoretical_loss": 3.959714943638023,
+ "tokens_seen": 481034240
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006902788365095286,
+ "loss": 1.4323,
+ "theoretical_loss": 3.9596568752819072,
+ "tokens_seen": 481099776
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006902627883650954,
+ "loss": 1.5344,
+ "theoretical_loss": 3.9595988170498915,
+ "tokens_seen": 481165312
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000690246740220662,
+ "loss": 1.5042,
+ "theoretical_loss": 3.959540768938832,
+ "tokens_seen": 481230848
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006902306920762287,
+ "loss": 1.6065,
+ "theoretical_loss": 3.9594827309455862,
+ "tokens_seen": 481296384
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006902146439317954,
+ "loss": 1.5888,
+ "theoretical_loss": 3.9594247030670133,
+ "tokens_seen": 481361920
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901985957873621,
+ "loss": 1.4552,
+ "theoretical_loss": 3.959366685299974,
+ "tokens_seen": 481427456
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901825476429288,
+ "loss": 1.564,
+ "theoretical_loss": 3.9593086776413298,
+ "tokens_seen": 481492992
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901664994984955,
+ "loss": 1.4304,
+ "theoretical_loss": 3.959250680087945,
+ "tokens_seen": 481558528
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901504513540622,
+ "loss": 1.4006,
+ "theoretical_loss": 3.959192692636683,
+ "tokens_seen": 481624064
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 166207,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4316575527191162,
+ "objective/train/theoretical_loss": 3.959134715284411,
+ "objective/train/tokens_used": 502149600,
+ "theoretical_loss": 3.959134715284411,
+ "tokens_seen": 481689600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901344032096289,
+ "loss": 1.5149,
+ "theoretical_loss": 3.959134715284411,
+ "tokens_seen": 481689600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901183550651957,
+ "loss": 1.5353,
+ "theoretical_loss": 3.959076748027996,
+ "tokens_seen": 481755136
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006901023069207623,
+ "loss": 1.5337,
+ "theoretical_loss": 3.959018790864307,
+ "tokens_seen": 481820672
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900862587763289,
+ "loss": 1.535,
+ "theoretical_loss": 3.9589608437902144,
+ "tokens_seen": 481886208
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900702106318957,
+ "loss": 1.5224,
+ "theoretical_loss": 3.9589029068025896,
+ "tokens_seen": 481951744
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900541624874623,
+ "loss": 1.4818,
+ "theoretical_loss": 3.9588449798983065,
+ "tokens_seen": 482017280
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900381143430291,
+ "loss": 1.4084,
+ "theoretical_loss": 3.9587870630742383,
+ "tokens_seen": 482082816
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900220661985957,
+ "loss": 1.6073,
+ "theoretical_loss": 3.958729156327262,
+ "tokens_seen": 482148352
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006900060180541625,
+ "loss": 1.5083,
+ "theoretical_loss": 3.9586712596542553,
+ "tokens_seen": 482213888
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006899899699097291,
+ "loss": 1.4613,
+ "theoretical_loss": 3.958613373052095,
+ "tokens_seen": 482279424
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006899739217652959,
+ "loss": 1.4953,
+ "theoretical_loss": 3.958555496517663,
+ "tokens_seen": 482344960
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006899578736208625,
+ "loss": 1.4788,
+ "theoretical_loss": 3.95849763004784,
+ "tokens_seen": 482410496
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006899418254764293,
+ "loss": 1.4487,
+ "theoretical_loss": 3.958439773639509,
+ "tokens_seen": 482476032
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000689925777331996,
+ "loss": 1.4259,
+ "theoretical_loss": 3.958381927289554,
+ "tokens_seen": 482541568
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006899097291875627,
+ "loss": 1.4236,
+ "theoretical_loss": 3.958324090994861,
+ "tokens_seen": 482607104
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898936810431294,
+ "loss": 1.4207,
+ "theoretical_loss": 3.958266264752317,
+ "tokens_seen": 482672640
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898776328986961,
+ "loss": 1.5267,
+ "theoretical_loss": 3.9582084485588096,
+ "tokens_seen": 482738176
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898615847542628,
+ "loss": 1.5434,
+ "theoretical_loss": 3.95815064241123,
+ "tokens_seen": 482803712
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898455366098295,
+ "loss": 1.4206,
+ "theoretical_loss": 3.958092846306468,
+ "tokens_seen": 482869248
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898294884653962,
+ "loss": 1.5472,
+ "theoretical_loss": 3.958035060241417,
+ "tokens_seen": 482934784
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006898134403209629,
+ "loss": 1.5298,
+ "theoretical_loss": 3.957977284212971,
+ "tokens_seen": 483000320
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897973921765296,
+ "loss": 1.3356,
+ "theoretical_loss": 3.9579195182180245,
+ "tokens_seen": 483065856
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897813440320963,
+ "loss": 1.4347,
+ "theoretical_loss": 3.957861762253475,
+ "tokens_seen": 483131392
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000689765295887663,
+ "loss": 1.4678,
+ "theoretical_loss": 3.9578040163162203,
+ "tokens_seen": 483196928
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897492477432297,
+ "loss": 1.5097,
+ "theoretical_loss": 3.95774628040316,
+ "tokens_seen": 483262464
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897331995987964,
+ "loss": 1.4883,
+ "theoretical_loss": 3.9576885545111944,
+ "tokens_seen": 483328000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897171514543631,
+ "loss": 1.4399,
+ "theoretical_loss": 3.9576308386372263,
+ "tokens_seen": 483393536
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006897011033099298,
+ "loss": 1.6046,
+ "theoretical_loss": 3.957573132778159,
+ "tokens_seen": 483459072
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006896850551654965,
+ "loss": 1.499,
+ "theoretical_loss": 3.9575154369308985,
+ "tokens_seen": 483524608
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006896690070210632,
+ "loss": 1.482,
+ "theoretical_loss": 3.957457751092349,
+ "tokens_seen": 483590144
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006896529588766299,
+ "loss": 1.3449,
+ "theoretical_loss": 3.9574000752594203,
+ "tokens_seen": 483655680
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006896369107321966,
+ "loss": 1.466,
+ "theoretical_loss": 3.9573424094290206,
+ "tokens_seen": 483721216
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006896208625877633,
+ "loss": 1.3929,
+ "theoretical_loss": 3.9572847535980604,
+ "tokens_seen": 483786752
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.00068960481444333,
+ "loss": 1.446,
+ "theoretical_loss": 3.9572271077634515,
+ "tokens_seen": 483852288
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895887662988967,
+ "loss": 1.5478,
+ "theoretical_loss": 3.957169471922107,
+ "tokens_seen": 483917824
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895727181544634,
+ "loss": 1.4874,
+ "theoretical_loss": 3.9571118460709425,
+ "tokens_seen": 483983360
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895566700100301,
+ "loss": 1.4437,
+ "theoretical_loss": 3.9570542302068725,
+ "tokens_seen": 484048896
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895406218655968,
+ "loss": 1.5434,
+ "theoretical_loss": 3.9569966243268153,
+ "tokens_seen": 484114432
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895245737211635,
+ "loss": 1.4222,
+ "theoretical_loss": 3.9569390284276893,
+ "tokens_seen": 484179968
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006895085255767302,
+ "loss": 1.4055,
+ "theoretical_loss": 3.9568814425064147,
+ "tokens_seen": 484245504
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894924774322969,
+ "loss": 1.5564,
+ "theoretical_loss": 3.9568238665599127,
+ "tokens_seen": 484311040
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894764292878637,
+ "loss": 1.5019,
+ "theoretical_loss": 3.9567663005851057,
+ "tokens_seen": 484376576
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894603811434303,
+ "loss": 1.4021,
+ "theoretical_loss": 3.956708744578919,
+ "tokens_seen": 484442112
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894443329989971,
+ "loss": 1.4476,
+ "theoretical_loss": 3.9566511985382773,
+ "tokens_seen": 484507648
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894282848545637,
+ "loss": 1.4908,
+ "theoretical_loss": 3.956593662460108,
+ "tokens_seen": 484573184
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006894122367101305,
+ "loss": 1.5107,
+ "theoretical_loss": 3.9565361363413385,
+ "tokens_seen": 484638720
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893961885656971,
+ "loss": 1.4567,
+ "theoretical_loss": 3.9564786201788995,
+ "tokens_seen": 484704256
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893801404212639,
+ "loss": 1.5791,
+ "theoretical_loss": 3.956421113969721,
+ "tokens_seen": 484769792
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893640922768305,
+ "loss": 1.4383,
+ "theoretical_loss": 3.9563636177107364,
+ "tokens_seen": 484835328
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893480441323973,
+ "loss": 1.4124,
+ "theoretical_loss": 3.956306131398878,
+ "tokens_seen": 484900864
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 167502,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4446344375610352,
+ "objective/train/theoretical_loss": 3.9562486550310823,
+ "objective/train/tokens_used": 505426400,
+ "theoretical_loss": 3.9562486550310823,
+ "tokens_seen": 484966400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893319959879639,
+ "loss": 1.3573,
+ "theoretical_loss": 3.9562486550310823,
+ "tokens_seen": 484966400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006893159478435307,
+ "loss": 1.4464,
+ "theoretical_loss": 3.9561911886042846,
+ "tokens_seen": 485031936
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892998996990973,
+ "loss": 1.4097,
+ "theoretical_loss": 3.9561337321154237,
+ "tokens_seen": 485097472
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892838515546641,
+ "loss": 1.4955,
+ "theoretical_loss": 3.956076285561438,
+ "tokens_seen": 485163008
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892678034102307,
+ "loss": 1.4266,
+ "theoretical_loss": 3.9560188489392676,
+ "tokens_seen": 485228544
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892517552657975,
+ "loss": 1.4735,
+ "theoretical_loss": 3.9559614222458555,
+ "tokens_seen": 485294080
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892357071213641,
+ "loss": 1.4313,
+ "theoretical_loss": 3.955904005478144,
+ "tokens_seen": 485359616
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892196589769309,
+ "loss": 1.4503,
+ "theoretical_loss": 3.9558465986330784,
+ "tokens_seen": 485425152
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006892036108324975,
+ "loss": 1.5692,
+ "theoretical_loss": 3.9557892017076037,
+ "tokens_seen": 485490688
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891875626880643,
+ "loss": 1.4592,
+ "theoretical_loss": 3.9557318146986677,
+ "tokens_seen": 485556224
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891715145436309,
+ "loss": 1.4865,
+ "theoretical_loss": 3.9556744376032196,
+ "tokens_seen": 485621760
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891554663991977,
+ "loss": 1.554,
+ "theoretical_loss": 3.9556170704182083,
+ "tokens_seen": 485687296
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891394182547643,
+ "loss": 1.4508,
+ "theoretical_loss": 3.955559713140585,
+ "tokens_seen": 485752832
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891233701103311,
+ "loss": 1.4282,
+ "theoretical_loss": 3.9555023657673036,
+ "tokens_seen": 485818368
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006891073219658977,
+ "loss": 1.4978,
+ "theoretical_loss": 3.955445028295317,
+ "tokens_seen": 485883904
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890912738214645,
+ "loss": 1.418,
+ "theoretical_loss": 3.955387700721581,
+ "tokens_seen": 485949440
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890752256770311,
+ "loss": 1.5058,
+ "theoretical_loss": 3.9553303830430524,
+ "tokens_seen": 486014976
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890591775325979,
+ "loss": 1.5994,
+ "theoretical_loss": 3.955273075256689,
+ "tokens_seen": 486080512
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890431293881645,
+ "loss": 1.4623,
+ "theoretical_loss": 3.9552157773594496,
+ "tokens_seen": 486146048
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890270812437313,
+ "loss": 1.4523,
+ "theoretical_loss": 3.9551584893482965,
+ "tokens_seen": 486211584
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006890110330992979,
+ "loss": 1.5572,
+ "theoretical_loss": 3.9551012112201906,
+ "tokens_seen": 486277120
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889949849548647,
+ "loss": 1.4486,
+ "theoretical_loss": 3.9550439429720954,
+ "tokens_seen": 486342656
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889789368104313,
+ "loss": 1.4769,
+ "theoretical_loss": 3.9549866846009762,
+ "tokens_seen": 486408192
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889628886659981,
+ "loss": 1.6685,
+ "theoretical_loss": 3.9549294361037988,
+ "tokens_seen": 486473728
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889468405215647,
+ "loss": 1.5421,
+ "theoretical_loss": 3.9548721974775303,
+ "tokens_seen": 486539264
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889307923771315,
+ "loss": 1.477,
+ "theoretical_loss": 3.95481496871914,
+ "tokens_seen": 486604800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006889147442326981,
+ "loss": 1.5481,
+ "theoretical_loss": 3.954757749825598,
+ "tokens_seen": 486670336
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006888986960882649,
+ "loss": 1.4039,
+ "theoretical_loss": 3.9547005407938753,
+ "tokens_seen": 486735872
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006888826479438315,
+ "loss": 1.3672,
+ "theoretical_loss": 3.9546433416209448,
+ "tokens_seen": 486801408
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006888665997993983,
+ "loss": 1.5134,
+ "theoretical_loss": 3.9545861523037815,
+ "tokens_seen": 486866944
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688850551654965,
+ "loss": 1.5121,
+ "theoretical_loss": 3.95452897283936,
+ "tokens_seen": 486932480
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006888345035105317,
+ "loss": 1.5094,
+ "theoretical_loss": 3.9544718032246573,
+ "tokens_seen": 486998016
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006888184553660983,
+ "loss": 1.5549,
+ "theoretical_loss": 3.9544146434566514,
+ "tokens_seen": 487063552
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688802407221665,
+ "loss": 1.4202,
+ "theoretical_loss": 3.9543574935323216,
+ "tokens_seen": 487129088
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887863590772317,
+ "loss": 1.4835,
+ "theoretical_loss": 3.95430035344865,
+ "tokens_seen": 487194624
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887703109327984,
+ "loss": 1.4217,
+ "theoretical_loss": 3.954243223202617,
+ "tokens_seen": 487260160
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887542627883652,
+ "loss": 1.4977,
+ "theoretical_loss": 3.9541861027912066,
+ "tokens_seen": 487325696
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887382146439318,
+ "loss": 1.5066,
+ "theoretical_loss": 3.9541289922114045,
+ "tokens_seen": 487391232
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887221664994986,
+ "loss": 1.5499,
+ "theoretical_loss": 3.954071891460196,
+ "tokens_seen": 487456768
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006887061183550652,
+ "loss": 1.4346,
+ "theoretical_loss": 3.954014800534569,
+ "tokens_seen": 487522304
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006886900702106318,
+ "loss": 1.4414,
+ "theoretical_loss": 3.9539577194315116,
+ "tokens_seen": 487587840
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006886740220661986,
+ "loss": 1.5233,
+ "theoretical_loss": 3.953900648148015,
+ "tokens_seen": 487653376
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006886579739217652,
+ "loss": 1.5257,
+ "theoretical_loss": 3.953843586681069,
+ "tokens_seen": 487718912
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688641925777332,
+ "loss": 1.4625,
+ "theoretical_loss": 3.9537865350276684,
+ "tokens_seen": 487784448
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006886258776328986,
+ "loss": 1.6209,
+ "theoretical_loss": 3.953729493184806,
+ "tokens_seen": 487849984
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006886098294884654,
+ "loss": 1.4365,
+ "theoretical_loss": 3.953672461149477,
+ "tokens_seen": 487915520
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688593781344032,
+ "loss": 1.5948,
+ "theoretical_loss": 3.953615438918679,
+ "tokens_seen": 487981056
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006885777331995988,
+ "loss": 1.4913,
+ "theoretical_loss": 3.9535584264894092,
+ "tokens_seen": 488046592
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006885616850551655,
+ "loss": 1.4834,
+ "theoretical_loss": 3.9535014238586683,
+ "tokens_seen": 488112128
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006885456369107322,
+ "loss": 1.5498,
+ "theoretical_loss": 3.953444431023456,
+ "tokens_seen": 488177664
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 168681,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5218801498413086,
+ "objective/train/theoretical_loss": 3.9533874479807745,
+ "objective/train/tokens_used": 508703200,
+ "theoretical_loss": 3.9533874479807745,
+ "tokens_seen": 488243200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006885295887662989,
+ "loss": 1.4728,
+ "theoretical_loss": 3.9533874479807745,
+ "tokens_seen": 488243200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006885135406218656,
+ "loss": 1.4928,
+ "theoretical_loss": 3.9533304747276263,
+ "tokens_seen": 488308736
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884974924774323,
+ "loss": 1.4964,
+ "theoretical_loss": 3.953273511261018,
+ "tokens_seen": 488374272
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688481444332999,
+ "loss": 1.4585,
+ "theoretical_loss": 3.953216557577954,
+ "tokens_seen": 488439808
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884653961885657,
+ "loss": 1.5129,
+ "theoretical_loss": 3.9531596136754423,
+ "tokens_seen": 488505344
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884493480441324,
+ "loss": 1.3073,
+ "theoretical_loss": 3.953102679550491,
+ "tokens_seen": 488570880
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884332998996991,
+ "loss": 1.6385,
+ "theoretical_loss": 3.953045755200111,
+ "tokens_seen": 488636416
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884172517552658,
+ "loss": 1.4704,
+ "theoretical_loss": 3.9529888406213125,
+ "tokens_seen": 488701952
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006884012036108325,
+ "loss": 1.4706,
+ "theoretical_loss": 3.952931935811108,
+ "tokens_seen": 488767488
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006883851554663992,
+ "loss": 1.4269,
+ "theoretical_loss": 3.9528750407665125,
+ "tokens_seen": 488833024
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006883691073219659,
+ "loss": 1.6005,
+ "theoretical_loss": 3.9528181554845405,
+ "tokens_seen": 488898560
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006883530591775326,
+ "loss": 1.3151,
+ "theoretical_loss": 3.9527612799622087,
+ "tokens_seen": 488964096
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006883370110330993,
+ "loss": 1.3791,
+ "theoretical_loss": 3.9527044141965346,
+ "tokens_seen": 489029632
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688320962888666,
+ "loss": 1.5089,
+ "theoretical_loss": 3.9526475581845375,
+ "tokens_seen": 489095168
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006883049147442327,
+ "loss": 1.4356,
+ "theoretical_loss": 3.9525907119232384,
+ "tokens_seen": 489160704
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006882888665997994,
+ "loss": 1.4122,
+ "theoretical_loss": 3.952533875409658,
+ "tokens_seen": 489226240
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006882728184553662,
+ "loss": 1.6054,
+ "theoretical_loss": 3.95247704864082,
+ "tokens_seen": 489291776
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006882567703109328,
+ "loss": 1.5323,
+ "theoretical_loss": 3.952420231613748,
+ "tokens_seen": 489357312
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006882407221664996,
+ "loss": 1.4921,
+ "theoretical_loss": 3.952363424325469,
+ "tokens_seen": 489422848
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006882246740220662,
+ "loss": 1.5348,
+ "theoretical_loss": 3.952306626773009,
+ "tokens_seen": 489488384
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688208625877633,
+ "loss": 1.2855,
+ "theoretical_loss": 3.9522498389533967,
+ "tokens_seen": 489553920
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006881925777331996,
+ "loss": 1.4844,
+ "theoretical_loss": 3.952193060863661,
+ "tokens_seen": 489619456
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006881765295887664,
+ "loss": 1.4188,
+ "theoretical_loss": 3.9521362925008336,
+ "tokens_seen": 489684992
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688160481444333,
+ "loss": 1.3951,
+ "theoretical_loss": 3.952079533861947,
+ "tokens_seen": 489750528
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006881444332998998,
+ "loss": 1.5724,
+ "theoretical_loss": 3.9520227849440337,
+ "tokens_seen": 489816064
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006881283851554664,
+ "loss": 1.3901,
+ "theoretical_loss": 3.951966045744129,
+ "tokens_seen": 489881600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006881123370110332,
+ "loss": 1.5279,
+ "theoretical_loss": 3.9519093162592687,
+ "tokens_seen": 489947136
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880962888665998,
+ "loss": 1.4731,
+ "theoretical_loss": 3.95185259648649,
+ "tokens_seen": 490012672
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880802407221666,
+ "loss": 1.4999,
+ "theoretical_loss": 3.951795886422833,
+ "tokens_seen": 490078208
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880641925777332,
+ "loss": 1.5234,
+ "theoretical_loss": 3.9517391860653364,
+ "tokens_seen": 490143744
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880481444333,
+ "loss": 1.5716,
+ "theoretical_loss": 3.951682495411041,
+ "tokens_seen": 490209280
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880320962888666,
+ "loss": 1.4127,
+ "theoretical_loss": 3.9516258144569907,
+ "tokens_seen": 490274816
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006880160481444334,
+ "loss": 1.4481,
+ "theoretical_loss": 3.9515691432002296,
+ "tokens_seen": 490340352
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000688,
+ "loss": 1.5564,
+ "theoretical_loss": 3.951512481637801,
+ "tokens_seen": 490405888
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879839518555668,
+ "loss": 1.5187,
+ "theoretical_loss": 3.9514558297667532,
+ "tokens_seen": 490471424
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879679037111334,
+ "loss": 1.5075,
+ "theoretical_loss": 3.9513991875841334,
+ "tokens_seen": 490536960
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879518555667002,
+ "loss": 1.4442,
+ "theoretical_loss": 3.95134255508699,
+ "tokens_seen": 490602496
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879358074222668,
+ "loss": 1.5834,
+ "theoretical_loss": 3.9512859322723743,
+ "tokens_seen": 490668032
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879197592778336,
+ "loss": 1.5289,
+ "theoretical_loss": 3.9512293191373375,
+ "tokens_seen": 490733568
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006879037111334002,
+ "loss": 1.6062,
+ "theoretical_loss": 3.9511727156789327,
+ "tokens_seen": 490799104
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687887662988967,
+ "loss": 1.5124,
+ "theoretical_loss": 3.951116121894214,
+ "tokens_seen": 490864640
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006878716148445336,
+ "loss": 1.4595,
+ "theoretical_loss": 3.951059537780237,
+ "tokens_seen": 490930176
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006878555667001004,
+ "loss": 1.5176,
+ "theoretical_loss": 3.951002963334059,
+ "tokens_seen": 490995712
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687839518555667,
+ "loss": 1.4955,
+ "theoretical_loss": 3.9509463985527375,
+ "tokens_seen": 491061248
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006878234704112338,
+ "loss": 1.4248,
+ "theoretical_loss": 3.9508898434333313,
+ "tokens_seen": 491126784
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006878074222668004,
+ "loss": 1.5181,
+ "theoretical_loss": 3.9508332979729026,
+ "tokens_seen": 491192320
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006877913741223672,
+ "loss": 1.4302,
+ "theoretical_loss": 3.950776762168512,
+ "tokens_seen": 491257856
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006877753259779338,
+ "loss": 1.4632,
+ "theoretical_loss": 3.9507202360172236,
+ "tokens_seen": 491323392
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006877592778335006,
+ "loss": 1.3747,
+ "theoretical_loss": 3.9506637195161014,
+ "tokens_seen": 491388928
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006877432296890672,
+ "loss": 1.3571,
+ "theoretical_loss": 3.950607212662211,
+ "tokens_seen": 491454464
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 169679,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.443345069885254,
+ "objective/train/theoretical_loss": 3.950550715452621,
+ "objective/train/tokens_used": 511980000,
+ "theoretical_loss": 3.950550715452621,
+ "tokens_seen": 491520000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687727181544634,
+ "loss": 1.4414,
+ "theoretical_loss": 3.950550715452621,
+ "tokens_seen": 491520000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006877111334002006,
+ "loss": 1.5504,
+ "theoretical_loss": 3.9504942278843984,
+ "tokens_seen": 491585536
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006876950852557674,
+ "loss": 1.4341,
+ "theoretical_loss": 3.950437749954613,
+ "tokens_seen": 491651072
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687679037111334,
+ "loss": 1.5328,
+ "theoretical_loss": 3.950381281660336,
+ "tokens_seen": 491716608
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006876629889669008,
+ "loss": 1.5646,
+ "theoretical_loss": 3.950324822998639,
+ "tokens_seen": 491782144
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006876469408224674,
+ "loss": 1.5351,
+ "theoretical_loss": 3.950268373966597,
+ "tokens_seen": 491847680
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006876308926780342,
+ "loss": 1.5962,
+ "theoretical_loss": 3.950211934561284,
+ "tokens_seen": 491913216
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006876148445336008,
+ "loss": 1.5011,
+ "theoretical_loss": 3.950155504779775,
+ "tokens_seen": 491978752
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006875987963891676,
+ "loss": 1.5318,
+ "theoretical_loss": 3.9500990846191484,
+ "tokens_seen": 492044288
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006875827482447342,
+ "loss": 1.4562,
+ "theoretical_loss": 3.950042674076483,
+ "tokens_seen": 492109824
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687566700100301,
+ "loss": 1.5587,
+ "theoretical_loss": 3.949986273148858,
+ "tokens_seen": 492175360
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006875506519558676,
+ "loss": 1.6206,
+ "theoretical_loss": 3.9499298818333557,
+ "tokens_seen": 492240896
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006875346038114344,
+ "loss": 1.5126,
+ "theoretical_loss": 3.9498735001270573,
+ "tokens_seen": 492306432
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687518555667001,
+ "loss": 1.6095,
+ "theoretical_loss": 3.949817128027047,
+ "tokens_seen": 492371968
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006875025075225678,
+ "loss": 1.4721,
+ "theoretical_loss": 3.9497607655304097,
+ "tokens_seen": 492437504
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874864593781344,
+ "loss": 1.4875,
+ "theoretical_loss": 3.949704412634232,
+ "tokens_seen": 492503040
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874704112337012,
+ "loss": 1.5354,
+ "theoretical_loss": 3.949648069335601,
+ "tokens_seen": 492568576
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874543630892678,
+ "loss": 1.5265,
+ "theoretical_loss": 3.9495917356316057,
+ "tokens_seen": 492634112
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874383149448346,
+ "loss": 1.4206,
+ "theoretical_loss": 3.9495354115193364,
+ "tokens_seen": 492699648
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874222668004012,
+ "loss": 1.5544,
+ "theoretical_loss": 3.9494790969958835,
+ "tokens_seen": 492765184
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006874062186559679,
+ "loss": 1.5174,
+ "theoretical_loss": 3.949422792058341,
+ "tokens_seen": 492830720
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006873901705115347,
+ "loss": 1.4034,
+ "theoretical_loss": 3.949366496703802,
+ "tokens_seen": 492896256
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006873741223671013,
+ "loss": 1.5041,
+ "theoretical_loss": 3.9493102109293616,
+ "tokens_seen": 492961792
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687358074222668,
+ "loss": 1.5309,
+ "theoretical_loss": 3.9492539347321167,
+ "tokens_seen": 493027328
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006873420260782347,
+ "loss": 1.3729,
+ "theoretical_loss": 3.9491976681091643,
+ "tokens_seen": 493092864
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006873259779338015,
+ "loss": 1.428,
+ "theoretical_loss": 3.949141411057604,
+ "tokens_seen": 493158400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006873099297893681,
+ "loss": 1.5152,
+ "theoretical_loss": 3.9490851635745354,
+ "tokens_seen": 493223936
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872938816449349,
+ "loss": 1.4597,
+ "theoretical_loss": 3.9490289256570605,
+ "tokens_seen": 493289472
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872778335005015,
+ "loss": 1.5602,
+ "theoretical_loss": 3.9489726973022816,
+ "tokens_seen": 493355008
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872617853560681,
+ "loss": 1.4065,
+ "theoretical_loss": 3.948916478507303,
+ "tokens_seen": 493420544
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872457372116349,
+ "loss": 1.4266,
+ "theoretical_loss": 3.94886026926923,
+ "tokens_seen": 493486080
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872296890672015,
+ "loss": 1.4491,
+ "theoretical_loss": 3.9488040695851687,
+ "tokens_seen": 493551616
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006872136409227683,
+ "loss": 1.4786,
+ "theoretical_loss": 3.9487478794522275,
+ "tokens_seen": 493617152
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000687197592778335,
+ "loss": 1.5208,
+ "theoretical_loss": 3.948691698867515,
+ "tokens_seen": 493682688
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871815446339017,
+ "loss": 1.4405,
+ "theoretical_loss": 3.9486355278281415,
+ "tokens_seen": 493748224
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871654964894684,
+ "loss": 1.4269,
+ "theoretical_loss": 3.948579366331219,
+ "tokens_seen": 493813760
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871494483450351,
+ "loss": 1.524,
+ "theoretical_loss": 3.94852321437386,
+ "tokens_seen": 493879296
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871334002006018,
+ "loss": 1.4391,
+ "theoretical_loss": 3.9484670719531785,
+ "tokens_seen": 493944832
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871173520561685,
+ "loss": 1.5082,
+ "theoretical_loss": 3.9484109390662896,
+ "tokens_seen": 494010368
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006871013039117352,
+ "loss": 1.5184,
+ "theoretical_loss": 3.94835481571031,
+ "tokens_seen": 494075904
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870852557673019,
+ "loss": 1.4863,
+ "theoretical_loss": 3.948298701882358,
+ "tokens_seen": 494141440
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870692076228687,
+ "loss": 1.5287,
+ "theoretical_loss": 3.9482425975795525,
+ "tokens_seen": 494206976
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870531594784353,
+ "loss": 1.5385,
+ "theoretical_loss": 3.9481865027990137,
+ "tokens_seen": 494272512
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870371113340021,
+ "loss": 1.5284,
+ "theoretical_loss": 3.9481304175378633,
+ "tokens_seen": 494338048
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870210631895687,
+ "loss": 1.4945,
+ "theoretical_loss": 3.948074341793224,
+ "tokens_seen": 494403584
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006870050150451355,
+ "loss": 1.5887,
+ "theoretical_loss": 3.94801827556222,
+ "tokens_seen": 494469120
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869889669007021,
+ "loss": 1.4482,
+ "theoretical_loss": 3.9479622188419765,
+ "tokens_seen": 494534656
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869729187562689,
+ "loss": 1.4412,
+ "theoretical_loss": 3.9479061716296204,
+ "tokens_seen": 494600192
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869568706118355,
+ "loss": 1.5186,
+ "theoretical_loss": 3.9478501339222793,
+ "tokens_seen": 494665728
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869408224674023,
+ "loss": 1.4655,
+ "theoretical_loss": 3.9477941057170822,
+ "tokens_seen": 494731264
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 170930,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5508875846862793,
+ "objective/train/theoretical_loss": 3.9477380870111602,
+ "objective/train/tokens_used": 515256800,
+ "theoretical_loss": 3.9477380870111602,
+ "tokens_seen": 494796800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869247743229689,
+ "loss": 1.4898,
+ "theoretical_loss": 3.9477380870111602,
+ "tokens_seen": 494796800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006869087261785357,
+ "loss": 1.5979,
+ "theoretical_loss": 3.9476820778016437,
+ "tokens_seen": 494862336
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868926780341023,
+ "loss": 1.5028,
+ "theoretical_loss": 3.9476260780856665,
+ "tokens_seen": 494927872
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868766298896691,
+ "loss": 1.5157,
+ "theoretical_loss": 3.947570087860362,
+ "tokens_seen": 494993408
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868605817452357,
+ "loss": 1.4887,
+ "theoretical_loss": 3.947514107122866,
+ "tokens_seen": 495058944
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868445336008025,
+ "loss": 1.5196,
+ "theoretical_loss": 3.9474581358703147,
+ "tokens_seen": 495124480
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868284854563691,
+ "loss": 1.4206,
+ "theoretical_loss": 3.947402174099847,
+ "tokens_seen": 495190016
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006868124373119359,
+ "loss": 1.4628,
+ "theoretical_loss": 3.9473462218085995,
+ "tokens_seen": 495255552
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867963891675025,
+ "loss": 1.468,
+ "theoretical_loss": 3.947290278993715,
+ "tokens_seen": 495321088
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867803410230693,
+ "loss": 1.4707,
+ "theoretical_loss": 3.947234345652334,
+ "tokens_seen": 495386624
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867642928786359,
+ "loss": 1.4693,
+ "theoretical_loss": 3.9471784217815995,
+ "tokens_seen": 495452160
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867482447342027,
+ "loss": 1.524,
+ "theoretical_loss": 3.9471225073786553,
+ "tokens_seen": 495517696
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867321965897693,
+ "loss": 1.5436,
+ "theoretical_loss": 3.9470666024406467,
+ "tokens_seen": 495583232
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867161484453361,
+ "loss": 1.5024,
+ "theoretical_loss": 3.9470107069647202,
+ "tokens_seen": 495648768
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006867001003009027,
+ "loss": 1.3839,
+ "theoretical_loss": 3.946954820948024,
+ "tokens_seen": 495714304
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866840521564695,
+ "loss": 1.5284,
+ "theoretical_loss": 3.946898944387706,
+ "tokens_seen": 495779840
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866680040120361,
+ "loss": 1.5712,
+ "theoretical_loss": 3.946843077280917,
+ "tokens_seen": 495845376
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866519558676029,
+ "loss": 1.6285,
+ "theoretical_loss": 3.946787219624809,
+ "tokens_seen": 495910912
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866359077231695,
+ "loss": 1.6073,
+ "theoretical_loss": 3.9467313714165337,
+ "tokens_seen": 495976448
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866198595787363,
+ "loss": 1.4352,
+ "theoretical_loss": 3.946675532653246,
+ "tokens_seen": 496041984
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006866038114343029,
+ "loss": 1.514,
+ "theoretical_loss": 3.9466197033321,
+ "tokens_seen": 496107520
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865877632898697,
+ "loss": 1.5126,
+ "theoretical_loss": 3.9465638834502528,
+ "tokens_seen": 496173056
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865717151454363,
+ "loss": 1.4771,
+ "theoretical_loss": 3.9465080730048614,
+ "tokens_seen": 496238592
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865556670010031,
+ "loss": 1.4483,
+ "theoretical_loss": 3.9464522719930857,
+ "tokens_seen": 496304128
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865396188565697,
+ "loss": 1.5381,
+ "theoretical_loss": 3.9463964804120844,
+ "tokens_seen": 496369664
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865235707121365,
+ "loss": 1.5251,
+ "theoretical_loss": 3.9463406982590197,
+ "tokens_seen": 496435200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006865075225677031,
+ "loss": 1.4486,
+ "theoretical_loss": 3.946284925531054,
+ "tokens_seen": 496500736
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864914744232699,
+ "loss": 1.5538,
+ "theoretical_loss": 3.9462291622253503,
+ "tokens_seen": 496566272
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864754262788365,
+ "loss": 1.3553,
+ "theoretical_loss": 3.946173408339075,
+ "tokens_seen": 496631808
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864593781344033,
+ "loss": 1.4294,
+ "theoretical_loss": 3.9461176638693933,
+ "tokens_seen": 496697344
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864433299899699,
+ "loss": 1.5686,
+ "theoretical_loss": 3.9460619288134726,
+ "tokens_seen": 496762880
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864272818455367,
+ "loss": 1.4742,
+ "theoretical_loss": 3.9460062031684817,
+ "tokens_seen": 496828416
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006864112337011033,
+ "loss": 1.3954,
+ "theoretical_loss": 3.945950486931591,
+ "tokens_seen": 496893952
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863951855566701,
+ "loss": 1.5157,
+ "theoretical_loss": 3.9458947800999713,
+ "tokens_seen": 496959488
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863791374122367,
+ "loss": 1.4627,
+ "theoretical_loss": 3.9458390826707945,
+ "tokens_seen": 497025024
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863630892678035,
+ "loss": 1.4251,
+ "theoretical_loss": 3.945783394641235,
+ "tokens_seen": 497090560
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863470411233701,
+ "loss": 1.6089,
+ "theoretical_loss": 3.9457277160084665,
+ "tokens_seen": 497156096
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863309929789369,
+ "loss": 1.5565,
+ "theoretical_loss": 3.9456720467696655,
+ "tokens_seen": 497221632
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006863149448345035,
+ "loss": 1.5502,
+ "theoretical_loss": 3.9456163869220093,
+ "tokens_seen": 497287168
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862988966900703,
+ "loss": 1.4875,
+ "theoretical_loss": 3.945560736462677,
+ "tokens_seen": 497352704
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862828485456369,
+ "loss": 1.4912,
+ "theoretical_loss": 3.945505095388847,
+ "tokens_seen": 497418240
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862668004012037,
+ "loss": 1.4862,
+ "theoretical_loss": 3.9454494636977007,
+ "tokens_seen": 497483776
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862507522567703,
+ "loss": 1.5397,
+ "theoretical_loss": 3.9453938413864207,
+ "tokens_seen": 497549312
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862347041123371,
+ "loss": 1.5082,
+ "theoretical_loss": 3.94533822845219,
+ "tokens_seen": 497614848
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862186559679037,
+ "loss": 1.5358,
+ "theoretical_loss": 3.9452826248921924,
+ "tokens_seen": 497680384
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006862026078234705,
+ "loss": 1.4845,
+ "theoretical_loss": 3.9452270307036144,
+ "tokens_seen": 497745920
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006861865596790371,
+ "loss": 1.4408,
+ "theoretical_loss": 3.945171445883643,
+ "tokens_seen": 497811456
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006861705115346039,
+ "loss": 1.475,
+ "theoretical_loss": 3.9451158704294658,
+ "tokens_seen": 497876992
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006861544633901705,
+ "loss": 1.4578,
+ "theoretical_loss": 3.945060304338273,
+ "tokens_seen": 497942528
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006861384152457373,
+ "loss": 1.4505,
+ "theoretical_loss": 3.9450047476072547,
+ "tokens_seen": 498008064
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 171822,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6084593534469604,
+ "objective/train/theoretical_loss": 3.9449492002336024,
+ "objective/train/tokens_used": 518533600,
+ "theoretical_loss": 3.9449492002336024,
+ "tokens_seen": 498073600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000686122367101304,
+ "loss": 1.5196,
+ "theoretical_loss": 3.9449492002336024,
+ "tokens_seen": 498073600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006861063189568707,
+ "loss": 1.5035,
+ "theoretical_loss": 3.9448936622145103,
+ "tokens_seen": 498139136
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860902708124373,
+ "loss": 1.49,
+ "theoretical_loss": 3.9448381335471714,
+ "tokens_seen": 498204672
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860742226680041,
+ "loss": 1.5774,
+ "theoretical_loss": 3.9447826142287816,
+ "tokens_seen": 498270208
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860581745235707,
+ "loss": 1.4464,
+ "theoretical_loss": 3.944727104256537,
+ "tokens_seen": 498335744
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860421263791375,
+ "loss": 1.5822,
+ "theoretical_loss": 3.9446716036276364,
+ "tokens_seen": 498401280
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860260782347042,
+ "loss": 1.4544,
+ "theoretical_loss": 3.944616112339279,
+ "tokens_seen": 498466816
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006860100300902708,
+ "loss": 1.4174,
+ "theoretical_loss": 3.944560630388664,
+ "tokens_seen": 498532352
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006859939819458376,
+ "loss": 1.4449,
+ "theoretical_loss": 3.9445051577729937,
+ "tokens_seen": 498597888
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006859779338014042,
+ "loss": 1.4734,
+ "theoretical_loss": 3.944449694489471,
+ "tokens_seen": 498663424
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685961885656971,
+ "loss": 1.5683,
+ "theoretical_loss": 3.9443942405352987,
+ "tokens_seen": 498728960
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006859458375125376,
+ "loss": 1.4521,
+ "theoretical_loss": 3.9443387959076825,
+ "tokens_seen": 498794496
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006859297893681044,
+ "loss": 1.4705,
+ "theoretical_loss": 3.944283360603829,
+ "tokens_seen": 498860032
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685913741223671,
+ "loss": 1.5388,
+ "theoretical_loss": 3.9442279346209457,
+ "tokens_seen": 498925568
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858976930792378,
+ "loss": 1.5262,
+ "theoretical_loss": 3.944172517956241,
+ "tokens_seen": 498991104
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858816449348044,
+ "loss": 1.5162,
+ "theoretical_loss": 3.944117110606925,
+ "tokens_seen": 499056640
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858655967903712,
+ "loss": 1.4281,
+ "theoretical_loss": 3.9440617125702087,
+ "tokens_seen": 499122176
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858495486459378,
+ "loss": 1.4013,
+ "theoretical_loss": 3.9440063238433045,
+ "tokens_seen": 499187712
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858335005015046,
+ "loss": 1.4952,
+ "theoretical_loss": 3.9439509444234258,
+ "tokens_seen": 499253248
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006858174523570712,
+ "loss": 1.4168,
+ "theoretical_loss": 3.943895574307788,
+ "tokens_seen": 499318784
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685801404212638,
+ "loss": 1.4904,
+ "theoretical_loss": 3.9438402134936057,
+ "tokens_seen": 499384320
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006857853560682046,
+ "loss": 1.4545,
+ "theoretical_loss": 3.9437848619780973,
+ "tokens_seen": 499449856
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006857693079237714,
+ "loss": 1.5395,
+ "theoretical_loss": 3.94372951975848,
+ "tokens_seen": 499515392
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685753259779338,
+ "loss": 1.5102,
+ "theoretical_loss": 3.9436741868319745,
+ "tokens_seen": 499580928
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006857372116349048,
+ "loss": 1.5157,
+ "theoretical_loss": 3.943618863195801,
+ "tokens_seen": 499646464
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006857211634904714,
+ "loss": 1.4785,
+ "theoretical_loss": 3.943563548847181,
+ "tokens_seen": 499712000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006857051153460382,
+ "loss": 1.6383,
+ "theoretical_loss": 3.943508243783338,
+ "tokens_seen": 499777536
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006856890672016048,
+ "loss": 1.5405,
+ "theoretical_loss": 3.9434529480014966,
+ "tokens_seen": 499843072
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006856730190571716,
+ "loss": 1.4985,
+ "theoretical_loss": 3.9433976614988815,
+ "tokens_seen": 499908608
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006856569709127382,
+ "loss": 1.4688,
+ "theoretical_loss": 3.9433423842727198,
+ "tokens_seen": 499974144
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685640922768305,
+ "loss": 1.5811,
+ "theoretical_loss": 3.9432871163202394,
+ "tokens_seen": 500039680
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006856248746238716,
+ "loss": 1.4328,
+ "theoretical_loss": 3.9432318576386693,
+ "tokens_seen": 500105216
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006856088264794384,
+ "loss": 1.3487,
+ "theoretical_loss": 3.94317660822524,
+ "tokens_seen": 500170752
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685592778335005,
+ "loss": 1.5404,
+ "theoretical_loss": 3.9431213680771826,
+ "tokens_seen": 500236288
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006855767301905718,
+ "loss": 1.6125,
+ "theoretical_loss": 3.94306613719173,
+ "tokens_seen": 500301824
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006855606820461384,
+ "loss": 1.5333,
+ "theoretical_loss": 3.9430109155661164,
+ "tokens_seen": 500367360
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006855446339017052,
+ "loss": 1.5013,
+ "theoretical_loss": 3.9429557031975753,
+ "tokens_seen": 500432896
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006855285857572718,
+ "loss": 1.3822,
+ "theoretical_loss": 3.9429005000833444,
+ "tokens_seen": 500498432
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006855125376128386,
+ "loss": 1.3441,
+ "theoretical_loss": 3.9428453062206605,
+ "tokens_seen": 500563968
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006854964894684052,
+ "loss": 1.5172,
+ "theoretical_loss": 3.942790121606763,
+ "tokens_seen": 500629504
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685480441323972,
+ "loss": 1.522,
+ "theoretical_loss": 3.9427349462388905,
+ "tokens_seen": 500695040
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006854643931795386,
+ "loss": 1.5221,
+ "theoretical_loss": 3.942679780114284,
+ "tokens_seen": 500760576
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006854483450351054,
+ "loss": 1.5553,
+ "theoretical_loss": 3.9426246232301865,
+ "tokens_seen": 500826112
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685432296890672,
+ "loss": 1.4746,
+ "theoretical_loss": 3.9425694755838405,
+ "tokens_seen": 500891648
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006854162487462388,
+ "loss": 1.5228,
+ "theoretical_loss": 3.9425143371724913,
+ "tokens_seen": 500957184
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006854002006018054,
+ "loss": 1.5312,
+ "theoretical_loss": 3.942459207993384,
+ "tokens_seen": 501022720
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006853841524573722,
+ "loss": 1.4847,
+ "theoretical_loss": 3.9424040880437654,
+ "tokens_seen": 501088256
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006853681043129388,
+ "loss": 1.5158,
+ "theoretical_loss": 3.9423489773208837,
+ "tokens_seen": 501153792
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006853520561685056,
+ "loss": 1.5185,
+ "theoretical_loss": 3.9422938758219885,
+ "tokens_seen": 501219328
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006853360080240722,
+ "loss": 1.5378,
+ "theoretical_loss": 3.9422387835443296,
+ "tokens_seen": 501284864
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 173143,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4856337308883667,
+ "objective/train/theoretical_loss": 3.9421837004851588,
+ "objective/train/tokens_used": 521810400,
+ "theoretical_loss": 3.9421837004851588,
+ "tokens_seen": 501350400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685319959879639,
+ "loss": 1.4838,
+ "theoretical_loss": 3.9421837004851588,
+ "tokens_seen": 501350400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006853039117352056,
+ "loss": 1.4919,
+ "theoretical_loss": 3.942128626641729,
+ "tokens_seen": 501415936
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006852878635907724,
+ "loss": 1.5944,
+ "theoretical_loss": 3.942073562011294,
+ "tokens_seen": 501481472
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685271815446339,
+ "loss": 1.4147,
+ "theoretical_loss": 3.9420185065911086,
+ "tokens_seen": 501547008
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006852557673019058,
+ "loss": 1.5705,
+ "theoretical_loss": 3.94196346037843,
+ "tokens_seen": 501612544
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006852397191574724,
+ "loss": 1.4792,
+ "theoretical_loss": 3.941908423370515,
+ "tokens_seen": 501678080
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006852236710130392,
+ "loss": 1.4602,
+ "theoretical_loss": 3.941853395564622,
+ "tokens_seen": 501743616
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006852076228686058,
+ "loss": 1.4439,
+ "theoretical_loss": 3.9417983769580114,
+ "tokens_seen": 501809152
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006851915747241726,
+ "loss": 1.5512,
+ "theoretical_loss": 3.941743367547944,
+ "tokens_seen": 501874688
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006851755265797392,
+ "loss": 1.416,
+ "theoretical_loss": 3.9416883673316816,
+ "tokens_seen": 501940224
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685159478435306,
+ "loss": 1.5061,
+ "theoretical_loss": 3.9416333763064886,
+ "tokens_seen": 502005760
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006851434302908726,
+ "loss": 1.4738,
+ "theoretical_loss": 3.941578394469628,
+ "tokens_seen": 502071296
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006851273821464394,
+ "loss": 1.4258,
+ "theoretical_loss": 3.941523421818366,
+ "tokens_seen": 502136832
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000685111334002006,
+ "loss": 1.5415,
+ "theoretical_loss": 3.94146845834997,
+ "tokens_seen": 502202368
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850952858575728,
+ "loss": 1.4216,
+ "theoretical_loss": 3.9414135040617078,
+ "tokens_seen": 502267904
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850792377131394,
+ "loss": 1.3867,
+ "theoretical_loss": 3.9413585589508484,
+ "tokens_seen": 502333440
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850631895687062,
+ "loss": 1.4369,
+ "theoretical_loss": 3.9413036230146625,
+ "tokens_seen": 502398976
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850471414242728,
+ "loss": 1.5165,
+ "theoretical_loss": 3.941248696250421,
+ "tokens_seen": 502464512
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850310932798396,
+ "loss": 1.5472,
+ "theoretical_loss": 3.941193778655397,
+ "tokens_seen": 502530048
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006850150451354062,
+ "loss": 1.5286,
+ "theoretical_loss": 3.9411388702268644,
+ "tokens_seen": 502595584
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000684998996990973,
+ "loss": 1.6273,
+ "theoretical_loss": 3.941083970962098,
+ "tokens_seen": 502661120
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006849829488465396,
+ "loss": 1.4459,
+ "theoretical_loss": 3.9410290808583746,
+ "tokens_seen": 502726656
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006849669007021064,
+ "loss": 1.4898,
+ "theoretical_loss": 3.9409741999129704,
+ "tokens_seen": 502792192
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000684950852557673,
+ "loss": 1.4672,
+ "theoretical_loss": 3.9409193281231647,
+ "tokens_seen": 502857728
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006849348044132398,
+ "loss": 1.4665,
+ "theoretical_loss": 3.9408644654862375,
+ "tokens_seen": 502923264
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006849187562688064,
+ "loss": 1.6371,
+ "theoretical_loss": 3.9408096119994687,
+ "tokens_seen": 502988800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006849027081243732,
+ "loss": 1.5104,
+ "theoretical_loss": 3.940754767660141,
+ "tokens_seen": 503054336
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006848866599799398,
+ "loss": 1.4865,
+ "theoretical_loss": 3.940699932465537,
+ "tokens_seen": 503119872
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006848706118355066,
+ "loss": 1.4007,
+ "theoretical_loss": 3.9406451064129415,
+ "tokens_seen": 503185408
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006848545636910732,
+ "loss": 1.5524,
+ "theoretical_loss": 3.94059028949964,
+ "tokens_seen": 503250944
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.00068483851554664,
+ "loss": 1.541,
+ "theoretical_loss": 3.940535481722919,
+ "tokens_seen": 503316480
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006848224674022066,
+ "loss": 1.4049,
+ "theoretical_loss": 3.9404806830800667,
+ "tokens_seen": 503382016
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006848064192577734,
+ "loss": 1.4502,
+ "theoretical_loss": 3.940425893568371,
+ "tokens_seen": 503447552
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.00068479037111334,
+ "loss": 1.4578,
+ "theoretical_loss": 3.9403711131851233,
+ "tokens_seen": 503513088
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006847743229689068,
+ "loss": 1.4604,
+ "theoretical_loss": 3.940316341927614,
+ "tokens_seen": 503578624
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006847582748244734,
+ "loss": 1.507,
+ "theoretical_loss": 3.9402615797931357,
+ "tokens_seen": 503644160
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006847422266800402,
+ "loss": 1.5525,
+ "theoretical_loss": 3.940206826778982,
+ "tokens_seen": 503709696
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006847261785356068,
+ "loss": 1.4689,
+ "theoretical_loss": 3.9401520828824483,
+ "tokens_seen": 503775232
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006847101303911736,
+ "loss": 1.6258,
+ "theoretical_loss": 3.9400973481008297,
+ "tokens_seen": 503840768
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006846940822467402,
+ "loss": 1.5935,
+ "theoretical_loss": 3.9400426224314233,
+ "tokens_seen": 503906304
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000684678034102307,
+ "loss": 1.4517,
+ "theoretical_loss": 3.9399879058715275,
+ "tokens_seen": 503971840
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006846619859578737,
+ "loss": 1.4911,
+ "theoretical_loss": 3.9399331984184416,
+ "tokens_seen": 504037376
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006846459378134404,
+ "loss": 1.5333,
+ "theoretical_loss": 3.939878500069466,
+ "tokens_seen": 504102912
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000684629889669007,
+ "loss": 1.5188,
+ "theoretical_loss": 3.939823810821903,
+ "tokens_seen": 504168448
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006846138415245737,
+ "loss": 1.5142,
+ "theoretical_loss": 3.9397691306730547,
+ "tokens_seen": 504233984
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845977933801405,
+ "loss": 1.5587,
+ "theoretical_loss": 3.9397144596202254,
+ "tokens_seen": 504299520
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845817452357071,
+ "loss": 1.4182,
+ "theoretical_loss": 3.93965979766072,
+ "tokens_seen": 504365056
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845656970912739,
+ "loss": 1.393,
+ "theoretical_loss": 3.9396051447918445,
+ "tokens_seen": 504430592
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845496489468405,
+ "loss": 1.4089,
+ "theoretical_loss": 3.939550501010907,
+ "tokens_seen": 504496128
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845336008024073,
+ "loss": 1.4498,
+ "theoretical_loss": 3.939495866315216,
+ "tokens_seen": 504561664
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 174029,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3834893703460693,
+ "objective/train/theoretical_loss": 3.93944124070208,
+ "objective/train/tokens_used": 525087200,
+ "theoretical_loss": 3.93944124070208,
+ "tokens_seen": 504627200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845175526579739,
+ "loss": 1.4513,
+ "theoretical_loss": 3.93944124070208,
+ "tokens_seen": 504627200
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006845015045135407,
+ "loss": 1.5259,
+ "theoretical_loss": 3.9393866241688116,
+ "tokens_seen": 504692736
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844854563691073,
+ "loss": 1.4621,
+ "theoretical_loss": 3.9393320167127213,
+ "tokens_seen": 504758272
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844694082246741,
+ "loss": 1.4741,
+ "theoretical_loss": 3.939277418331123,
+ "tokens_seen": 504823808
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844533600802407,
+ "loss": 1.5203,
+ "theoretical_loss": 3.9392228290213316,
+ "tokens_seen": 504889344
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844373119358075,
+ "loss": 1.5894,
+ "theoretical_loss": 3.939168248780661,
+ "tokens_seen": 504954880
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844212637913741,
+ "loss": 1.4675,
+ "theoretical_loss": 3.9391136776064286,
+ "tokens_seen": 505020416
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006844052156469409,
+ "loss": 1.5257,
+ "theoretical_loss": 3.939059115495952,
+ "tokens_seen": 505085952
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843891675025075,
+ "loss": 1.4885,
+ "theoretical_loss": 3.93900456244655,
+ "tokens_seen": 505151488
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843731193580743,
+ "loss": 1.4047,
+ "theoretical_loss": 3.938950018455543,
+ "tokens_seen": 505217024
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843570712136409,
+ "loss": 1.4484,
+ "theoretical_loss": 3.938895483520252,
+ "tokens_seen": 505282560
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843410230692077,
+ "loss": 1.4234,
+ "theoretical_loss": 3.938840957637998,
+ "tokens_seen": 505348096
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843249749247743,
+ "loss": 1.5042,
+ "theoretical_loss": 3.9387864408061066,
+ "tokens_seen": 505413632
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006843089267803411,
+ "loss": 1.3835,
+ "theoretical_loss": 3.9387319330219004,
+ "tokens_seen": 505479168
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842928786359077,
+ "loss": 1.4732,
+ "theoretical_loss": 3.938677434282706,
+ "tokens_seen": 505544704
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842768304914745,
+ "loss": 1.5227,
+ "theoretical_loss": 3.93862294458585,
+ "tokens_seen": 505610240
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842607823470411,
+ "loss": 1.5028,
+ "theoretical_loss": 3.938568463928661,
+ "tokens_seen": 505675776
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842447342026079,
+ "loss": 1.5603,
+ "theoretical_loss": 3.9385139923084678,
+ "tokens_seen": 505741312
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842286860581745,
+ "loss": 1.5817,
+ "theoretical_loss": 3.9384595297225995,
+ "tokens_seen": 505806848
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006842126379137413,
+ "loss": 1.4385,
+ "theoretical_loss": 3.9384050761683884,
+ "tokens_seen": 505872384
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841965897693079,
+ "loss": 1.4871,
+ "theoretical_loss": 3.938350631643168,
+ "tokens_seen": 505937920
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841805416248747,
+ "loss": 1.5354,
+ "theoretical_loss": 3.9382961961442695,
+ "tokens_seen": 506003456
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841644934804413,
+ "loss": 1.593,
+ "theoretical_loss": 3.93824176966903,
+ "tokens_seen": 506068992
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841484453360081,
+ "loss": 1.6305,
+ "theoretical_loss": 3.9381873522147837,
+ "tokens_seen": 506134528
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841323971915747,
+ "loss": 1.5211,
+ "theoretical_loss": 3.938132943778869,
+ "tokens_seen": 506200064
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841163490471415,
+ "loss": 1.5019,
+ "theoretical_loss": 3.938078544358623,
+ "tokens_seen": 506265600
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006841003009027081,
+ "loss": 1.4484,
+ "theoretical_loss": 3.9380241539513854,
+ "tokens_seen": 506331136
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840842527582749,
+ "loss": 1.5122,
+ "theoretical_loss": 3.9379697725544967,
+ "tokens_seen": 506396672
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840682046138415,
+ "loss": 1.5404,
+ "theoretical_loss": 3.9379154001652985,
+ "tokens_seen": 506462208
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840521564694083,
+ "loss": 1.4067,
+ "theoretical_loss": 3.9378610367811326,
+ "tokens_seen": 506527744
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840361083249749,
+ "loss": 1.5415,
+ "theoretical_loss": 3.9378066823993443,
+ "tokens_seen": 506593280
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840200601805417,
+ "loss": 1.5637,
+ "theoretical_loss": 3.937752337017278,
+ "tokens_seen": 506658816
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006840040120361083,
+ "loss": 1.4408,
+ "theoretical_loss": 3.9376980006322784,
+ "tokens_seen": 506724352
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839879638916751,
+ "loss": 1.4735,
+ "theoretical_loss": 3.937643673241695,
+ "tokens_seen": 506789888
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839719157472417,
+ "loss": 1.3714,
+ "theoretical_loss": 3.9375893548428738,
+ "tokens_seen": 506855424
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839558676028085,
+ "loss": 1.4486,
+ "theoretical_loss": 3.9375350454331657,
+ "tokens_seen": 506920960
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839398194583751,
+ "loss": 1.4675,
+ "theoretical_loss": 3.937480745009921,
+ "tokens_seen": 506986496
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839237713139419,
+ "loss": 1.5516,
+ "theoretical_loss": 3.937426453570491,
+ "tokens_seen": 507052032
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006839077231695085,
+ "loss": 1.3864,
+ "theoretical_loss": 3.9373721711122287,
+ "tokens_seen": 507117568
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838916750250753,
+ "loss": 1.4253,
+ "theoretical_loss": 3.937317897632488,
+ "tokens_seen": 507183104
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838756268806419,
+ "loss": 1.4409,
+ "theoretical_loss": 3.9372636331286244,
+ "tokens_seen": 507248640
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838595787362087,
+ "loss": 1.5317,
+ "theoretical_loss": 3.937209377597993,
+ "tokens_seen": 507314176
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838435305917753,
+ "loss": 1.4448,
+ "theoretical_loss": 3.9371551310379527,
+ "tokens_seen": 507379712
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838274824473421,
+ "loss": 1.5911,
+ "theoretical_loss": 3.93710089344586,
+ "tokens_seen": 507445248
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006838114343029087,
+ "loss": 1.4736,
+ "theoretical_loss": 3.937046664819076,
+ "tokens_seen": 507510784
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837953861584755,
+ "loss": 1.4326,
+ "theoretical_loss": 3.93699244515496,
+ "tokens_seen": 507576320
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837793380140421,
+ "loss": 1.3809,
+ "theoretical_loss": 3.936938234450875,
+ "tokens_seen": 507641856
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837632898696089,
+ "loss": 1.4961,
+ "theoretical_loss": 3.936884032704183,
+ "tokens_seen": 507707392
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837472417251755,
+ "loss": 1.4297,
+ "theoretical_loss": 3.9368298399122486,
+ "tokens_seen": 507772928
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837311935807423,
+ "loss": 1.5158,
+ "theoretical_loss": 3.9367756560724363,
+ "tokens_seen": 507838464
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 175381,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2951419353485107,
+ "objective/train/theoretical_loss": 3.936721481182113,
+ "objective/train/tokens_used": 528364000,
+ "theoretical_loss": 3.936721481182113,
+ "tokens_seen": 507904000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006837151454363089,
+ "loss": 1.4057,
+ "theoretical_loss": 3.936721481182113,
+ "tokens_seen": 507904000
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836990972918757,
+ "loss": 1.4186,
+ "theoretical_loss": 3.9366673152386458,
+ "tokens_seen": 507969536
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836830491474423,
+ "loss": 1.4403,
+ "theoretical_loss": 3.9366131582394024,
+ "tokens_seen": 508035072
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836670010030091,
+ "loss": 1.5401,
+ "theoretical_loss": 3.936559010181753,
+ "tokens_seen": 508100608
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836509528585757,
+ "loss": 1.4831,
+ "theoretical_loss": 3.936504871063069,
+ "tokens_seen": 508166144
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836349047141425,
+ "loss": 1.4929,
+ "theoretical_loss": 3.9364507408807206,
+ "tokens_seen": 508231680
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836188565697091,
+ "loss": 1.4945,
+ "theoretical_loss": 3.936396619632082,
+ "tokens_seen": 508297216
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006836028084252759,
+ "loss": 1.4826,
+ "theoretical_loss": 3.936342507314527,
+ "tokens_seen": 508362752
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835867602808425,
+ "loss": 1.459,
+ "theoretical_loss": 3.9362884039254302,
+ "tokens_seen": 508428288
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835707121364093,
+ "loss": 1.3537,
+ "theoretical_loss": 3.9362343094621677,
+ "tokens_seen": 508493824
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835546639919759,
+ "loss": 1.4776,
+ "theoretical_loss": 3.936180223922118,
+ "tokens_seen": 508559360
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835386158475427,
+ "loss": 1.5716,
+ "theoretical_loss": 3.936126147302658,
+ "tokens_seen": 508624896
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835225677031093,
+ "loss": 1.5438,
+ "theoretical_loss": 3.9360720796011686,
+ "tokens_seen": 508690432
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006835065195586761,
+ "loss": 1.5372,
+ "theoretical_loss": 3.93601802081503,
+ "tokens_seen": 508755968
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834904714142427,
+ "loss": 1.4193,
+ "theoretical_loss": 3.9359639709416236,
+ "tokens_seen": 508821504
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834744232698095,
+ "loss": 1.5186,
+ "theoretical_loss": 3.9359099299783327,
+ "tokens_seen": 508887040
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834583751253763,
+ "loss": 1.4852,
+ "theoretical_loss": 3.935855897922541,
+ "tokens_seen": 508952576
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834423269809429,
+ "loss": 1.4447,
+ "theoretical_loss": 3.935801874771634,
+ "tokens_seen": 509018112
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834262788365097,
+ "loss": 1.4547,
+ "theoretical_loss": 3.935747860522997,
+ "tokens_seen": 509083648
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006834102306920763,
+ "loss": 1.4285,
+ "theoretical_loss": 3.9356938551740184,
+ "tokens_seen": 509149184
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833941825476431,
+ "loss": 1.5165,
+ "theoretical_loss": 3.9356398587220864,
+ "tokens_seen": 509214720
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833781344032097,
+ "loss": 1.5487,
+ "theoretical_loss": 3.9355858711645895,
+ "tokens_seen": 509280256
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833620862587765,
+ "loss": 1.5374,
+ "theoretical_loss": 3.9355318924989193,
+ "tokens_seen": 509345792
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833460381143431,
+ "loss": 1.3921,
+ "theoretical_loss": 3.9354779227224674,
+ "tokens_seen": 509411328
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833299899699097,
+ "loss": 1.4561,
+ "theoretical_loss": 3.935423961832626,
+ "tokens_seen": 509476864
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006833139418254765,
+ "loss": 1.5147,
+ "theoretical_loss": 3.93537000982679,
+ "tokens_seen": 509542400
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832978936810431,
+ "loss": 1.4813,
+ "theoretical_loss": 3.9353160667023532,
+ "tokens_seen": 509607936
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832818455366099,
+ "loss": 1.4487,
+ "theoretical_loss": 3.935262132456713,
+ "tokens_seen": 509673472
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832657973921766,
+ "loss": 1.4581,
+ "theoretical_loss": 3.9352082070872654,
+ "tokens_seen": 509739008
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832497492477433,
+ "loss": 1.5631,
+ "theoretical_loss": 3.9351542905914094,
+ "tokens_seen": 509804544
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.00068323370110331,
+ "loss": 1.4613,
+ "theoretical_loss": 3.935100382966544,
+ "tokens_seen": 509870080
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832176529588766,
+ "loss": 1.4773,
+ "theoretical_loss": 3.93504648421007,
+ "tokens_seen": 509935616
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006832016048144434,
+ "loss": 1.49,
+ "theoretical_loss": 3.9349925943193886,
+ "tokens_seen": 510001152
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.00068318555667001,
+ "loss": 1.457,
+ "theoretical_loss": 3.934938713291903,
+ "tokens_seen": 510066688
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006831695085255768,
+ "loss": 1.4519,
+ "theoretical_loss": 3.934884841125017,
+ "tokens_seen": 510132224
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006831534603811434,
+ "loss": 1.3485,
+ "theoretical_loss": 3.9348309778161346,
+ "tokens_seen": 510197760
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006831374122367102,
+ "loss": 1.4214,
+ "theoretical_loss": 3.9347771233626627,
+ "tokens_seen": 510263296
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006831213640922768,
+ "loss": 1.365,
+ "theoretical_loss": 3.934723277762008,
+ "tokens_seen": 510328832
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006831053159478436,
+ "loss": 1.4693,
+ "theoretical_loss": 3.934669441011579,
+ "tokens_seen": 510394368
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006830892678034102,
+ "loss": 1.4993,
+ "theoretical_loss": 3.934615613108784,
+ "tokens_seen": 510459904
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000683073219658977,
+ "loss": 1.4738,
+ "theoretical_loss": 3.9345617940510342,
+ "tokens_seen": 510525440
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006830571715145436,
+ "loss": 1.3746,
+ "theoretical_loss": 3.934507983835741,
+ "tokens_seen": 510590976
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006830411233701104,
+ "loss": 1.4135,
+ "theoretical_loss": 3.9344541824603163,
+ "tokens_seen": 510656512
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000683025075225677,
+ "loss": 1.5538,
+ "theoretical_loss": 3.934400389922174,
+ "tokens_seen": 510722048
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006830090270812438,
+ "loss": 1.5079,
+ "theoretical_loss": 3.934346606218729,
+ "tokens_seen": 510787584
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006829929789368104,
+ "loss": 1.5737,
+ "theoretical_loss": 3.9342928313473964,
+ "tokens_seen": 510853120
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006829769307923772,
+ "loss": 1.4594,
+ "theoretical_loss": 3.9342390653055945,
+ "tokens_seen": 510918656
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006829608826479438,
+ "loss": 1.4723,
+ "theoretical_loss": 3.93418530809074,
+ "tokens_seen": 510984192
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006829448345035106,
+ "loss": 1.4413,
+ "theoretical_loss": 3.9341315597002517,
+ "tokens_seen": 511049728
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006829287863590772,
+ "loss": 1.4883,
+ "theoretical_loss": 3.934077820131551,
+ "tokens_seen": 511115264
+ },
+ {
+ "epoch": 0.15,
+ "objective/train/docs_used": 176389,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4067481756210327,
+ "objective/train/theoretical_loss": 3.934024089382058,
+ "objective/train/tokens_used": 531640800,
+ "theoretical_loss": 3.934024089382058,
+ "tokens_seen": 511180800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000682912738214644,
+ "loss": 1.4437,
+ "theoretical_loss": 3.934024089382058,
+ "tokens_seen": 511180800
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006828966900702106,
+ "loss": 1.4399,
+ "theoretical_loss": 3.933970367449195,
+ "tokens_seen": 511246336
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006828806419257774,
+ "loss": 1.5355,
+ "theoretical_loss": 3.9339166543303863,
+ "tokens_seen": 511311872
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.000682864593781344,
+ "loss": 1.4865,
+ "theoretical_loss": 3.9338629500230553,
+ "tokens_seen": 511377408
+ },
+ {
+ "epoch": 0.15,
+ "learning_rate": 0.0006828485456369108,
+ "loss": 1.4984,
+ "theoretical_loss": 3.9338092545246286,
+ "tokens_seen": 511442944
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006828324974924774,
+ "loss": 1.4916,
+ "theoretical_loss": 3.933755567832532,
+ "tokens_seen": 511508480
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006828164493480442,
+ "loss": 1.5456,
+ "theoretical_loss": 3.933701889944194,
+ "tokens_seen": 511574016
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006828004012036108,
+ "loss": 1.5706,
+ "theoretical_loss": 3.933648220857042,
+ "tokens_seen": 511639552
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006827843530591776,
+ "loss": 1.4834,
+ "theoretical_loss": 3.9335945605685074,
+ "tokens_seen": 511705088
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006827683049147442,
+ "loss": 1.4373,
+ "theoretical_loss": 3.9335409090760196,
+ "tokens_seen": 511770624
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682752256770311,
+ "loss": 1.443,
+ "theoretical_loss": 3.933487266377012,
+ "tokens_seen": 511836160
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006827362086258776,
+ "loss": 1.4371,
+ "theoretical_loss": 3.933433632468917,
+ "tokens_seen": 511901696
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006827201604814444,
+ "loss": 1.5022,
+ "theoretical_loss": 3.9333800073491694,
+ "tokens_seen": 511967232
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682704112337011,
+ "loss": 1.4562,
+ "theoretical_loss": 3.9333263910152034,
+ "tokens_seen": 512032768
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826880641925778,
+ "loss": 1.441,
+ "theoretical_loss": 3.9332727834644565,
+ "tokens_seen": 512098304
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826720160481444,
+ "loss": 1.4132,
+ "theoretical_loss": 3.933219184694365,
+ "tokens_seen": 512163840
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826559679037112,
+ "loss": 1.5127,
+ "theoretical_loss": 3.933165594702368,
+ "tokens_seen": 512229376
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826399197592778,
+ "loss": 1.4125,
+ "theoretical_loss": 3.9331120134859052,
+ "tokens_seen": 512294912
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826238716148446,
+ "loss": 1.4094,
+ "theoretical_loss": 3.9330584410424168,
+ "tokens_seen": 512360448
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006826078234704112,
+ "loss": 1.5539,
+ "theoretical_loss": 3.9330048773693447,
+ "tokens_seen": 512425984
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682591775325978,
+ "loss": 1.5014,
+ "theoretical_loss": 3.932951322464132,
+ "tokens_seen": 512491520
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006825757271815446,
+ "loss": 1.4989,
+ "theoretical_loss": 3.9328977763242223,
+ "tokens_seen": 512557056
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006825596790371114,
+ "loss": 1.5426,
+ "theoretical_loss": 3.93284423894706,
+ "tokens_seen": 512622592
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682543630892678,
+ "loss": 1.4491,
+ "theoretical_loss": 3.932790710330092,
+ "tokens_seen": 512688128
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006825275827482448,
+ "loss": 1.4457,
+ "theoretical_loss": 3.932737190470765,
+ "tokens_seen": 512753664
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006825115346038114,
+ "loss": 1.4772,
+ "theoretical_loss": 3.9326836793665265,
+ "tokens_seen": 512819200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006824954864593782,
+ "loss": 1.6298,
+ "theoretical_loss": 3.9326301770148264,
+ "tokens_seen": 512884736
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006824794383149448,
+ "loss": 1.5865,
+ "theoretical_loss": 3.9325766834131155,
+ "tokens_seen": 512950272
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006824633901705116,
+ "loss": 1.4053,
+ "theoretical_loss": 3.932523198558844,
+ "tokens_seen": 513015808
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006824473420260782,
+ "loss": 1.3849,
+ "theoretical_loss": 3.932469722449464,
+ "tokens_seen": 513081344
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682431293881645,
+ "loss": 1.3922,
+ "theoretical_loss": 3.932416255082431,
+ "tokens_seen": 513146880
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006824152457372116,
+ "loss": 1.4653,
+ "theoretical_loss": 3.932362796455198,
+ "tokens_seen": 513212416
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823991975927784,
+ "loss": 1.3947,
+ "theoretical_loss": 3.932309346565221,
+ "tokens_seen": 513277952
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682383149448345,
+ "loss": 1.4863,
+ "theoretical_loss": 3.9322559054099564,
+ "tokens_seen": 513343488
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823671013039118,
+ "loss": 1.5967,
+ "theoretical_loss": 3.932202472986862,
+ "tokens_seen": 513409024
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823510531594784,
+ "loss": 1.4689,
+ "theoretical_loss": 3.932149049293397,
+ "tokens_seen": 513474560
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823350050150452,
+ "loss": 1.3834,
+ "theoretical_loss": 3.9320956343270206,
+ "tokens_seen": 513540096
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823189568706118,
+ "loss": 1.4431,
+ "theoretical_loss": 3.9320422280851943,
+ "tokens_seen": 513605632
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006823029087261786,
+ "loss": 1.4312,
+ "theoretical_loss": 3.9319888305653805,
+ "tokens_seen": 513671168
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006822868605817452,
+ "loss": 1.4436,
+ "theoretical_loss": 3.9319354417650416,
+ "tokens_seen": 513736704
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682270812437312,
+ "loss": 1.4754,
+ "theoretical_loss": 3.931882061681641,
+ "tokens_seen": 513802240
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006822547642928787,
+ "loss": 1.4687,
+ "theoretical_loss": 3.9318286903126456,
+ "tokens_seen": 513867776
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006822387161484454,
+ "loss": 1.536,
+ "theoretical_loss": 3.93177532765552,
+ "tokens_seen": 513933312
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006822226680040121,
+ "loss": 1.4208,
+ "theoretical_loss": 3.9317219737077327,
+ "tokens_seen": 513998848
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006822066198595788,
+ "loss": 1.5384,
+ "theoretical_loss": 3.9316686284667517,
+ "tokens_seen": 514064384
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006821905717151455,
+ "loss": 1.5302,
+ "theoretical_loss": 3.9316152919300458,
+ "tokens_seen": 514129920
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006821745235707122,
+ "loss": 1.5152,
+ "theoretical_loss": 3.9315619640950863,
+ "tokens_seen": 514195456
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682158475426279,
+ "loss": 1.4262,
+ "theoretical_loss": 3.9315086449593446,
+ "tokens_seen": 514260992
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006821424272818456,
+ "loss": 1.5165,
+ "theoretical_loss": 3.931455334520293,
+ "tokens_seen": 514326528
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006821263791374124,
+ "loss": 1.5204,
+ "theoretical_loss": 3.931402032775405,
+ "tokens_seen": 514392064
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 177587,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4545185565948486,
+ "objective/train/theoretical_loss": 3.9313487397221554,
+ "objective/train/tokens_used": 534917600,
+ "theoretical_loss": 3.9313487397221554,
+ "tokens_seen": 514457600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000682110330992979,
+ "loss": 1.442,
+ "theoretical_loss": 3.9313487397221554,
+ "tokens_seen": 514457600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820942828485458,
+ "loss": 1.4798,
+ "theoretical_loss": 3.9312954553580206,
+ "tokens_seen": 514523136
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820782347041124,
+ "loss": 1.5157,
+ "theoretical_loss": 3.931242179680477,
+ "tokens_seen": 514588672
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820621865596792,
+ "loss": 1.4057,
+ "theoretical_loss": 3.931188912687002,
+ "tokens_seen": 514654208
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820461384152458,
+ "loss": 1.4713,
+ "theoretical_loss": 3.931135654375075,
+ "tokens_seen": 514719744
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820300902708126,
+ "loss": 1.4986,
+ "theoretical_loss": 3.9310824047421757,
+ "tokens_seen": 514785280
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006820140421263792,
+ "loss": 1.5335,
+ "theoretical_loss": 3.9310291637857855,
+ "tokens_seen": 514850816
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000681997993981946,
+ "loss": 1.4282,
+ "theoretical_loss": 3.930975931503386,
+ "tokens_seen": 514916352
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006819819458375126,
+ "loss": 1.5608,
+ "theoretical_loss": 3.930922707892461,
+ "tokens_seen": 514981888
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006819658976930794,
+ "loss": 1.3664,
+ "theoretical_loss": 3.930869492950494,
+ "tokens_seen": 515047424
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000681949849548646,
+ "loss": 1.4341,
+ "theoretical_loss": 3.9308162866749705,
+ "tokens_seen": 515112960
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006819338014042126,
+ "loss": 1.3829,
+ "theoretical_loss": 3.930763089063377,
+ "tokens_seen": 515178496
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006819177532597794,
+ "loss": 1.517,
+ "theoretical_loss": 3.9307099001132,
+ "tokens_seen": 515244032
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000681901705115346,
+ "loss": 1.5803,
+ "theoretical_loss": 3.930656719821929,
+ "tokens_seen": 515309568
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818856569709128,
+ "loss": 1.4556,
+ "theoretical_loss": 3.9306035481870527,
+ "tokens_seen": 515375104
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818696088264795,
+ "loss": 1.5231,
+ "theoretical_loss": 3.9305503852060615,
+ "tokens_seen": 515440640
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818535606820462,
+ "loss": 1.5268,
+ "theoretical_loss": 3.9304972308764476,
+ "tokens_seen": 515506176
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818375125376129,
+ "loss": 1.4497,
+ "theoretical_loss": 3.9304440851957025,
+ "tokens_seen": 515571712
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818214643931796,
+ "loss": 1.5811,
+ "theoretical_loss": 3.930390948161321,
+ "tokens_seen": 515637248
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006818054162487463,
+ "loss": 1.5048,
+ "theoretical_loss": 3.9303378197707968,
+ "tokens_seen": 515702784
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817893681043129,
+ "loss": 1.4945,
+ "theoretical_loss": 3.9302847000216263,
+ "tokens_seen": 515768320
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817733199598797,
+ "loss": 1.5569,
+ "theoretical_loss": 3.9302315889113055,
+ "tokens_seen": 515833856
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817572718154463,
+ "loss": 1.4553,
+ "theoretical_loss": 3.930178486437333,
+ "tokens_seen": 515899392
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817412236710131,
+ "loss": 1.3959,
+ "theoretical_loss": 3.930125392597207,
+ "tokens_seen": 515964928
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817251755265797,
+ "loss": 1.3709,
+ "theoretical_loss": 3.9300723073884276,
+ "tokens_seen": 516030464
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006817091273821465,
+ "loss": 1.4881,
+ "theoretical_loss": 3.9300192308084956,
+ "tokens_seen": 516096000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816930792377131,
+ "loss": 1.447,
+ "theoretical_loss": 3.929966162854913,
+ "tokens_seen": 516161536
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816770310932799,
+ "loss": 1.5391,
+ "theoretical_loss": 3.929913103525183,
+ "tokens_seen": 516227072
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816609829488465,
+ "loss": 1.4131,
+ "theoretical_loss": 3.929860052816809,
+ "tokens_seen": 516292608
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816449348044133,
+ "loss": 1.5676,
+ "theoretical_loss": 3.9298070107272967,
+ "tokens_seen": 516358144
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816288866599799,
+ "loss": 1.5717,
+ "theoretical_loss": 3.929753977254152,
+ "tokens_seen": 516423680
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006816128385155467,
+ "loss": 1.5477,
+ "theoretical_loss": 3.929700952394882,
+ "tokens_seen": 516489216
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815967903711133,
+ "loss": 1.616,
+ "theoretical_loss": 3.9296479361469947,
+ "tokens_seen": 516554752
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815807422266801,
+ "loss": 1.5208,
+ "theoretical_loss": 3.929594928508,
+ "tokens_seen": 516620288
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815646940822467,
+ "loss": 1.5779,
+ "theoretical_loss": 3.9295419294754077,
+ "tokens_seen": 516685824
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815486459378135,
+ "loss": 1.5485,
+ "theoretical_loss": 3.9294889390467285,
+ "tokens_seen": 516751360
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815325977933801,
+ "loss": 1.4398,
+ "theoretical_loss": 3.929435957219476,
+ "tokens_seen": 516816896
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815165496489469,
+ "loss": 1.4488,
+ "theoretical_loss": 3.9293829839911623,
+ "tokens_seen": 516882432
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006815005015045135,
+ "loss": 1.4929,
+ "theoretical_loss": 3.9293300193593024,
+ "tokens_seen": 516947968
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814844533600803,
+ "loss": 1.5023,
+ "theoretical_loss": 3.929277063321412,
+ "tokens_seen": 517013504
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814684052156469,
+ "loss": 1.5235,
+ "theoretical_loss": 3.929224115875007,
+ "tokens_seen": 517079040
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814523570712137,
+ "loss": 1.4841,
+ "theoretical_loss": 3.929171177017605,
+ "tokens_seen": 517144576
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814363089267803,
+ "loss": 1.5463,
+ "theoretical_loss": 3.9291182467467247,
+ "tokens_seen": 517210112
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814202607823471,
+ "loss": 1.4908,
+ "theoretical_loss": 3.9290653250598857,
+ "tokens_seen": 517275648
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006814042126379137,
+ "loss": 1.4972,
+ "theoretical_loss": 3.9290124119546084,
+ "tokens_seen": 517341184
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813881644934805,
+ "loss": 1.4464,
+ "theoretical_loss": 3.9289595074284143,
+ "tokens_seen": 517406720
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813721163490471,
+ "loss": 1.5389,
+ "theoretical_loss": 3.9289066114788267,
+ "tokens_seen": 517472256
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813560682046139,
+ "loss": 1.4443,
+ "theoretical_loss": 3.9288537241033685,
+ "tokens_seen": 517537792
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813400200601805,
+ "loss": 1.404,
+ "theoretical_loss": 3.9288008452995644,
+ "tokens_seen": 517603328
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813239719157473,
+ "loss": 1.5549,
+ "theoretical_loss": 3.9287479750649412,
+ "tokens_seen": 517668864
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 178536,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4981567859649658,
+ "objective/train/theoretical_loss": 3.9286951133970245,
+ "objective/train/tokens_used": 538194400,
+ "theoretical_loss": 3.9286951133970245,
+ "tokens_seen": 517734400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006813079237713139,
+ "loss": 1.4896,
+ "theoretical_loss": 3.9286951133970245,
+ "tokens_seen": 517734400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812918756268807,
+ "loss": 1.3853,
+ "theoretical_loss": 3.9286422602933424,
+ "tokens_seen": 517799936
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812758274824473,
+ "loss": 1.4381,
+ "theoretical_loss": 3.928589415751424,
+ "tokens_seen": 517865472
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812597793380141,
+ "loss": 1.4726,
+ "theoretical_loss": 3.9285365797687986,
+ "tokens_seen": 517931008
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812437311935807,
+ "loss": 1.5329,
+ "theoretical_loss": 3.9284837523429976,
+ "tokens_seen": 517996544
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812276830491475,
+ "loss": 1.3644,
+ "theoretical_loss": 3.9284309334715526,
+ "tokens_seen": 518062080
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006812116349047141,
+ "loss": 1.516,
+ "theoretical_loss": 3.9283781231519965,
+ "tokens_seen": 518127616
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811955867602809,
+ "loss": 1.5407,
+ "theoretical_loss": 3.9283253213818634,
+ "tokens_seen": 518193152
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811795386158475,
+ "loss": 1.5017,
+ "theoretical_loss": 3.9282725281586885,
+ "tokens_seen": 518258688
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811634904714143,
+ "loss": 1.5143,
+ "theoretical_loss": 3.928219743480007,
+ "tokens_seen": 518324224
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811474423269809,
+ "loss": 1.5884,
+ "theoretical_loss": 3.9281669673433566,
+ "tokens_seen": 518389760
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811313941825477,
+ "loss": 1.552,
+ "theoretical_loss": 3.9281141997462754,
+ "tokens_seen": 518455296
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006811153460381143,
+ "loss": 1.5766,
+ "theoretical_loss": 3.9280614406863013,
+ "tokens_seen": 518520832
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810992978936811,
+ "loss": 1.474,
+ "theoretical_loss": 3.9280086901609756,
+ "tokens_seen": 518586368
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810832497492477,
+ "loss": 1.4442,
+ "theoretical_loss": 3.927955948167839,
+ "tokens_seen": 518651904
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810672016048145,
+ "loss": 1.4731,
+ "theoretical_loss": 3.9279032147044335,
+ "tokens_seen": 518717440
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810511534603812,
+ "loss": 1.4481,
+ "theoretical_loss": 3.9278504897683018,
+ "tokens_seen": 518782976
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810351053159479,
+ "loss": 1.5885,
+ "theoretical_loss": 3.927797773356989,
+ "tokens_seen": 518848512
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810190571715146,
+ "loss": 1.3918,
+ "theoretical_loss": 3.9277450654680397,
+ "tokens_seen": 518914048
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006810030090270813,
+ "loss": 1.4471,
+ "theoretical_loss": 3.9276923660990004,
+ "tokens_seen": 518979584
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680986960882648,
+ "loss": 1.5669,
+ "theoretical_loss": 3.9276396752474176,
+ "tokens_seen": 519045120
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006809709127382147,
+ "loss": 1.5003,
+ "theoretical_loss": 3.9275869929108405,
+ "tokens_seen": 519110656
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006809548645937814,
+ "loss": 1.4697,
+ "theoretical_loss": 3.927534319086817,
+ "tokens_seen": 519176192
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006809388164493481,
+ "loss": 1.438,
+ "theoretical_loss": 3.9274816537728983,
+ "tokens_seen": 519241728
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006809227683049148,
+ "loss": 1.428,
+ "theoretical_loss": 3.9274289969666354,
+ "tokens_seen": 519307264
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006809067201604815,
+ "loss": 1.5598,
+ "theoretical_loss": 3.9273763486655806,
+ "tokens_seen": 519372800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006808906720160482,
+ "loss": 1.4526,
+ "theoretical_loss": 3.9273237088672874,
+ "tokens_seen": 519438336
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006808746238716149,
+ "loss": 1.4842,
+ "theoretical_loss": 3.92727107756931,
+ "tokens_seen": 519503872
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006808585757271816,
+ "loss": 1.4134,
+ "theoretical_loss": 3.927218454769203,
+ "tokens_seen": 519569408
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006808425275827483,
+ "loss": 1.4679,
+ "theoretical_loss": 3.9271658404645233,
+ "tokens_seen": 519634944
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680826479438315,
+ "loss": 1.5984,
+ "theoretical_loss": 3.9271132346528286,
+ "tokens_seen": 519700480
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006808104312938817,
+ "loss": 1.5113,
+ "theoretical_loss": 3.9270606373316763,
+ "tokens_seen": 519766016
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807943831494484,
+ "loss": 1.5097,
+ "theoretical_loss": 3.9270080484986267,
+ "tokens_seen": 519831552
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807783350050151,
+ "loss": 1.5268,
+ "theoretical_loss": 3.9269554681512395,
+ "tokens_seen": 519897088
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807622868605819,
+ "loss": 1.4653,
+ "theoretical_loss": 3.926902896287076,
+ "tokens_seen": 519962624
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807462387161485,
+ "loss": 1.4649,
+ "theoretical_loss": 3.9268503329036992,
+ "tokens_seen": 520028160
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807301905717153,
+ "loss": 1.5718,
+ "theoretical_loss": 3.926797777998672,
+ "tokens_seen": 520093696
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006807141424272819,
+ "loss": 1.357,
+ "theoretical_loss": 3.9267452315695586,
+ "tokens_seen": 520159232
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806980942828487,
+ "loss": 1.468,
+ "theoretical_loss": 3.9266926936139246,
+ "tokens_seen": 520224768
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806820461384153,
+ "loss": 1.4507,
+ "theoretical_loss": 3.9266401641293367,
+ "tokens_seen": 520290304
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680665997993982,
+ "loss": 1.3782,
+ "theoretical_loss": 3.926587643113362,
+ "tokens_seen": 520355840
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806499498495487,
+ "loss": 1.4731,
+ "theoretical_loss": 3.9265351305635687,
+ "tokens_seen": 520421376
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806339017051155,
+ "loss": 1.4242,
+ "theoretical_loss": 3.9264826264775268,
+ "tokens_seen": 520486912
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806178535606821,
+ "loss": 1.5955,
+ "theoretical_loss": 3.9264301308528067,
+ "tokens_seen": 520552448
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006806018054162489,
+ "loss": 1.4677,
+ "theoretical_loss": 3.926377643686979,
+ "tokens_seen": 520617984
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006805857572718155,
+ "loss": 1.539,
+ "theoretical_loss": 3.926325164977616,
+ "tokens_seen": 520683520
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006805697091273823,
+ "loss": 1.4778,
+ "theoretical_loss": 3.9262726947222926,
+ "tokens_seen": 520749056
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006805536609829489,
+ "loss": 1.4355,
+ "theoretical_loss": 3.926220232918582,
+ "tokens_seen": 520814592
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006805376128385156,
+ "loss": 1.5164,
+ "theoretical_loss": 3.92616777956406,
+ "tokens_seen": 520880128
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006805215646940823,
+ "loss": 1.3718,
+ "theoretical_loss": 3.926115334656303,
+ "tokens_seen": 520945664
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 179706,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5349757671356201,
+ "objective/train/theoretical_loss": 3.926062898192888,
+ "objective/train/tokens_used": 541471200,
+ "theoretical_loss": 3.926062898192888,
+ "tokens_seen": 521011200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680505516549649,
+ "loss": 1.5216,
+ "theoretical_loss": 3.926062898192888,
+ "tokens_seen": 521011200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804894684052157,
+ "loss": 1.4301,
+ "theoretical_loss": 3.926010470171394,
+ "tokens_seen": 521076736
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804734202607824,
+ "loss": 1.5091,
+ "theoretical_loss": 3.9259580505894007,
+ "tokens_seen": 521142272
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804573721163491,
+ "loss": 1.4445,
+ "theoretical_loss": 3.925905639444488,
+ "tokens_seen": 521207808
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804413239719158,
+ "loss": 1.5386,
+ "theoretical_loss": 3.9258532367342367,
+ "tokens_seen": 521273344
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804252758274825,
+ "loss": 1.4605,
+ "theoretical_loss": 3.9258008424562307,
+ "tokens_seen": 521338880
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006804092276830492,
+ "loss": 1.5114,
+ "theoretical_loss": 3.9257484566080523,
+ "tokens_seen": 521404416
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006803931795386158,
+ "loss": 1.5215,
+ "theoretical_loss": 3.9256960791872855,
+ "tokens_seen": 521469952
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006803771313941826,
+ "loss": 1.4821,
+ "theoretical_loss": 3.925643710191517,
+ "tokens_seen": 521535488
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006803610832497492,
+ "loss": 1.4289,
+ "theoretical_loss": 3.9255913496183332,
+ "tokens_seen": 521601024
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680345035105316,
+ "loss": 1.4774,
+ "theoretical_loss": 3.9255389974653196,
+ "tokens_seen": 521666560
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006803289869608826,
+ "loss": 1.496,
+ "theoretical_loss": 3.9254866537300668,
+ "tokens_seen": 521732096
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006803129388164494,
+ "loss": 1.5232,
+ "theoretical_loss": 3.925434318410163,
+ "tokens_seen": 521797632
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680296890672016,
+ "loss": 1.5735,
+ "theoretical_loss": 3.9253819915031993,
+ "tokens_seen": 521863168
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802808425275828,
+ "loss": 1.4249,
+ "theoretical_loss": 3.9253296730067664,
+ "tokens_seen": 521928704
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802647943831494,
+ "loss": 1.5218,
+ "theoretical_loss": 3.9252773629184565,
+ "tokens_seen": 521994240
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802487462387162,
+ "loss": 1.394,
+ "theoretical_loss": 3.925225061235863,
+ "tokens_seen": 522059776
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802326980942828,
+ "loss": 1.3936,
+ "theoretical_loss": 3.925172767956581,
+ "tokens_seen": 522125312
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802166499498496,
+ "loss": 1.6192,
+ "theoretical_loss": 3.9251204830782056,
+ "tokens_seen": 522190848
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006802006018054162,
+ "loss": 1.3342,
+ "theoretical_loss": 3.925068206598332,
+ "tokens_seen": 522256384
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680184553660983,
+ "loss": 1.5168,
+ "theoretical_loss": 3.9250159385145595,
+ "tokens_seen": 522321920
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006801685055165496,
+ "loss": 1.4312,
+ "theoretical_loss": 3.9249636788244846,
+ "tokens_seen": 522387456
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006801524573721164,
+ "loss": 1.4457,
+ "theoretical_loss": 3.924911427525708,
+ "tokens_seen": 522452992
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000680136409227683,
+ "loss": 1.4418,
+ "theoretical_loss": 3.9248591846158285,
+ "tokens_seen": 522518528
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006801203610832498,
+ "loss": 1.4766,
+ "theoretical_loss": 3.9248069500924485,
+ "tokens_seen": 522584064
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006801043129388164,
+ "loss": 1.4706,
+ "theoretical_loss": 3.9247547239531695,
+ "tokens_seen": 522649600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006800882647943832,
+ "loss": 1.3625,
+ "theoretical_loss": 3.924702506195595,
+ "tokens_seen": 522715136
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006800722166499498,
+ "loss": 1.6163,
+ "theoretical_loss": 3.9246502968173296,
+ "tokens_seen": 522780672
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006800561685055166,
+ "loss": 1.4221,
+ "theoretical_loss": 3.9245980958159787,
+ "tokens_seen": 522846208
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006800401203610832,
+ "loss": 1.5205,
+ "theoretical_loss": 3.924545903189147,
+ "tokens_seen": 522911744
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.00068002407221665,
+ "loss": 1.5592,
+ "theoretical_loss": 3.9244937189344435,
+ "tokens_seen": 522977280
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006800080240722166,
+ "loss": 1.4894,
+ "theoretical_loss": 3.9244415430494755,
+ "tokens_seen": 523042816
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006799919759277834,
+ "loss": 1.4424,
+ "theoretical_loss": 3.9243893755318515,
+ "tokens_seen": 523108352
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.00067997592778335,
+ "loss": 1.3891,
+ "theoretical_loss": 3.924337216379183,
+ "tokens_seen": 523173888
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006799598796389168,
+ "loss": 1.4588,
+ "theoretical_loss": 3.92428506558908,
+ "tokens_seen": 523239424
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006799438314944834,
+ "loss": 1.5871,
+ "theoretical_loss": 3.924232923159155,
+ "tokens_seen": 523304960
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006799277833500502,
+ "loss": 1.4269,
+ "theoretical_loss": 3.9241807890870213,
+ "tokens_seen": 523370496
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006799117352056168,
+ "loss": 1.5134,
+ "theoretical_loss": 3.9241286633702925,
+ "tokens_seen": 523436032
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006798956870611836,
+ "loss": 1.5179,
+ "theoretical_loss": 3.9240765460065834,
+ "tokens_seen": 523501568
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006798796389167503,
+ "loss": 1.515,
+ "theoretical_loss": 3.924024436993511,
+ "tokens_seen": 523567104
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679863590772317,
+ "loss": 1.481,
+ "theoretical_loss": 3.9239723363286916,
+ "tokens_seen": 523632640
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006798475426278837,
+ "loss": 1.4266,
+ "theoretical_loss": 3.923920244009743,
+ "tokens_seen": 523698176
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006798314944834504,
+ "loss": 1.4077,
+ "theoretical_loss": 3.9238681600342846,
+ "tokens_seen": 523763712
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006798154463390171,
+ "loss": 1.4432,
+ "theoretical_loss": 3.9238160843999363,
+ "tokens_seen": 523829248
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797993981945838,
+ "loss": 1.4181,
+ "theoretical_loss": 3.923764017104318,
+ "tokens_seen": 523894784
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797833500501505,
+ "loss": 1.4425,
+ "theoretical_loss": 3.9237119581450535,
+ "tokens_seen": 523960320
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797673019057172,
+ "loss": 1.5611,
+ "theoretical_loss": 3.923659907519764,
+ "tokens_seen": 524025856
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797512537612839,
+ "loss": 1.3971,
+ "theoretical_loss": 3.9236078652260735,
+ "tokens_seen": 524091392
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797352056168506,
+ "loss": 1.5388,
+ "theoretical_loss": 3.9235558312616075,
+ "tokens_seen": 524156928
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006797191574724173,
+ "loss": 1.4835,
+ "theoretical_loss": 3.9235038056239917,
+ "tokens_seen": 524222464
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 180943,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5657446384429932,
+ "objective/train/theoretical_loss": 3.9234517883108517,
+ "objective/train/tokens_used": 544748000,
+ "theoretical_loss": 3.9234517883108517,
+ "tokens_seen": 524288000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679703109327984,
+ "loss": 1.5641,
+ "theoretical_loss": 3.9234517883108517,
+ "tokens_seen": 524288000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796870611835507,
+ "loss": 1.4348,
+ "theoretical_loss": 3.9233997793198165,
+ "tokens_seen": 524353536
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796710130391174,
+ "loss": 1.5001,
+ "theoretical_loss": 3.9233477786485143,
+ "tokens_seen": 524419072
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796549648946841,
+ "loss": 1.5435,
+ "theoretical_loss": 3.923295786294575,
+ "tokens_seen": 524484608
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796389167502508,
+ "loss": 1.4204,
+ "theoretical_loss": 3.9232438022556293,
+ "tokens_seen": 524550144
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796228686058175,
+ "loss": 1.4743,
+ "theoretical_loss": 3.923191826529308,
+ "tokens_seen": 524615680
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006796068204613842,
+ "loss": 1.471,
+ "theoretical_loss": 3.9231398591132445,
+ "tokens_seen": 524681216
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006795907723169509,
+ "loss": 1.4699,
+ "theoretical_loss": 3.9230879000050716,
+ "tokens_seen": 524746752
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006795747241725176,
+ "loss": 1.526,
+ "theoretical_loss": 3.9230359492024247,
+ "tokens_seen": 524812288
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006795586760280843,
+ "loss": 1.5139,
+ "theoretical_loss": 3.9229840067029382,
+ "tokens_seen": 524877824
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679542627883651,
+ "loss": 1.4791,
+ "theoretical_loss": 3.92293207250425,
+ "tokens_seen": 524943360
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006795265797392177,
+ "loss": 1.4533,
+ "theoretical_loss": 3.9228801466039958,
+ "tokens_seen": 525008896
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006795105315947844,
+ "loss": 1.4722,
+ "theoretical_loss": 3.9228282289998155,
+ "tokens_seen": 525074432
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006794944834503511,
+ "loss": 1.358,
+ "theoretical_loss": 3.9227763196893477,
+ "tokens_seen": 525139968
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006794784353059178,
+ "loss": 1.5151,
+ "theoretical_loss": 3.9227244186702332,
+ "tokens_seen": 525205504
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006794623871614845,
+ "loss": 1.4097,
+ "theoretical_loss": 3.9226725259401123,
+ "tokens_seen": 525271040
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006794463390170512,
+ "loss": 1.5695,
+ "theoretical_loss": 3.9226206414966285,
+ "tokens_seen": 525336576
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679430290872618,
+ "loss": 1.3796,
+ "theoretical_loss": 3.9225687653374237,
+ "tokens_seen": 525402112
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006794142427281846,
+ "loss": 1.4475,
+ "theoretical_loss": 3.922516897460143,
+ "tokens_seen": 525467648
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793981945837513,
+ "loss": 1.4249,
+ "theoretical_loss": 3.9224650378624313,
+ "tokens_seen": 525533184
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679382146439318,
+ "loss": 1.5164,
+ "theoretical_loss": 3.9224131865419345,
+ "tokens_seen": 525598720
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793660982948848,
+ "loss": 1.4282,
+ "theoretical_loss": 3.9223613434963003,
+ "tokens_seen": 525664256
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793500501504514,
+ "loss": 1.4594,
+ "theoretical_loss": 3.922309508723176,
+ "tokens_seen": 525729792
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793340020060182,
+ "loss": 1.4641,
+ "theoretical_loss": 3.9222576822202107,
+ "tokens_seen": 525795328
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793179538615848,
+ "loss": 1.5236,
+ "theoretical_loss": 3.922205863985055,
+ "tokens_seen": 525860864
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006793019057171516,
+ "loss": 1.4442,
+ "theoretical_loss": 3.9221540540153588,
+ "tokens_seen": 525926400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006792858575727182,
+ "loss": 1.3884,
+ "theoretical_loss": 3.922102252308775,
+ "tokens_seen": 525991936
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679269809428285,
+ "loss": 1.5972,
+ "theoretical_loss": 3.9220504588629552,
+ "tokens_seen": 526057472
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006792537612838516,
+ "loss": 1.4631,
+ "theoretical_loss": 3.9219986736755548,
+ "tokens_seen": 526123008
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006792377131394184,
+ "loss": 1.5174,
+ "theoretical_loss": 3.921946896744227,
+ "tokens_seen": 526188544
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679221664994985,
+ "loss": 1.3494,
+ "theoretical_loss": 3.9218951280666285,
+ "tokens_seen": 526254080
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006792056168505518,
+ "loss": 1.3861,
+ "theoretical_loss": 3.9218433676404154,
+ "tokens_seen": 526319616
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791895687061184,
+ "loss": 1.3822,
+ "theoretical_loss": 3.9217916154632455,
+ "tokens_seen": 526385152
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791735205616852,
+ "loss": 1.5359,
+ "theoretical_loss": 3.9217398715327776,
+ "tokens_seen": 526450688
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791574724172518,
+ "loss": 1.5608,
+ "theoretical_loss": 3.9216881358466713,
+ "tokens_seen": 526516224
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791414242728185,
+ "loss": 1.3272,
+ "theoretical_loss": 3.9216364084025868,
+ "tokens_seen": 526581760
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791253761283852,
+ "loss": 1.5005,
+ "theoretical_loss": 3.921584689198185,
+ "tokens_seen": 526647296
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006791093279839519,
+ "loss": 1.3935,
+ "theoretical_loss": 3.921532978231129,
+ "tokens_seen": 526712832
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006790932798395186,
+ "loss": 1.4883,
+ "theoretical_loss": 3.921481275499082,
+ "tokens_seen": 526778368
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006790772316950853,
+ "loss": 1.5002,
+ "theoretical_loss": 3.921429580999709,
+ "tokens_seen": 526843904
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000679061183550652,
+ "loss": 1.538,
+ "theoretical_loss": 3.921377894730674,
+ "tokens_seen": 526909440
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006790451354062187,
+ "loss": 1.5555,
+ "theoretical_loss": 3.9213262166896445,
+ "tokens_seen": 526974976
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006790290872617854,
+ "loss": 1.4427,
+ "theoretical_loss": 3.9212745468742867,
+ "tokens_seen": 527040512
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006790130391173521,
+ "loss": 1.3753,
+ "theoretical_loss": 3.921222885282268,
+ "tokens_seen": 527106048
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789969909729187,
+ "loss": 1.3694,
+ "theoretical_loss": 3.9211712319112597,
+ "tokens_seen": 527171584
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789809428284855,
+ "loss": 1.4033,
+ "theoretical_loss": 3.92111958675893,
+ "tokens_seen": 527237120
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789648946840521,
+ "loss": 1.5529,
+ "theoretical_loss": 3.9210679498229504,
+ "tokens_seen": 527302656
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789488465396189,
+ "loss": 1.5309,
+ "theoretical_loss": 3.9210163211009927,
+ "tokens_seen": 527368192
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789327983951855,
+ "loss": 1.4576,
+ "theoretical_loss": 3.92096470059073,
+ "tokens_seen": 527433728
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789167502507523,
+ "loss": 1.4784,
+ "theoretical_loss": 3.9209130882898364,
+ "tokens_seen": 527499264
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 181844,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4468753337860107,
+ "objective/train/theoretical_loss": 3.920861484195986,
+ "objective/train/tokens_used": 548024800,
+ "theoretical_loss": 3.920861484195986,
+ "tokens_seen": 527564800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006789007021063189,
+ "loss": 1.4834,
+ "theoretical_loss": 3.920861484195986,
+ "tokens_seen": 527564800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788846539618857,
+ "loss": 1.3464,
+ "theoretical_loss": 3.920809888306854,
+ "tokens_seen": 527630336
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788686058174523,
+ "loss": 1.4429,
+ "theoretical_loss": 3.9207583006201183,
+ "tokens_seen": 527695872
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788525576730191,
+ "loss": 1.5239,
+ "theoretical_loss": 3.920706721133456,
+ "tokens_seen": 527761408
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788365095285857,
+ "loss": 1.4154,
+ "theoretical_loss": 3.9206551498445465,
+ "tokens_seen": 527826944
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788204613841525,
+ "loss": 1.3948,
+ "theoretical_loss": 3.920603586751068,
+ "tokens_seen": 527892480
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006788044132397191,
+ "loss": 1.5827,
+ "theoretical_loss": 3.9205520318507006,
+ "tokens_seen": 527958016
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787883650952859,
+ "loss": 1.4211,
+ "theoretical_loss": 3.9205004851411274,
+ "tokens_seen": 528023552
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787723169508525,
+ "loss": 1.378,
+ "theoretical_loss": 3.920448946620029,
+ "tokens_seen": 528089088
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787562688064193,
+ "loss": 1.4033,
+ "theoretical_loss": 3.9203974162850903,
+ "tokens_seen": 528154624
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787402206619859,
+ "loss": 1.5083,
+ "theoretical_loss": 3.920345894133994,
+ "tokens_seen": 528220160
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787241725175527,
+ "loss": 1.4078,
+ "theoretical_loss": 3.9202943801644263,
+ "tokens_seen": 528285696
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006787081243731193,
+ "loss": 1.5383,
+ "theoretical_loss": 3.9202428743740727,
+ "tokens_seen": 528351232
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786920762286861,
+ "loss": 1.3915,
+ "theoretical_loss": 3.92019137676062,
+ "tokens_seen": 528416768
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786760280842528,
+ "loss": 1.4579,
+ "theoretical_loss": 3.920139887321757,
+ "tokens_seen": 528482304
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786599799398195,
+ "loss": 1.4671,
+ "theoretical_loss": 3.9200884060551724,
+ "tokens_seen": 528547840
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786439317953862,
+ "loss": 1.3536,
+ "theoretical_loss": 3.9200369329585563,
+ "tokens_seen": 528613376
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786278836509529,
+ "loss": 1.4441,
+ "theoretical_loss": 3.9199854680295982,
+ "tokens_seen": 528678912
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006786118355065196,
+ "loss": 1.3964,
+ "theoretical_loss": 3.9199340112659913,
+ "tokens_seen": 528744448
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006785957873620863,
+ "loss": 1.4516,
+ "theoretical_loss": 3.9198825626654275,
+ "tokens_seen": 528809984
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000678579739217653,
+ "loss": 1.4887,
+ "theoretical_loss": 3.9198311222256,
+ "tokens_seen": 528875520
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006785636910732197,
+ "loss": 1.5368,
+ "theoretical_loss": 3.919779689944205,
+ "tokens_seen": 528941056
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006785476429287864,
+ "loss": 1.4402,
+ "theoretical_loss": 3.9197282658189363,
+ "tokens_seen": 529006592
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006785315947843531,
+ "loss": 1.5339,
+ "theoretical_loss": 3.919676849847491,
+ "tokens_seen": 529072128
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006785155466399198,
+ "loss": 1.4247,
+ "theoretical_loss": 3.9196254420275665,
+ "tokens_seen": 529137664
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784994984954865,
+ "loss": 1.4668,
+ "theoretical_loss": 3.919574042356861,
+ "tokens_seen": 529203200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784834503510532,
+ "loss": 1.4803,
+ "theoretical_loss": 3.9195226508330734,
+ "tokens_seen": 529268736
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784674022066199,
+ "loss": 1.3684,
+ "theoretical_loss": 3.919471267453905,
+ "tokens_seen": 529334272
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784513540621866,
+ "loss": 1.524,
+ "theoretical_loss": 3.919419892217055,
+ "tokens_seen": 529399808
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784353059177533,
+ "loss": 1.4664,
+ "theoretical_loss": 3.919368525120227,
+ "tokens_seen": 529465344
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.00067841925777332,
+ "loss": 1.4618,
+ "theoretical_loss": 3.9193171661611235,
+ "tokens_seen": 529530880
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006784032096288867,
+ "loss": 1.4584,
+ "theoretical_loss": 3.9192658153374484,
+ "tokens_seen": 529596416
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783871614844534,
+ "loss": 1.4449,
+ "theoretical_loss": 3.9192144726469063,
+ "tokens_seen": 529661952
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783711133400201,
+ "loss": 1.5014,
+ "theoretical_loss": 3.919163138087203,
+ "tokens_seen": 529727488
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783550651955868,
+ "loss": 1.5635,
+ "theoretical_loss": 3.919111811656046,
+ "tokens_seen": 529793024
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783390170511535,
+ "loss": 1.4419,
+ "theoretical_loss": 3.9190604933511417,
+ "tokens_seen": 529858560
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783229689067202,
+ "loss": 1.448,
+ "theoretical_loss": 3.9190091831701994,
+ "tokens_seen": 529924096
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006783069207622869,
+ "loss": 1.3953,
+ "theoretical_loss": 3.918957881110928,
+ "tokens_seen": 529989632
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006782908726178536,
+ "loss": 1.5078,
+ "theoretical_loss": 3.9189065871710387,
+ "tokens_seen": 530055168
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006782748244734203,
+ "loss": 1.3386,
+ "theoretical_loss": 3.918855301348242,
+ "tokens_seen": 530120704
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000678258776328987,
+ "loss": 1.5389,
+ "theoretical_loss": 3.9188040236402513,
+ "tokens_seen": 530186240
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006782427281845537,
+ "loss": 1.4875,
+ "theoretical_loss": 3.918752754044778,
+ "tokens_seen": 530251776
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006782266800401204,
+ "loss": 1.3773,
+ "theoretical_loss": 3.9187014925595385,
+ "tokens_seen": 530317312
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006782106318956871,
+ "loss": 1.564,
+ "theoretical_loss": 3.918650239182246,
+ "tokens_seen": 530382848
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781945837512538,
+ "loss": 1.5289,
+ "theoretical_loss": 3.918598993910617,
+ "tokens_seen": 530448384
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781785356068205,
+ "loss": 1.4389,
+ "theoretical_loss": 3.9185477567423685,
+ "tokens_seen": 530513920
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781624874623872,
+ "loss": 1.5537,
+ "theoretical_loss": 3.9184965276752184,
+ "tokens_seen": 530579456
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781464393179539,
+ "loss": 1.385,
+ "theoretical_loss": 3.9184453067068854,
+ "tokens_seen": 530644992
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781303911735206,
+ "loss": 1.3913,
+ "theoretical_loss": 3.918394093835089,
+ "tokens_seen": 530710528
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006781143430290873,
+ "loss": 1.4841,
+ "theoretical_loss": 3.91834288905755,
+ "tokens_seen": 530776064
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 183006,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5242764949798584,
+ "objective/train/theoretical_loss": 3.91829169237199,
+ "objective/train/tokens_used": 551301600,
+ "theoretical_loss": 3.91829169237199,
+ "tokens_seen": 530841600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000678098294884654,
+ "loss": 1.5453,
+ "theoretical_loss": 3.91829169237199,
+ "tokens_seen": 530841600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780822467402207,
+ "loss": 1.5267,
+ "theoretical_loss": 3.918240503776131,
+ "tokens_seen": 530907136
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780661985957874,
+ "loss": 1.5107,
+ "theoretical_loss": 3.9181893232676965,
+ "tokens_seen": 530972672
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780501504513541,
+ "loss": 1.4461,
+ "theoretical_loss": 3.9181381508444106,
+ "tokens_seen": 531038208
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780341023069208,
+ "loss": 1.4501,
+ "theoretical_loss": 3.918086986504,
+ "tokens_seen": 531103744
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780180541624875,
+ "loss": 1.46,
+ "theoretical_loss": 3.9180358302441887,
+ "tokens_seen": 531169280
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006780020060180543,
+ "loss": 1.388,
+ "theoretical_loss": 3.9179846820627047,
+ "tokens_seen": 531234816
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006779859578736209,
+ "loss": 1.3978,
+ "theoretical_loss": 3.917933541957276,
+ "tokens_seen": 531300352
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006779699097291877,
+ "loss": 1.5003,
+ "theoretical_loss": 3.9178824099256317,
+ "tokens_seen": 531365888
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006779538615847543,
+ "loss": 1.5086,
+ "theoretical_loss": 3.9178312859655016,
+ "tokens_seen": 531431424
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677937813440321,
+ "loss": 1.5874,
+ "theoretical_loss": 3.9177801700746153,
+ "tokens_seen": 531496960
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006779217652958877,
+ "loss": 1.4059,
+ "theoretical_loss": 3.9177290622507055,
+ "tokens_seen": 531562496
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006779057171514545,
+ "loss": 1.5381,
+ "theoretical_loss": 3.9176779624915055,
+ "tokens_seen": 531628032
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778896690070211,
+ "loss": 1.4416,
+ "theoretical_loss": 3.917626870794747,
+ "tokens_seen": 531693568
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778736208625879,
+ "loss": 1.4973,
+ "theoretical_loss": 3.9175757871581647,
+ "tokens_seen": 531759104
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778575727181545,
+ "loss": 1.4864,
+ "theoretical_loss": 3.917524711579495,
+ "tokens_seen": 531824640
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778415245737213,
+ "loss": 1.5042,
+ "theoretical_loss": 3.9174736440564732,
+ "tokens_seen": 531890176
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778254764292879,
+ "loss": 1.6653,
+ "theoretical_loss": 3.917422584586837,
+ "tokens_seen": 531955712
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006778094282848547,
+ "loss": 1.408,
+ "theoretical_loss": 3.9173715331683243,
+ "tokens_seen": 532021248
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777933801404213,
+ "loss": 1.3939,
+ "theoretical_loss": 3.9173204897986738,
+ "tokens_seen": 532086784
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777773319959881,
+ "loss": 1.4191,
+ "theoretical_loss": 3.9172694544756252,
+ "tokens_seen": 532152320
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777612838515547,
+ "loss": 1.4525,
+ "theoretical_loss": 3.91721842719692,
+ "tokens_seen": 532217856
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777452357071214,
+ "loss": 1.4859,
+ "theoretical_loss": 3.9171674079603,
+ "tokens_seen": 532283392
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777291875626881,
+ "loss": 1.4312,
+ "theoretical_loss": 3.9171163967635065,
+ "tokens_seen": 532348928
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006777131394182548,
+ "loss": 1.5149,
+ "theoretical_loss": 3.9170653936042843,
+ "tokens_seen": 532414464
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776970912738215,
+ "loss": 1.4395,
+ "theoretical_loss": 3.917014398480377,
+ "tokens_seen": 532480000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776810431293882,
+ "loss": 1.5004,
+ "theoretical_loss": 3.9169634113895304,
+ "tokens_seen": 532545536
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776649949849549,
+ "loss": 1.4336,
+ "theoretical_loss": 3.9169124323294913,
+ "tokens_seen": 532611072
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776489468405216,
+ "loss": 1.5267,
+ "theoretical_loss": 3.916861461298005,
+ "tokens_seen": 532676608
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776328986960883,
+ "loss": 1.5136,
+ "theoretical_loss": 3.916810498292822,
+ "tokens_seen": 532742144
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677616850551655,
+ "loss": 1.4758,
+ "theoretical_loss": 3.9167595433116897,
+ "tokens_seen": 532807680
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006776008024072216,
+ "loss": 1.4975,
+ "theoretical_loss": 3.9167085963523585,
+ "tokens_seen": 532873216
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006775847542627884,
+ "loss": 1.6303,
+ "theoretical_loss": 3.916657657412579,
+ "tokens_seen": 532938752
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677568706118355,
+ "loss": 1.5686,
+ "theoretical_loss": 3.9166067264901026,
+ "tokens_seen": 533004288
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006775526579739218,
+ "loss": 1.5364,
+ "theoretical_loss": 3.916555803582683,
+ "tokens_seen": 533069824
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006775366098294884,
+ "loss": 1.4351,
+ "theoretical_loss": 3.9165048886880722,
+ "tokens_seen": 533135360
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006775205616850552,
+ "loss": 1.5138,
+ "theoretical_loss": 3.916453981804026,
+ "tokens_seen": 533200896
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006775045135406218,
+ "loss": 1.3943,
+ "theoretical_loss": 3.916403082928299,
+ "tokens_seen": 533266432
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006774884653961886,
+ "loss": 1.4264,
+ "theoretical_loss": 3.916352192058648,
+ "tokens_seen": 533331968
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006774724172517553,
+ "loss": 1.4411,
+ "theoretical_loss": 3.9163013091928294,
+ "tokens_seen": 533397504
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677456369107322,
+ "loss": 1.4073,
+ "theoretical_loss": 3.9162504343286013,
+ "tokens_seen": 533463040
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006774403209628887,
+ "loss": 1.4156,
+ "theoretical_loss": 3.9161995674637233,
+ "tokens_seen": 533528576
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006774242728184554,
+ "loss": 1.4649,
+ "theoretical_loss": 3.916148708595955,
+ "tokens_seen": 533594112
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006774082246740221,
+ "loss": 1.4853,
+ "theoretical_loss": 3.9160978577230567,
+ "tokens_seen": 533659648
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773921765295888,
+ "loss": 1.5875,
+ "theoretical_loss": 3.91604701484279,
+ "tokens_seen": 533725184
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773761283851555,
+ "loss": 1.4837,
+ "theoretical_loss": 3.915996179952918,
+ "tokens_seen": 533790720
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773600802407222,
+ "loss": 1.44,
+ "theoretical_loss": 3.915945353051205,
+ "tokens_seen": 533856256
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773440320962889,
+ "loss": 1.5419,
+ "theoretical_loss": 3.9158945341354134,
+ "tokens_seen": 533921792
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773279839518556,
+ "loss": 1.5078,
+ "theoretical_loss": 3.9158437232033094,
+ "tokens_seen": 533987328
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006773119358074223,
+ "loss": 1.4417,
+ "theoretical_loss": 3.9157929202526596,
+ "tokens_seen": 534052864
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 184018,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4757671356201172,
+ "objective/train/theoretical_loss": 3.9157421252812306,
+ "objective/train/tokens_used": 554578400,
+ "theoretical_loss": 3.9157421252812306,
+ "tokens_seen": 534118400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677295887662989,
+ "loss": 1.4674,
+ "theoretical_loss": 3.9157421252812306,
+ "tokens_seen": 534118400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006772798395185557,
+ "loss": 1.5966,
+ "theoretical_loss": 3.9156913382867895,
+ "tokens_seen": 534183936
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006772637913741224,
+ "loss": 1.4379,
+ "theoretical_loss": 3.915640559267107,
+ "tokens_seen": 534249472
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006772477432296891,
+ "loss": 1.5897,
+ "theoretical_loss": 3.9155897882199513,
+ "tokens_seen": 534315008
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006772316950852558,
+ "loss": 1.393,
+ "theoretical_loss": 3.9155390251430937,
+ "tokens_seen": 534380544
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006772156469408225,
+ "loss": 1.5403,
+ "theoretical_loss": 3.915488270034306,
+ "tokens_seen": 534446080
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771995987963892,
+ "loss": 1.4583,
+ "theoretical_loss": 3.9154375228913594,
+ "tokens_seen": 534511616
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771835506519559,
+ "loss": 1.3791,
+ "theoretical_loss": 3.9153867837120284,
+ "tokens_seen": 534577152
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771675025075226,
+ "loss": 1.4518,
+ "theoretical_loss": 3.9153360524940872,
+ "tokens_seen": 534642688
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771514543630893,
+ "loss": 1.4759,
+ "theoretical_loss": 3.915285329235311,
+ "tokens_seen": 534708224
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000677135406218656,
+ "loss": 1.5145,
+ "theoretical_loss": 3.915234613933475,
+ "tokens_seen": 534773760
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771193580742227,
+ "loss": 1.5178,
+ "theoretical_loss": 3.915183906586357,
+ "tokens_seen": 534839296
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006771033099297894,
+ "loss": 1.3855,
+ "theoretical_loss": 3.915133207191734,
+ "tokens_seen": 534904832
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770872617853561,
+ "loss": 1.4519,
+ "theoretical_loss": 3.915082515747385,
+ "tokens_seen": 534970368
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770712136409228,
+ "loss": 1.5169,
+ "theoretical_loss": 3.9150318322510893,
+ "tokens_seen": 535035904
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770551654964895,
+ "loss": 1.4437,
+ "theoretical_loss": 3.9149811567006285,
+ "tokens_seen": 535101440
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770391173520562,
+ "loss": 1.4334,
+ "theoretical_loss": 3.9149304890937833,
+ "tokens_seen": 535166976
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770230692076229,
+ "loss": 1.4117,
+ "theoretical_loss": 3.9148798294283353,
+ "tokens_seen": 535232512
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006770070210631896,
+ "loss": 1.5312,
+ "theoretical_loss": 3.9148291777020683,
+ "tokens_seen": 535298048
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006769909729187563,
+ "loss": 1.4469,
+ "theoretical_loss": 3.9147785339127665,
+ "tokens_seen": 535363584
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676974924774323,
+ "loss": 1.5636,
+ "theoretical_loss": 3.914727898058215,
+ "tokens_seen": 535429120
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006769588766298897,
+ "loss": 1.4036,
+ "theoretical_loss": 3.914677270136199,
+ "tokens_seen": 535494656
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006769428284854564,
+ "loss": 1.5086,
+ "theoretical_loss": 3.914626650144505,
+ "tokens_seen": 535560192
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006769267803410231,
+ "loss": 1.3943,
+ "theoretical_loss": 3.914576038080922,
+ "tokens_seen": 535625728
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006769107321965898,
+ "loss": 1.3243,
+ "theoretical_loss": 3.9145254339432367,
+ "tokens_seen": 535691264
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006768946840521565,
+ "loss": 1.4929,
+ "theoretical_loss": 3.9144748377292395,
+ "tokens_seen": 535756800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006768786359077232,
+ "loss": 1.4033,
+ "theoretical_loss": 3.9144242494367205,
+ "tokens_seen": 535822336
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006768625877632899,
+ "loss": 1.4823,
+ "theoretical_loss": 3.914373669063471,
+ "tokens_seen": 535887872
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006768465396188566,
+ "loss": 1.4225,
+ "theoretical_loss": 3.914323096607283,
+ "tokens_seen": 535953408
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006768304914744233,
+ "loss": 1.4116,
+ "theoretical_loss": 3.914272532065949,
+ "tokens_seen": 536018944
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.00067681444332999,
+ "loss": 1.4862,
+ "theoretical_loss": 3.914221975437264,
+ "tokens_seen": 536084480
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767983951855567,
+ "loss": 1.4409,
+ "theoretical_loss": 3.914171426719021,
+ "tokens_seen": 536150016
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767823470411234,
+ "loss": 1.5067,
+ "theoretical_loss": 3.9141208859090164,
+ "tokens_seen": 536215552
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767662988966901,
+ "loss": 1.4402,
+ "theoretical_loss": 3.9140703530050467,
+ "tokens_seen": 536281088
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767502507522568,
+ "loss": 1.6256,
+ "theoretical_loss": 3.914019828004909,
+ "tokens_seen": 536346624
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767342026078235,
+ "loss": 1.4947,
+ "theoretical_loss": 3.9139693109064018,
+ "tokens_seen": 536412160
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006767181544633902,
+ "loss": 1.4468,
+ "theoretical_loss": 3.9139188017073243,
+ "tokens_seen": 536477696
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676702106318957,
+ "loss": 1.4703,
+ "theoretical_loss": 3.9138683004054764,
+ "tokens_seen": 536543232
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006766860581745236,
+ "loss": 1.3957,
+ "theoretical_loss": 3.913817806998659,
+ "tokens_seen": 536608768
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006766700100300903,
+ "loss": 1.4385,
+ "theoretical_loss": 3.9137673214846735,
+ "tokens_seen": 536674304
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676653961885657,
+ "loss": 1.3962,
+ "theoretical_loss": 3.913716843861323,
+ "tokens_seen": 536739840
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006766379137412238,
+ "loss": 1.5503,
+ "theoretical_loss": 3.91366637412641,
+ "tokens_seen": 536805376
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006766218655967904,
+ "loss": 1.571,
+ "theoretical_loss": 3.9136159122777405,
+ "tokens_seen": 536870912
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006766058174523572,
+ "loss": 1.546,
+ "theoretical_loss": 3.9135654583131183,
+ "tokens_seen": 536936448
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006765897693079238,
+ "loss": 1.5006,
+ "theoretical_loss": 3.9135150122303504,
+ "tokens_seen": 537001984
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006765737211634906,
+ "loss": 1.5824,
+ "theoretical_loss": 3.913464574027244,
+ "tokens_seen": 537067520
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006765576730190572,
+ "loss": 1.5226,
+ "theoretical_loss": 3.913414143701606,
+ "tokens_seen": 537133056
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676541624874624,
+ "loss": 1.5175,
+ "theoretical_loss": 3.913363721251246,
+ "tokens_seen": 537198592
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006765255767301906,
+ "loss": 1.3869,
+ "theoretical_loss": 3.913313306673974,
+ "tokens_seen": 537264128
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006765095285857574,
+ "loss": 1.5287,
+ "theoretical_loss": 3.913262899967599,
+ "tokens_seen": 537329664
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 185299,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3653303384780884,
+ "objective/train/theoretical_loss": 3.9132125011299337,
+ "objective/train/tokens_used": 557855200,
+ "theoretical_loss": 3.9132125011299337,
+ "tokens_seen": 537395200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676493480441324,
+ "loss": 1.3709,
+ "theoretical_loss": 3.9132125011299337,
+ "tokens_seen": 537395200
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006764774322968908,
+ "loss": 1.4701,
+ "theoretical_loss": 3.9131621101587903,
+ "tokens_seen": 537460736
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006764613841524574,
+ "loss": 1.4026,
+ "theoretical_loss": 3.913111727051982,
+ "tokens_seen": 537526272
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006764453360080242,
+ "loss": 1.6027,
+ "theoretical_loss": 3.9130613518073227,
+ "tokens_seen": 537591808
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006764292878635908,
+ "loss": 1.4285,
+ "theoretical_loss": 3.913010984422627,
+ "tokens_seen": 537657344
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006764132397191576,
+ "loss": 1.4628,
+ "theoretical_loss": 3.9129606248957103,
+ "tokens_seen": 537722880
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006763971915747242,
+ "loss": 1.5315,
+ "theoretical_loss": 3.91291027322439,
+ "tokens_seen": 537788416
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676381143430291,
+ "loss": 1.3342,
+ "theoretical_loss": 3.9128599294064834,
+ "tokens_seen": 537853952
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006763650952858576,
+ "loss": 1.6502,
+ "theoretical_loss": 3.9128095934398095,
+ "tokens_seen": 537919488
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006763490471414244,
+ "loss": 1.4484,
+ "theoretical_loss": 3.9127592653221868,
+ "tokens_seen": 537985024
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676332998996991,
+ "loss": 1.4775,
+ "theoretical_loss": 3.912708945051435,
+ "tokens_seen": 538050560
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006763169508525577,
+ "loss": 1.6435,
+ "theoretical_loss": 3.9126586326253765,
+ "tokens_seen": 538116096
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006763009027081244,
+ "loss": 1.5078,
+ "theoretical_loss": 3.9126083280418325,
+ "tokens_seen": 538181632
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762848545636911,
+ "loss": 1.4297,
+ "theoretical_loss": 3.9125580312986257,
+ "tokens_seen": 538247168
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762688064192578,
+ "loss": 1.4096,
+ "theoretical_loss": 3.9125077423935792,
+ "tokens_seen": 538312704
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762527582748245,
+ "loss": 1.4451,
+ "theoretical_loss": 3.912457461324519,
+ "tokens_seen": 538378240
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762367101303912,
+ "loss": 1.5723,
+ "theoretical_loss": 3.9124071880892686,
+ "tokens_seen": 538443776
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762206619859579,
+ "loss": 1.4255,
+ "theoretical_loss": 3.912356922685655,
+ "tokens_seen": 538509312
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006762046138415246,
+ "loss": 1.6908,
+ "theoretical_loss": 3.912306665111506,
+ "tokens_seen": 538574848
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006761885656970913,
+ "loss": 1.4579,
+ "theoretical_loss": 3.9122564153646486,
+ "tokens_seen": 538640384
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676172517552658,
+ "loss": 1.4806,
+ "theoretical_loss": 3.9122061734429128,
+ "tokens_seen": 538705920
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006761564694082247,
+ "loss": 1.3943,
+ "theoretical_loss": 3.9121559393441268,
+ "tokens_seen": 538771456
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006761404212637914,
+ "loss": 1.4735,
+ "theoretical_loss": 3.912105713066122,
+ "tokens_seen": 538836992
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006761243731193581,
+ "loss": 1.5643,
+ "theoretical_loss": 3.91205549460673,
+ "tokens_seen": 538902528
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006761083249749248,
+ "loss": 1.4788,
+ "theoretical_loss": 3.9120052839637824,
+ "tokens_seen": 538968064
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006760922768304915,
+ "loss": 1.5398,
+ "theoretical_loss": 3.9119550811351127,
+ "tokens_seen": 539033600
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006760762286860582,
+ "loss": 1.5037,
+ "theoretical_loss": 3.9119048861185552,
+ "tokens_seen": 539099136
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006760601805416249,
+ "loss": 1.4558,
+ "theoretical_loss": 3.9118546989119447,
+ "tokens_seen": 539164672
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006760441323971916,
+ "loss": 1.5235,
+ "theoretical_loss": 3.9118045195131166,
+ "tokens_seen": 539230208
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006760280842527583,
+ "loss": 1.4225,
+ "theoretical_loss": 3.911754347919908,
+ "tokens_seen": 539295744
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000676012036108325,
+ "loss": 1.4986,
+ "theoretical_loss": 3.911704184130156,
+ "tokens_seen": 539361280
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759959879638917,
+ "loss": 1.4661,
+ "theoretical_loss": 3.911654028141699,
+ "tokens_seen": 539426816
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759799398194584,
+ "loss": 1.4318,
+ "theoretical_loss": 3.9116038799523762,
+ "tokens_seen": 539492352
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759638916750251,
+ "loss": 1.478,
+ "theoretical_loss": 3.911553739560028,
+ "tokens_seen": 539557888
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759478435305918,
+ "loss": 1.4314,
+ "theoretical_loss": 3.9115036069624947,
+ "tokens_seen": 539623424
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759317953861585,
+ "loss": 1.471,
+ "theoretical_loss": 3.9114534821576186,
+ "tokens_seen": 539688960
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006759157472417252,
+ "loss": 1.3858,
+ "theoretical_loss": 3.9114033651432414,
+ "tokens_seen": 539754496
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758996990972919,
+ "loss": 1.4939,
+ "theoretical_loss": 3.9113532559172084,
+ "tokens_seen": 539820032
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758836509528586,
+ "loss": 1.5082,
+ "theoretical_loss": 3.9113031544773618,
+ "tokens_seen": 539885568
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758676028084253,
+ "loss": 1.545,
+ "theoretical_loss": 3.9112530608215486,
+ "tokens_seen": 539951104
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000675851554663992,
+ "loss": 1.5214,
+ "theoretical_loss": 3.9112029749476145,
+ "tokens_seen": 540016640
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758355065195587,
+ "loss": 1.5001,
+ "theoretical_loss": 3.9111528968534053,
+ "tokens_seen": 540082176
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758194583751254,
+ "loss": 1.5725,
+ "theoretical_loss": 3.91110282653677,
+ "tokens_seen": 540147712
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006758034102306921,
+ "loss": 1.4774,
+ "theoretical_loss": 3.9110527639955563,
+ "tokens_seen": 540213248
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757873620862588,
+ "loss": 1.5428,
+ "theoretical_loss": 3.911002709227615,
+ "tokens_seen": 540278784
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757713139418255,
+ "loss": 1.5055,
+ "theoretical_loss": 3.9109526622307955,
+ "tokens_seen": 540344320
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757552657973922,
+ "loss": 1.409,
+ "theoretical_loss": 3.910902623002949,
+ "tokens_seen": 540409856
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757392176529589,
+ "loss": 1.4916,
+ "theoretical_loss": 3.9108525915419277,
+ "tokens_seen": 540475392
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757231695085256,
+ "loss": 1.4927,
+ "theoretical_loss": 3.9108025678455847,
+ "tokens_seen": 540540928
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006757071213640923,
+ "loss": 1.4452,
+ "theoretical_loss": 3.9107525519117736,
+ "tokens_seen": 540606464
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 186273,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.601438283920288,
+ "objective/train/theoretical_loss": 3.910702543738349,
+ "objective/train/tokens_used": 561132000,
+ "theoretical_loss": 3.910702543738349,
+ "tokens_seen": 540672000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000675691073219659,
+ "loss": 1.559,
+ "theoretical_loss": 3.910702543738349,
+ "tokens_seen": 540672000
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006756750250752257,
+ "loss": 1.4619,
+ "theoretical_loss": 3.9106525433231663,
+ "tokens_seen": 540737536
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006756589769307924,
+ "loss": 1.4786,
+ "theoretical_loss": 3.9106025506640822,
+ "tokens_seen": 540803072
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006756429287863591,
+ "loss": 1.4277,
+ "theoretical_loss": 3.9105525657589535,
+ "tokens_seen": 540868608
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006756268806419258,
+ "loss": 1.5143,
+ "theoretical_loss": 3.910502588605638,
+ "tokens_seen": 540934144
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006756108324974925,
+ "loss": 1.5267,
+ "theoretical_loss": 3.9104526192019957,
+ "tokens_seen": 540999680
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006755947843530592,
+ "loss": 1.4738,
+ "theoretical_loss": 3.9104026575458857,
+ "tokens_seen": 541065216
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006755787362086259,
+ "loss": 1.4764,
+ "theoretical_loss": 3.9103527036351684,
+ "tokens_seen": 541130752
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006755626880641926,
+ "loss": 1.4191,
+ "theoretical_loss": 3.910302757467705,
+ "tokens_seen": 541196288
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006755466399197593,
+ "loss": 1.3898,
+ "theoretical_loss": 3.910252819041358,
+ "tokens_seen": 541261824
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000675530591775326,
+ "loss": 1.4358,
+ "theoretical_loss": 3.910202888353991,
+ "tokens_seen": 541327360
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006755145436308927,
+ "loss": 1.5174,
+ "theoretical_loss": 3.9101529654034675,
+ "tokens_seen": 541392896
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754984954864594,
+ "loss": 1.5144,
+ "theoretical_loss": 3.9101030501876526,
+ "tokens_seen": 541458432
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754824473420261,
+ "loss": 1.4981,
+ "theoretical_loss": 3.9100531427044114,
+ "tokens_seen": 541523968
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754663991975928,
+ "loss": 1.4326,
+ "theoretical_loss": 3.9100032429516114,
+ "tokens_seen": 541589504
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754503510531595,
+ "loss": 1.4579,
+ "theoretical_loss": 3.909953350927119,
+ "tokens_seen": 541655040
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754343029087262,
+ "loss": 1.4463,
+ "theoretical_loss": 3.909903466628803,
+ "tokens_seen": 541720576
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754182547642929,
+ "loss": 1.5862,
+ "theoretical_loss": 3.9098535900545324,
+ "tokens_seen": 541786112
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006754022066198596,
+ "loss": 1.5375,
+ "theoretical_loss": 3.909803721202177,
+ "tokens_seen": 541851648
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006753861584754263,
+ "loss": 1.5433,
+ "theoretical_loss": 3.909753860069608,
+ "tokens_seen": 541917184
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000675370110330993,
+ "loss": 1.5082,
+ "theoretical_loss": 3.909704006654695,
+ "tokens_seen": 541982720
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006753540621865597,
+ "loss": 1.4676,
+ "theoretical_loss": 3.9096541609553137,
+ "tokens_seen": 542048256
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006753380140421264,
+ "loss": 1.4795,
+ "theoretical_loss": 3.909604322969335,
+ "tokens_seen": 542113792
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006753219658976931,
+ "loss": 1.5508,
+ "theoretical_loss": 3.9095544926946344,
+ "tokens_seen": 542179328
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006753059177532598,
+ "loss": 1.4308,
+ "theoretical_loss": 3.909504670129085,
+ "tokens_seen": 542244864
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006752898696088265,
+ "loss": 1.4758,
+ "theoretical_loss": 3.909454855270565,
+ "tokens_seen": 542310400
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006752738214643933,
+ "loss": 1.4732,
+ "theoretical_loss": 3.909405048116949,
+ "tokens_seen": 542375936
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006752577733199599,
+ "loss": 1.4179,
+ "theoretical_loss": 3.909355248666116,
+ "tokens_seen": 542441472
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006752417251755267,
+ "loss": 1.4859,
+ "theoretical_loss": 3.9093054569159436,
+ "tokens_seen": 542507008
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006752256770310933,
+ "loss": 1.4269,
+ "theoretical_loss": 3.909255672864311,
+ "tokens_seen": 542572544
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.00067520962888666,
+ "loss": 1.4713,
+ "theoretical_loss": 3.909205896509098,
+ "tokens_seen": 542638080
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751935807422267,
+ "loss": 1.4976,
+ "theoretical_loss": 3.9091561278481866,
+ "tokens_seen": 542703616
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751775325977935,
+ "loss": 1.5367,
+ "theoretical_loss": 3.909106366879457,
+ "tokens_seen": 542769152
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751614844533601,
+ "loss": 1.4783,
+ "theoretical_loss": 3.909056613600793,
+ "tokens_seen": 542834688
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751454363089269,
+ "loss": 1.5162,
+ "theoretical_loss": 3.9090068680100765,
+ "tokens_seen": 542900224
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751293881644935,
+ "loss": 1.4326,
+ "theoretical_loss": 3.9089571301051933,
+ "tokens_seen": 542965760
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006751133400200603,
+ "loss": 1.45,
+ "theoretical_loss": 3.9089073998840274,
+ "tokens_seen": 543031296
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750972918756269,
+ "loss": 1.5227,
+ "theoretical_loss": 3.908857677344465,
+ "tokens_seen": 543096832
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750812437311937,
+ "loss": 1.3709,
+ "theoretical_loss": 3.908807962484393,
+ "tokens_seen": 543162368
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750651955867603,
+ "loss": 1.5308,
+ "theoretical_loss": 3.908758255301699,
+ "tokens_seen": 543227904
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750491474423271,
+ "loss": 1.471,
+ "theoretical_loss": 3.9087085557942713,
+ "tokens_seen": 543293440
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750330992978937,
+ "loss": 1.5801,
+ "theoretical_loss": 3.908658863959999,
+ "tokens_seen": 543358976
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750170511534605,
+ "loss": 1.4352,
+ "theoretical_loss": 3.9086091797967724,
+ "tokens_seen": 543424512
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006750010030090271,
+ "loss": 1.3739,
+ "theoretical_loss": 3.908559503302482,
+ "tokens_seen": 543490048
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749849548645939,
+ "loss": 1.474,
+ "theoretical_loss": 3.9085098344750193,
+ "tokens_seen": 543555584
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749689067201605,
+ "loss": 1.3797,
+ "theoretical_loss": 3.908460173312278,
+ "tokens_seen": 543621120
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749528585757273,
+ "loss": 1.5149,
+ "theoretical_loss": 3.9084105198121506,
+ "tokens_seen": 543686656
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749368104312939,
+ "loss": 1.5111,
+ "theoretical_loss": 3.9083608739725317,
+ "tokens_seen": 543752192
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749207622868606,
+ "loss": 1.4539,
+ "theoretical_loss": 3.9083112357913166,
+ "tokens_seen": 543817728
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006749047141424273,
+ "loss": 1.4421,
+ "theoretical_loss": 3.9082616052664,
+ "tokens_seen": 543883264
+ },
+ {
+ "epoch": 0.16,
+ "objective/train/docs_used": 187447,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5180284976959229,
+ "objective/train/theoretical_loss": 3.90821198239568,
+ "objective/train/tokens_used": 564408800,
+ "theoretical_loss": 3.90821198239568,
+ "tokens_seen": 543948800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.000674888665997994,
+ "loss": 1.4843,
+ "theoretical_loss": 3.90821198239568,
+ "tokens_seen": 543948800
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006748726178535607,
+ "loss": 1.3792,
+ "theoretical_loss": 3.9081623671770536,
+ "tokens_seen": 544014336
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006748565697091274,
+ "loss": 1.6008,
+ "theoretical_loss": 3.9081127596084193,
+ "tokens_seen": 544079872
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006748405215646941,
+ "loss": 1.4053,
+ "theoretical_loss": 3.908063159687676,
+ "tokens_seen": 544145408
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006748244734202608,
+ "loss": 1.474,
+ "theoretical_loss": 3.9080135674127243,
+ "tokens_seen": 544210944
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006748084252758275,
+ "loss": 1.532,
+ "theoretical_loss": 3.9079639827814647,
+ "tokens_seen": 544276480
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006747923771313942,
+ "loss": 1.4896,
+ "theoretical_loss": 3.907914405791799,
+ "tokens_seen": 544342016
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006747763289869609,
+ "loss": 1.4168,
+ "theoretical_loss": 3.9078648364416297,
+ "tokens_seen": 544407552
+ },
+ {
+ "epoch": 0.16,
+ "learning_rate": 0.0006747602808425276,
+ "loss": 1.4467,
+ "theoretical_loss": 3.9078152747288604,
+ "tokens_seen": 544473088
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006747442326980943,
+ "loss": 1.5824,
+ "theoretical_loss": 3.907765720651395,
+ "tokens_seen": 544538624
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000674728184553661,
+ "loss": 1.5888,
+ "theoretical_loss": 3.907716174207139,
+ "tokens_seen": 544604160
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006747121364092277,
+ "loss": 1.5042,
+ "theoretical_loss": 3.9076666353939977,
+ "tokens_seen": 544669696
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746960882647944,
+ "loss": 1.5183,
+ "theoretical_loss": 3.9076171042098777,
+ "tokens_seen": 544735232
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746800401203611,
+ "loss": 1.4646,
+ "theoretical_loss": 3.907567580652687,
+ "tokens_seen": 544800768
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746639919759278,
+ "loss": 1.4199,
+ "theoretical_loss": 3.9075180647203336,
+ "tokens_seen": 544866304
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746479438314945,
+ "loss": 1.4331,
+ "theoretical_loss": 3.907468556410727,
+ "tokens_seen": 544931840
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746318956870612,
+ "loss": 1.4481,
+ "theoretical_loss": 3.907419055721777,
+ "tokens_seen": 544997376
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006746158475426279,
+ "loss": 1.4933,
+ "theoretical_loss": 3.907369562651394,
+ "tokens_seen": 545062912
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745997993981946,
+ "loss": 1.4835,
+ "theoretical_loss": 3.9073200771974905,
+ "tokens_seen": 545128448
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745837512537613,
+ "loss": 1.4513,
+ "theoretical_loss": 3.907270599357978,
+ "tokens_seen": 545193984
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000674567703109328,
+ "loss": 1.5083,
+ "theoretical_loss": 3.9072211291307704,
+ "tokens_seen": 545259520
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745516549648947,
+ "loss": 1.4462,
+ "theoretical_loss": 3.907171666513781,
+ "tokens_seen": 545325056
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745356068204614,
+ "loss": 1.4709,
+ "theoretical_loss": 3.907122211504926,
+ "tokens_seen": 545390592
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745195586760281,
+ "loss": 1.5978,
+ "theoretical_loss": 3.9070727641021206,
+ "tokens_seen": 545456128
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006745035105315948,
+ "loss": 1.4448,
+ "theoretical_loss": 3.907023324303281,
+ "tokens_seen": 545521664
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006744874623871615,
+ "loss": 1.4641,
+ "theoretical_loss": 3.9069738921063246,
+ "tokens_seen": 545587200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006744714142427282,
+ "loss": 1.5723,
+ "theoretical_loss": 3.9069244675091697,
+ "tokens_seen": 545652736
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006744553660982949,
+ "loss": 1.4894,
+ "theoretical_loss": 3.9068750505097354,
+ "tokens_seen": 545718272
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006744393179538616,
+ "loss": 1.4264,
+ "theoretical_loss": 3.906825641105942,
+ "tokens_seen": 545783808
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006744232698094283,
+ "loss": 1.4719,
+ "theoretical_loss": 3.9067762392957093,
+ "tokens_seen": 545849344
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000674407221664995,
+ "loss": 1.468,
+ "theoretical_loss": 3.90672684507696,
+ "tokens_seen": 545914880
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743911735205617,
+ "loss": 1.3939,
+ "theoretical_loss": 3.9066774584476147,
+ "tokens_seen": 545980416
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743751253761284,
+ "loss": 1.4936,
+ "theoretical_loss": 3.9066280794055976,
+ "tokens_seen": 546045952
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743590772316951,
+ "loss": 1.4917,
+ "theoretical_loss": 3.9065787079488326,
+ "tokens_seen": 546111488
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743430290872618,
+ "loss": 1.5062,
+ "theoretical_loss": 3.9065293440752447,
+ "tokens_seen": 546177024
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743269809428285,
+ "loss": 1.4176,
+ "theoretical_loss": 3.9064799877827587,
+ "tokens_seen": 546242560
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006743109327983952,
+ "loss": 1.5045,
+ "theoretical_loss": 3.9064306390693018,
+ "tokens_seen": 546308096
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006742948846539619,
+ "loss": 1.5332,
+ "theoretical_loss": 3.9063812979328,
+ "tokens_seen": 546373632
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006742788365095286,
+ "loss": 1.4321,
+ "theoretical_loss": 3.9063319643711827,
+ "tokens_seen": 546439168
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006742627883650953,
+ "loss": 1.4583,
+ "theoretical_loss": 3.906282638382377,
+ "tokens_seen": 546504704
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000674246740220662,
+ "loss": 1.4618,
+ "theoretical_loss": 3.906233319964315,
+ "tokens_seen": 546570240
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006742306920762287,
+ "loss": 1.4734,
+ "theoretical_loss": 3.9061840091149254,
+ "tokens_seen": 546635776
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006742146439317954,
+ "loss": 1.4433,
+ "theoretical_loss": 3.90613470583214,
+ "tokens_seen": 546701312
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741985957873621,
+ "loss": 1.4171,
+ "theoretical_loss": 3.9060854101138904,
+ "tokens_seen": 546766848
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741825476429288,
+ "loss": 1.552,
+ "theoretical_loss": 3.90603612195811,
+ "tokens_seen": 546832384
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741664994984955,
+ "loss": 1.4249,
+ "theoretical_loss": 3.9059868413627328,
+ "tokens_seen": 546897920
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741504513540622,
+ "loss": 1.4977,
+ "theoretical_loss": 3.905937568325692,
+ "tokens_seen": 546963456
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741344032096289,
+ "loss": 1.4874,
+ "theoretical_loss": 3.9058883028449247,
+ "tokens_seen": 547028992
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741183550651956,
+ "loss": 1.573,
+ "theoretical_loss": 3.905839044918366,
+ "tokens_seen": 547094528
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006741023069207623,
+ "loss": 1.4674,
+ "theoretical_loss": 3.905789794543953,
+ "tokens_seen": 547160064
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 188319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3917874097824097,
+ "objective/train/theoretical_loss": 3.9057405517196235,
+ "objective/train/tokens_used": 567685600,
+ "theoretical_loss": 3.9057405517196235,
+ "tokens_seen": 547225600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000674086258776329,
+ "loss": 1.3673,
+ "theoretical_loss": 3.9057405517196235,
+ "tokens_seen": 547225600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006740702106318957,
+ "loss": 1.4805,
+ "theoretical_loss": 3.905691316443316,
+ "tokens_seen": 547291136
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006740541624874624,
+ "loss": 1.4456,
+ "theoretical_loss": 3.90564208871297,
+ "tokens_seen": 547356672
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006740381143430291,
+ "loss": 1.4466,
+ "theoretical_loss": 3.9055928685265258,
+ "tokens_seen": 547422208
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006740220661985958,
+ "loss": 1.3941,
+ "theoretical_loss": 3.9055436558819245,
+ "tokens_seen": 547487744
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006740060180541625,
+ "loss": 1.3788,
+ "theoretical_loss": 3.905494450777107,
+ "tokens_seen": 547553280
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006739899699097292,
+ "loss": 1.6164,
+ "theoretical_loss": 3.9054452532100177,
+ "tokens_seen": 547618816
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673973921765296,
+ "loss": 1.4857,
+ "theoretical_loss": 3.905396063178598,
+ "tokens_seen": 547684352
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006739578736208626,
+ "loss": 1.4982,
+ "theoretical_loss": 3.905346880680794,
+ "tokens_seen": 547749888
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006739418254764293,
+ "loss": 1.355,
+ "theoretical_loss": 3.905297705714549,
+ "tokens_seen": 547815424
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673925777331996,
+ "loss": 1.496,
+ "theoretical_loss": 3.9052485382778097,
+ "tokens_seen": 547880960
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006739097291875627,
+ "loss": 1.4519,
+ "theoretical_loss": 3.905199378368523,
+ "tokens_seen": 547946496
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006738936810431294,
+ "loss": 1.5708,
+ "theoretical_loss": 3.9051502259846362,
+ "tokens_seen": 548012032
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006738776328986962,
+ "loss": 1.4104,
+ "theoretical_loss": 3.9051010811240974,
+ "tokens_seen": 548077568
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006738615847542629,
+ "loss": 1.6264,
+ "theoretical_loss": 3.905051943784856,
+ "tokens_seen": 548143104
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006738455366098296,
+ "loss": 1.4225,
+ "theoretical_loss": 3.905002813964861,
+ "tokens_seen": 548208640
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006738294884653963,
+ "loss": 1.4701,
+ "theoretical_loss": 3.904953691662064,
+ "tokens_seen": 548274176
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673813440320963,
+ "loss": 1.4369,
+ "theoretical_loss": 3.904904576874416,
+ "tokens_seen": 548339712
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737973921765297,
+ "loss": 1.4516,
+ "theoretical_loss": 3.90485546959987,
+ "tokens_seen": 548405248
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737813440320964,
+ "loss": 1.4014,
+ "theoretical_loss": 3.904806369836378,
+ "tokens_seen": 548470784
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737652958876631,
+ "loss": 1.4709,
+ "theoretical_loss": 3.904757277581895,
+ "tokens_seen": 548536320
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737492477432298,
+ "loss": 1.55,
+ "theoretical_loss": 3.9047081928343745,
+ "tokens_seen": 548601856
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737331995987965,
+ "loss": 1.497,
+ "theoretical_loss": 3.904659115591773,
+ "tokens_seen": 548667392
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737171514543632,
+ "loss": 1.4167,
+ "theoretical_loss": 3.904610045852046,
+ "tokens_seen": 548732928
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006737011033099299,
+ "loss": 1.5021,
+ "theoretical_loss": 3.904560983613152,
+ "tokens_seen": 548798464
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006736850551654966,
+ "loss": 1.4396,
+ "theoretical_loss": 3.9045119288730468,
+ "tokens_seen": 548864000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006736690070210632,
+ "loss": 1.5089,
+ "theoretical_loss": 3.9044628816296907,
+ "tokens_seen": 548929536
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.00067365295887663,
+ "loss": 1.3947,
+ "theoretical_loss": 3.9044138418810426,
+ "tokens_seen": 548995072
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006736369107321966,
+ "loss": 1.4812,
+ "theoretical_loss": 3.904364809625063,
+ "tokens_seen": 549060608
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006736208625877634,
+ "loss": 1.3479,
+ "theoretical_loss": 3.9043157848597128,
+ "tokens_seen": 549126144
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.00067360481444333,
+ "loss": 1.3436,
+ "theoretical_loss": 3.904266767582954,
+ "tokens_seen": 549191680
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735887662988968,
+ "loss": 1.4552,
+ "theoretical_loss": 3.9042177577927495,
+ "tokens_seen": 549257216
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735727181544634,
+ "loss": 1.4793,
+ "theoretical_loss": 3.904168755487062,
+ "tokens_seen": 549322752
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735566700100302,
+ "loss": 1.4728,
+ "theoretical_loss": 3.9041197606638565,
+ "tokens_seen": 549388288
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735406218655968,
+ "loss": 1.5343,
+ "theoretical_loss": 3.9040707733210978,
+ "tokens_seen": 549453824
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735245737211635,
+ "loss": 1.5119,
+ "theoretical_loss": 3.9040217934567525,
+ "tokens_seen": 549519360
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006735085255767302,
+ "loss": 1.4605,
+ "theoretical_loss": 3.903972821068786,
+ "tokens_seen": 549584896
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006734924774322969,
+ "loss": 1.3218,
+ "theoretical_loss": 3.903923856155167,
+ "tokens_seen": 549650432
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006734764292878636,
+ "loss": 1.4982,
+ "theoretical_loss": 3.9038748987138625,
+ "tokens_seen": 549715968
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006734603811434303,
+ "loss": 1.4314,
+ "theoretical_loss": 3.9038259487428424,
+ "tokens_seen": 549781504
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673444332998997,
+ "loss": 1.4938,
+ "theoretical_loss": 3.9037770062400767,
+ "tokens_seen": 549847040
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006734282848545637,
+ "loss": 1.4628,
+ "theoretical_loss": 3.903728071203535,
+ "tokens_seen": 549912576
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006734122367101304,
+ "loss": 1.3642,
+ "theoretical_loss": 3.90367914363119,
+ "tokens_seen": 549978112
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733961885656971,
+ "loss": 1.4482,
+ "theoretical_loss": 3.903630223521013,
+ "tokens_seen": 550043648
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733801404212638,
+ "loss": 1.5198,
+ "theoretical_loss": 3.903581310870978,
+ "tokens_seen": 550109184
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733640922768305,
+ "loss": 1.4351,
+ "theoretical_loss": 3.9035324056790577,
+ "tokens_seen": 550174720
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733480441323972,
+ "loss": 1.4349,
+ "theoretical_loss": 3.9034835079432275,
+ "tokens_seen": 550240256
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733319959879639,
+ "loss": 1.5413,
+ "theoretical_loss": 3.9034346176614623,
+ "tokens_seen": 550305792
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006733159478435306,
+ "loss": 1.5968,
+ "theoretical_loss": 3.9033857348317387,
+ "tokens_seen": 550371328
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732998996990973,
+ "loss": 1.5534,
+ "theoretical_loss": 3.9033368594520335,
+ "tokens_seen": 550436864
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 189606,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5754977464675903,
+ "objective/train/theoretical_loss": 3.903287991520324,
+ "objective/train/tokens_used": 570962400,
+ "theoretical_loss": 3.903287991520324,
+ "tokens_seen": 550502400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673283851554664,
+ "loss": 1.502,
+ "theoretical_loss": 3.903287991520324,
+ "tokens_seen": 550502400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732678034102307,
+ "loss": 1.4596,
+ "theoretical_loss": 3.903239131034589,
+ "tokens_seen": 550567936
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732517552657974,
+ "loss": 1.4343,
+ "theoretical_loss": 3.9031902779928087,
+ "tokens_seen": 550633472
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732357071213641,
+ "loss": 1.4176,
+ "theoretical_loss": 3.903141432392962,
+ "tokens_seen": 550699008
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732196589769308,
+ "loss": 1.4912,
+ "theoretical_loss": 3.903092594233031,
+ "tokens_seen": 550764544
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006732036108324975,
+ "loss": 1.4471,
+ "theoretical_loss": 3.903043763510996,
+ "tokens_seen": 550830080
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006731875626880642,
+ "loss": 1.4121,
+ "theoretical_loss": 3.9029949402248407,
+ "tokens_seen": 550895616
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006731715145436309,
+ "loss": 1.4698,
+ "theoretical_loss": 3.902946124372548,
+ "tokens_seen": 550961152
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006731554663991976,
+ "loss": 1.4843,
+ "theoretical_loss": 3.902897315952102,
+ "tokens_seen": 551026688
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006731394182547643,
+ "loss": 1.4583,
+ "theoretical_loss": 3.9028485149614873,
+ "tokens_seen": 551092224
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000673123370110331,
+ "loss": 1.4265,
+ "theoretical_loss": 3.90279972139869,
+ "tokens_seen": 551157760
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006731073219658977,
+ "loss": 1.4766,
+ "theoretical_loss": 3.902750935261696,
+ "tokens_seen": 551223296
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730912738214644,
+ "loss": 1.5156,
+ "theoretical_loss": 3.902702156548493,
+ "tokens_seen": 551288832
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730752256770311,
+ "loss": 1.5538,
+ "theoretical_loss": 3.9026533852570684,
+ "tokens_seen": 551354368
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730591775325978,
+ "loss": 1.4477,
+ "theoretical_loss": 3.9026046213854118,
+ "tokens_seen": 551419904
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730431293881645,
+ "loss": 1.4707,
+ "theoretical_loss": 3.9025558649315117,
+ "tokens_seen": 551485440
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730270812437312,
+ "loss": 1.5043,
+ "theoretical_loss": 3.9025071158933597,
+ "tokens_seen": 551550976
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006730110330992979,
+ "loss": 1.4054,
+ "theoretical_loss": 3.9024583742689463,
+ "tokens_seen": 551616512
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006729949849548646,
+ "loss": 1.3994,
+ "theoretical_loss": 3.902409640056263,
+ "tokens_seen": 551682048
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006729789368104313,
+ "loss": 1.427,
+ "theoretical_loss": 3.902360913253303,
+ "tokens_seen": 551747584
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672962888665998,
+ "loss": 1.4008,
+ "theoretical_loss": 3.9023121938580605,
+ "tokens_seen": 551813120
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006729468405215647,
+ "loss": 1.4376,
+ "theoretical_loss": 3.9022634818685282,
+ "tokens_seen": 551878656
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006729307923771314,
+ "loss": 1.554,
+ "theoretical_loss": 3.902214777282702,
+ "tokens_seen": 551944192
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006729147442326981,
+ "loss": 1.3726,
+ "theoretical_loss": 3.902166080098578,
+ "tokens_seen": 552009728
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728986960882648,
+ "loss": 1.394,
+ "theoretical_loss": 3.902117390314152,
+ "tokens_seen": 552075264
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728826479438315,
+ "loss": 1.5451,
+ "theoretical_loss": 3.902068707927422,
+ "tokens_seen": 552140800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728665997993982,
+ "loss": 1.5443,
+ "theoretical_loss": 3.9020200329363863,
+ "tokens_seen": 552206336
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728505516549649,
+ "loss": 1.4525,
+ "theoretical_loss": 3.901971365339043,
+ "tokens_seen": 552271872
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728345035105316,
+ "loss": 1.345,
+ "theoretical_loss": 3.9019227051333925,
+ "tokens_seen": 552337408
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006728184553660983,
+ "loss": 1.4898,
+ "theoretical_loss": 3.901874052317435,
+ "tokens_seen": 552402944
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672802407221665,
+ "loss": 1.5711,
+ "theoretical_loss": 3.9018254068891727,
+ "tokens_seen": 552468480
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727863590772317,
+ "loss": 1.6135,
+ "theoretical_loss": 3.901776768846606,
+ "tokens_seen": 552534016
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727703109327984,
+ "loss": 1.6138,
+ "theoretical_loss": 3.901728138187739,
+ "tokens_seen": 552599552
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727542627883651,
+ "loss": 1.3739,
+ "theoretical_loss": 3.9016795149105747,
+ "tokens_seen": 552665088
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727382146439318,
+ "loss": 1.3964,
+ "theoretical_loss": 3.901630899013118,
+ "tokens_seen": 552730624
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727221664994985,
+ "loss": 1.4253,
+ "theoretical_loss": 3.901582290493374,
+ "tokens_seen": 552796160
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006727061183550652,
+ "loss": 1.4719,
+ "theoretical_loss": 3.901533689349348,
+ "tokens_seen": 552861696
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006726900702106319,
+ "loss": 1.4899,
+ "theoretical_loss": 3.9014850955790474,
+ "tokens_seen": 552927232
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006726740220661986,
+ "loss": 1.3868,
+ "theoretical_loss": 3.9014365091804795,
+ "tokens_seen": 552992768
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006726579739217654,
+ "loss": 1.3792,
+ "theoretical_loss": 3.9013879301516523,
+ "tokens_seen": 553058304
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672641925777332,
+ "loss": 1.5142,
+ "theoretical_loss": 3.901339358490575,
+ "tokens_seen": 553123840
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006726258776328988,
+ "loss": 1.3746,
+ "theoretical_loss": 3.9012907941952575,
+ "tokens_seen": 553189376
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006726098294884654,
+ "loss": 1.4594,
+ "theoretical_loss": 3.9012422372637103,
+ "tokens_seen": 553254912
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006725937813440322,
+ "loss": 1.5213,
+ "theoretical_loss": 3.901193687693945,
+ "tokens_seen": 553320448
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006725777331995988,
+ "loss": 1.4227,
+ "theoretical_loss": 3.9011451454839734,
+ "tokens_seen": 553385984
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006725616850551656,
+ "loss": 1.5528,
+ "theoretical_loss": 3.901096610631808,
+ "tokens_seen": 553451520
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006725456369107322,
+ "loss": 1.4144,
+ "theoretical_loss": 3.901048083135464,
+ "tokens_seen": 553517056
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672529588766299,
+ "loss": 1.4536,
+ "theoretical_loss": 3.900999562992954,
+ "tokens_seen": 553582592
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006725135406218657,
+ "loss": 1.4655,
+ "theoretical_loss": 3.9009510502022944,
+ "tokens_seen": 553648128
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724974924774324,
+ "loss": 1.5144,
+ "theoretical_loss": 3.9009025447615002,
+ "tokens_seen": 553713664
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 190485,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3175182342529297,
+ "objective/train/theoretical_loss": 3.9008540466685897,
+ "objective/train/tokens_used": 574239200,
+ "theoretical_loss": 3.9008540466685897,
+ "tokens_seen": 553779200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672481444332999,
+ "loss": 1.3439,
+ "theoretical_loss": 3.9008540466685897,
+ "tokens_seen": 553779200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724653961885658,
+ "loss": 1.4714,
+ "theoretical_loss": 3.900805555921579,
+ "tokens_seen": 553844736
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724493480441325,
+ "loss": 1.4252,
+ "theoretical_loss": 3.9007570725184872,
+ "tokens_seen": 553910272
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724332998996992,
+ "loss": 1.4653,
+ "theoretical_loss": 3.900708596457333,
+ "tokens_seen": 553975808
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724172517552659,
+ "loss": 1.4083,
+ "theoretical_loss": 3.900660127736136,
+ "tokens_seen": 554041344
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006724012036108326,
+ "loss": 1.4722,
+ "theoretical_loss": 3.9006116663529173,
+ "tokens_seen": 554106880
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006723851554663993,
+ "loss": 1.476,
+ "theoretical_loss": 3.9005632123056984,
+ "tokens_seen": 554172416
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672369107321966,
+ "loss": 1.5502,
+ "theoretical_loss": 3.9005147655925008,
+ "tokens_seen": 554237952
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006723530591775327,
+ "loss": 1.4592,
+ "theoretical_loss": 3.9004663262113475,
+ "tokens_seen": 554303488
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006723370110330994,
+ "loss": 1.5817,
+ "theoretical_loss": 3.900417894160263,
+ "tokens_seen": 554369024
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006723209628886661,
+ "loss": 1.4163,
+ "theoretical_loss": 3.900369469437271,
+ "tokens_seen": 554434560
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006723049147442328,
+ "loss": 1.4168,
+ "theoretical_loss": 3.900321052040397,
+ "tokens_seen": 554500096
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722888665997995,
+ "loss": 1.3777,
+ "theoretical_loss": 3.900272641967666,
+ "tokens_seen": 554565632
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722728184553661,
+ "loss": 1.4316,
+ "theoretical_loss": 3.900224239217106,
+ "tokens_seen": 554631168
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722567703109329,
+ "loss": 1.4393,
+ "theoretical_loss": 3.9001758437867435,
+ "tokens_seen": 554696704
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722407221664995,
+ "loss": 1.5377,
+ "theoretical_loss": 3.900127455674608,
+ "tokens_seen": 554762240
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722246740220663,
+ "loss": 1.371,
+ "theoretical_loss": 3.9000790748787275,
+ "tokens_seen": 554827776
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006722086258776329,
+ "loss": 1.4876,
+ "theoretical_loss": 3.9000307013971325,
+ "tokens_seen": 554893312
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721925777331997,
+ "loss": 1.5017,
+ "theoretical_loss": 3.899982335227852,
+ "tokens_seen": 554958848
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721765295887663,
+ "loss": 1.4599,
+ "theoretical_loss": 3.89993397636892,
+ "tokens_seen": 555024384
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721604814443331,
+ "loss": 1.5532,
+ "theoretical_loss": 3.899885624818366,
+ "tokens_seen": 555089920
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721444332998997,
+ "loss": 1.4082,
+ "theoretical_loss": 3.8998372805742245,
+ "tokens_seen": 555155456
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721283851554664,
+ "loss": 1.4126,
+ "theoretical_loss": 3.8997889436345283,
+ "tokens_seen": 555220992
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006721123370110331,
+ "loss": 1.5716,
+ "theoretical_loss": 3.899740613997312,
+ "tokens_seen": 555286528
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720962888665998,
+ "loss": 1.5568,
+ "theoretical_loss": 3.8996922916606103,
+ "tokens_seen": 555352064
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720802407221665,
+ "loss": 1.4742,
+ "theoretical_loss": 3.89964397662246,
+ "tokens_seen": 555417600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720641925777332,
+ "loss": 1.4739,
+ "theoretical_loss": 3.899595668880897,
+ "tokens_seen": 555483136
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720481444332999,
+ "loss": 1.418,
+ "theoretical_loss": 3.899547368433959,
+ "tokens_seen": 555548672
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720320962888666,
+ "loss": 1.4476,
+ "theoretical_loss": 3.899499075279684,
+ "tokens_seen": 555614208
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006720160481444333,
+ "loss": 1.4526,
+ "theoretical_loss": 3.899450789416111,
+ "tokens_seen": 555679744
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000672,
+ "loss": 1.522,
+ "theoretical_loss": 3.8994025108412793,
+ "tokens_seen": 555745280
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719839518555667,
+ "loss": 1.4267,
+ "theoretical_loss": 3.8993542395532304,
+ "tokens_seen": 555810816
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719679037111334,
+ "loss": 1.5176,
+ "theoretical_loss": 3.8993059755500044,
+ "tokens_seen": 555876352
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719518555667001,
+ "loss": 1.4775,
+ "theoretical_loss": 3.8992577188296433,
+ "tokens_seen": 555941888
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719358074222668,
+ "loss": 1.5034,
+ "theoretical_loss": 3.8992094693901906,
+ "tokens_seen": 556007424
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719197592778335,
+ "loss": 1.428,
+ "theoretical_loss": 3.8991612272296896,
+ "tokens_seen": 556072960
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006719037111334002,
+ "loss": 1.4193,
+ "theoretical_loss": 3.899112992346184,
+ "tokens_seen": 556138496
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006718876629889669,
+ "loss": 1.4228,
+ "theoretical_loss": 3.899064764737719,
+ "tokens_seen": 556204032
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006718716148445336,
+ "loss": 1.4621,
+ "theoretical_loss": 3.8990165444023406,
+ "tokens_seen": 556269568
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006718555667001003,
+ "loss": 1.5343,
+ "theoretical_loss": 3.8989683313380947,
+ "tokens_seen": 556335104
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000671839518555667,
+ "loss": 1.4186,
+ "theoretical_loss": 3.8989201255430292,
+ "tokens_seen": 556400640
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006718234704112337,
+ "loss": 1.5301,
+ "theoretical_loss": 3.8988719270151924,
+ "tokens_seen": 556466176
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006718074222668004,
+ "loss": 1.4505,
+ "theoretical_loss": 3.8988237357526314,
+ "tokens_seen": 556531712
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717913741223671,
+ "loss": 1.4219,
+ "theoretical_loss": 3.8987755517533973,
+ "tokens_seen": 556597248
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717753259779338,
+ "loss": 1.5398,
+ "theoretical_loss": 3.8987273750155396,
+ "tokens_seen": 556662784
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717592778335005,
+ "loss": 1.539,
+ "theoretical_loss": 3.89867920553711,
+ "tokens_seen": 556728320
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717432296890672,
+ "loss": 1.4986,
+ "theoretical_loss": 3.8986310433161595,
+ "tokens_seen": 556793856
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717271815446339,
+ "loss": 1.4106,
+ "theoretical_loss": 3.898582888350741,
+ "tokens_seen": 556859392
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006717111334002006,
+ "loss": 1.4857,
+ "theoretical_loss": 3.8985347406389077,
+ "tokens_seen": 556924928
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006716950852557673,
+ "loss": 1.3531,
+ "theoretical_loss": 3.898486600178714,
+ "tokens_seen": 556990464
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 191608,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5120327472686768,
+ "objective/train/theoretical_loss": 3.898438466968214,
+ "objective/train/tokens_used": 577516000,
+ "theoretical_loss": 3.898438466968214,
+ "tokens_seen": 557056000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000671679037111334,
+ "loss": 1.4921,
+ "theoretical_loss": 3.898438466968214,
+ "tokens_seen": 557056000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006716629889669007,
+ "loss": 1.4352,
+ "theoretical_loss": 3.8983903410054634,
+ "tokens_seen": 557121536
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006716469408224674,
+ "loss": 1.4466,
+ "theoretical_loss": 3.8983422222885187,
+ "tokens_seen": 557187072
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006716308926780341,
+ "loss": 1.4628,
+ "theoretical_loss": 3.8982941108154368,
+ "tokens_seen": 557252608
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006716148445336008,
+ "loss": 1.3886,
+ "theoretical_loss": 3.8982460065842757,
+ "tokens_seen": 557318144
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715987963891675,
+ "loss": 1.4199,
+ "theoretical_loss": 3.8981979095930934,
+ "tokens_seen": 557383680
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715827482447342,
+ "loss": 1.5233,
+ "theoretical_loss": 3.89814981983995,
+ "tokens_seen": 557449216
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715667001003009,
+ "loss": 1.39,
+ "theoretical_loss": 3.8981017373229045,
+ "tokens_seen": 557514752
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715506519558676,
+ "loss": 1.4756,
+ "theoretical_loss": 3.8980536620400184,
+ "tokens_seen": 557580288
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715346038114343,
+ "loss": 1.3975,
+ "theoretical_loss": 3.8980055939893523,
+ "tokens_seen": 557645824
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000671518555667001,
+ "loss": 1.4887,
+ "theoretical_loss": 3.89795753316897,
+ "tokens_seen": 557711360
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006715025075225677,
+ "loss": 1.4431,
+ "theoretical_loss": 3.8979094795769336,
+ "tokens_seen": 557776896
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714864593781344,
+ "loss": 1.5638,
+ "theoretical_loss": 3.8978614332113066,
+ "tokens_seen": 557842432
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714704112337011,
+ "loss": 1.5003,
+ "theoretical_loss": 3.8978133940701536,
+ "tokens_seen": 557907968
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714543630892679,
+ "loss": 1.3973,
+ "theoretical_loss": 3.8977653621515405,
+ "tokens_seen": 557973504
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714383149448345,
+ "loss": 1.4663,
+ "theoretical_loss": 3.8977173374535328,
+ "tokens_seen": 558039040
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714222668004013,
+ "loss": 1.5457,
+ "theoretical_loss": 3.8976693199741974,
+ "tokens_seen": 558104576
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006714062186559679,
+ "loss": 1.4807,
+ "theoretical_loss": 3.8976213097116013,
+ "tokens_seen": 558170112
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713901705115347,
+ "loss": 1.4396,
+ "theoretical_loss": 3.8975733066638134,
+ "tokens_seen": 558235648
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713741223671013,
+ "loss": 1.5034,
+ "theoretical_loss": 3.897525310828902,
+ "tokens_seen": 558301184
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713580742226681,
+ "loss": 1.3895,
+ "theoretical_loss": 3.8974773222049377,
+ "tokens_seen": 558366720
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713420260782347,
+ "loss": 1.4724,
+ "theoretical_loss": 3.89742934078999,
+ "tokens_seen": 558432256
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713259779338015,
+ "loss": 1.3561,
+ "theoretical_loss": 3.8973813665821306,
+ "tokens_seen": 558497792
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006713099297893681,
+ "loss": 1.5061,
+ "theoretical_loss": 3.8973333995794315,
+ "tokens_seen": 558563328
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006712938816449349,
+ "loss": 1.4293,
+ "theoretical_loss": 3.897285439779965,
+ "tokens_seen": 558628864
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006712778335005015,
+ "loss": 1.3869,
+ "theoretical_loss": 3.8972374871818047,
+ "tokens_seen": 558694400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006712617853560683,
+ "loss": 1.5328,
+ "theoretical_loss": 3.8971895417830247,
+ "tokens_seen": 558759936
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000671245737211635,
+ "loss": 1.4658,
+ "theoretical_loss": 3.8971416035817006,
+ "tokens_seen": 558825472
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006712296890672017,
+ "loss": 1.4214,
+ "theoretical_loss": 3.8970936725759073,
+ "tokens_seen": 558891008
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006712136409227683,
+ "loss": 1.4339,
+ "theoretical_loss": 3.8970457487637216,
+ "tokens_seen": 558956544
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711975927783351,
+ "loss": 1.4222,
+ "theoretical_loss": 3.89699783214322,
+ "tokens_seen": 559022080
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711815446339017,
+ "loss": 1.5009,
+ "theoretical_loss": 3.8969499227124804,
+ "tokens_seen": 559087616
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711654964894685,
+ "loss": 1.3898,
+ "theoretical_loss": 3.8969020204695823,
+ "tokens_seen": 559153152
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711494483450352,
+ "loss": 1.5702,
+ "theoretical_loss": 3.8968541254126046,
+ "tokens_seen": 559218688
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711334002006019,
+ "loss": 1.4097,
+ "theoretical_loss": 3.896806237539627,
+ "tokens_seen": 559284224
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711173520561686,
+ "loss": 1.5051,
+ "theoretical_loss": 3.8967583568487307,
+ "tokens_seen": 559349760
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006711013039117353,
+ "loss": 1.4303,
+ "theoretical_loss": 3.8967104833379973,
+ "tokens_seen": 559415296
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000671085255767302,
+ "loss": 1.5536,
+ "theoretical_loss": 3.8966626170055085,
+ "tokens_seen": 559480832
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006710692076228687,
+ "loss": 1.4769,
+ "theoretical_loss": 3.896614757849348,
+ "tokens_seen": 559546368
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006710531594784354,
+ "loss": 1.4104,
+ "theoretical_loss": 3.8965669058675996,
+ "tokens_seen": 559611904
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006710371113340021,
+ "loss": 1.5694,
+ "theoretical_loss": 3.8965190610583473,
+ "tokens_seen": 559677440
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006710210631895688,
+ "loss": 1.4871,
+ "theoretical_loss": 3.896471223419677,
+ "tokens_seen": 559742976
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006710050150451355,
+ "loss": 1.3676,
+ "theoretical_loss": 3.896423392949674,
+ "tokens_seen": 559808512
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006709889669007022,
+ "loss": 1.4637,
+ "theoretical_loss": 3.896375569646425,
+ "tokens_seen": 559874048
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006709729187562689,
+ "loss": 1.3481,
+ "theoretical_loss": 3.896327753508018,
+ "tokens_seen": 559939584
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006709568706118356,
+ "loss": 1.5005,
+ "theoretical_loss": 3.896279944532541,
+ "tokens_seen": 560005120
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006709408224674023,
+ "loss": 1.5627,
+ "theoretical_loss": 3.8962321427180826,
+ "tokens_seen": 560070656
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670924774322969,
+ "loss": 1.4232,
+ "theoretical_loss": 3.8961843480627323,
+ "tokens_seen": 560136192
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006709087261785357,
+ "loss": 1.4844,
+ "theoretical_loss": 3.8961365605645817,
+ "tokens_seen": 560201728
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708926780341024,
+ "loss": 1.4625,
+ "theoretical_loss": 3.89608878022172,
+ "tokens_seen": 560267264
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 192944,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4706947803497314,
+ "objective/train/theoretical_loss": 3.896041007032241,
+ "objective/train/tokens_used": 580792800,
+ "theoretical_loss": 3.896041007032241,
+ "tokens_seen": 560332800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708766298896691,
+ "loss": 1.4728,
+ "theoretical_loss": 3.896041007032241,
+ "tokens_seen": 560332800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708605817452358,
+ "loss": 1.5247,
+ "theoretical_loss": 3.895993240994236,
+ "tokens_seen": 560398336
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708445336008024,
+ "loss": 1.4447,
+ "theoretical_loss": 3.895945482105798,
+ "tokens_seen": 560463872
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708284854563692,
+ "loss": 1.4128,
+ "theoretical_loss": 3.8958977303650224,
+ "tokens_seen": 560529408
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006708124373119358,
+ "loss": 1.5002,
+ "theoretical_loss": 3.8958499857700035,
+ "tokens_seen": 560594944
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006707963891675026,
+ "loss": 1.424,
+ "theoretical_loss": 3.8958022483188364,
+ "tokens_seen": 560660480
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006707803410230692,
+ "loss": 1.5017,
+ "theoretical_loss": 3.895754518009617,
+ "tokens_seen": 560726016
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670764292878636,
+ "loss": 1.4005,
+ "theoretical_loss": 3.8957067948404434,
+ "tokens_seen": 560791552
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006707482447342026,
+ "loss": 1.5501,
+ "theoretical_loss": 3.895659078809412,
+ "tokens_seen": 560857088
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006707321965897693,
+ "loss": 1.5037,
+ "theoretical_loss": 3.895611369914622,
+ "tokens_seen": 560922624
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670716148445336,
+ "loss": 1.4869,
+ "theoretical_loss": 3.895563668154173,
+ "tokens_seen": 560988160
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006707001003009027,
+ "loss": 1.5724,
+ "theoretical_loss": 3.8955159735261637,
+ "tokens_seen": 561053696
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706840521564694,
+ "loss": 1.4572,
+ "theoretical_loss": 3.895468286028695,
+ "tokens_seen": 561119232
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706680040120361,
+ "loss": 1.3411,
+ "theoretical_loss": 3.8954206056598695,
+ "tokens_seen": 561184768
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706519558676028,
+ "loss": 1.4937,
+ "theoretical_loss": 3.8953729324177875,
+ "tokens_seen": 561250304
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706359077231695,
+ "loss": 1.436,
+ "theoretical_loss": 3.895325266300553,
+ "tokens_seen": 561315840
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706198595787362,
+ "loss": 1.545,
+ "theoretical_loss": 3.8952776073062685,
+ "tokens_seen": 561381376
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006706038114343029,
+ "loss": 1.404,
+ "theoretical_loss": 3.895229955433039,
+ "tokens_seen": 561446912
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006705877632898696,
+ "loss": 1.3153,
+ "theoretical_loss": 3.8951823106789694,
+ "tokens_seen": 561512448
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006705717151454363,
+ "loss": 1.4576,
+ "theoretical_loss": 3.895134673042165,
+ "tokens_seen": 561577984
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670555667001003,
+ "loss": 1.4926,
+ "theoretical_loss": 3.895087042520733,
+ "tokens_seen": 561643520
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006705396188565697,
+ "loss": 1.4651,
+ "theoretical_loss": 3.895039419112779,
+ "tokens_seen": 561709056
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006705235707121364,
+ "loss": 1.4288,
+ "theoretical_loss": 3.8949918028164126,
+ "tokens_seen": 561774592
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006705075225677031,
+ "loss": 1.4602,
+ "theoretical_loss": 3.8949441936297413,
+ "tokens_seen": 561840128
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704914744232698,
+ "loss": 1.4691,
+ "theoretical_loss": 3.894896591550875,
+ "tokens_seen": 561905664
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704754262788365,
+ "loss": 1.4379,
+ "theoretical_loss": 3.894848996577924,
+ "tokens_seen": 561971200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704593781344032,
+ "loss": 1.5564,
+ "theoretical_loss": 3.8948014087089975,
+ "tokens_seen": 562036736
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704433299899699,
+ "loss": 1.414,
+ "theoretical_loss": 3.8947538279422083,
+ "tokens_seen": 562102272
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704272818455366,
+ "loss": 1.2931,
+ "theoretical_loss": 3.8947062542756683,
+ "tokens_seen": 562167808
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006704112337011033,
+ "loss": 1.5324,
+ "theoretical_loss": 3.8946586877074907,
+ "tokens_seen": 562233344
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.00067039518555667,
+ "loss": 1.5057,
+ "theoretical_loss": 3.894611128235789,
+ "tokens_seen": 562298880
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006703791374122367,
+ "loss": 1.4521,
+ "theoretical_loss": 3.894563575858677,
+ "tokens_seen": 562364416
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006703630892678034,
+ "loss": 1.5724,
+ "theoretical_loss": 3.8945160305742705,
+ "tokens_seen": 562429952
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006703470411233701,
+ "loss": 1.5203,
+ "theoretical_loss": 3.8944684923806845,
+ "tokens_seen": 562495488
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006703309929789368,
+ "loss": 1.4232,
+ "theoretical_loss": 3.8944209612760368,
+ "tokens_seen": 562561024
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006703149448345035,
+ "loss": 1.4636,
+ "theoretical_loss": 3.894373437258443,
+ "tokens_seen": 562626560
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702988966900702,
+ "loss": 1.454,
+ "theoretical_loss": 3.8943259203260228,
+ "tokens_seen": 562692096
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702828485456369,
+ "loss": 1.4675,
+ "theoretical_loss": 3.8942784104768937,
+ "tokens_seen": 562757632
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702668004012036,
+ "loss": 1.3461,
+ "theoretical_loss": 3.8942309077091757,
+ "tokens_seen": 562823168
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702507522567704,
+ "loss": 1.4379,
+ "theoretical_loss": 3.894183412020988,
+ "tokens_seen": 562888704
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670234704112337,
+ "loss": 1.5108,
+ "theoretical_loss": 3.894135923410453,
+ "tokens_seen": 562954240
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702186559679038,
+ "loss": 1.418,
+ "theoretical_loss": 3.8940884418756907,
+ "tokens_seen": 563019776
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006702026078234704,
+ "loss": 1.491,
+ "theoretical_loss": 3.894040967414824,
+ "tokens_seen": 563085312
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006701865596790372,
+ "loss": 1.3897,
+ "theoretical_loss": 3.893993500025976,
+ "tokens_seen": 563150848
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006701705115346038,
+ "loss": 1.53,
+ "theoretical_loss": 3.893946039707271,
+ "tokens_seen": 563216384
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006701544633901706,
+ "loss": 1.5299,
+ "theoretical_loss": 3.893898586456832,
+ "tokens_seen": 563281920
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006701384152457372,
+ "loss": 1.3898,
+ "theoretical_loss": 3.893851140272785,
+ "tokens_seen": 563347456
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670122367101304,
+ "loss": 1.4234,
+ "theoretical_loss": 3.8938037011532556,
+ "tokens_seen": 563412992
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006701063189568706,
+ "loss": 1.506,
+ "theoretical_loss": 3.893756269096371,
+ "tokens_seen": 563478528
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006700902708124374,
+ "loss": 1.5202,
+ "theoretical_loss": 3.8937088441002574,
+ "tokens_seen": 563544064
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 193835,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5278165340423584,
+ "objective/train/theoretical_loss": 3.893661426163044,
+ "objective/train/tokens_used": 584069600,
+ "theoretical_loss": 3.893661426163044,
+ "tokens_seen": 563609600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000670074222668004,
+ "loss": 1.5765,
+ "theoretical_loss": 3.893661426163044,
+ "tokens_seen": 563609600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006700581745235708,
+ "loss": 1.502,
+ "theoretical_loss": 3.8936140152828584,
+ "tokens_seen": 563675136
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006700421263791374,
+ "loss": 1.4192,
+ "theoretical_loss": 3.8935666114578305,
+ "tokens_seen": 563740672
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006700260782347042,
+ "loss": 1.4857,
+ "theoretical_loss": 3.8935192146860906,
+ "tokens_seen": 563806208
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006700100300902708,
+ "loss": 1.5107,
+ "theoretical_loss": 3.8934718249657694,
+ "tokens_seen": 563871744
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006699939819458376,
+ "loss": 1.4932,
+ "theoretical_loss": 3.8934244422949984,
+ "tokens_seen": 563937280
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006699779338014042,
+ "loss": 1.4328,
+ "theoretical_loss": 3.89337706667191,
+ "tokens_seen": 564002816
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669961885656971,
+ "loss": 1.4154,
+ "theoretical_loss": 3.8933296980946372,
+ "tokens_seen": 564068352
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006699458375125376,
+ "loss": 1.4524,
+ "theoretical_loss": 3.8932823365613136,
+ "tokens_seen": 564133888
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006699297893681044,
+ "loss": 1.5674,
+ "theoretical_loss": 3.893234982070074,
+ "tokens_seen": 564199424
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669913741223671,
+ "loss": 1.478,
+ "theoretical_loss": 3.8931876346190517,
+ "tokens_seen": 564264960
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698976930792378,
+ "loss": 1.48,
+ "theoretical_loss": 3.8931402942063853,
+ "tokens_seen": 564330496
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698816449348044,
+ "loss": 1.5694,
+ "theoretical_loss": 3.89309296083021,
+ "tokens_seen": 564396032
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698655967903712,
+ "loss": 1.4293,
+ "theoretical_loss": 3.8930456344886624,
+ "tokens_seen": 564461568
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698495486459378,
+ "loss": 1.3529,
+ "theoretical_loss": 3.8929983151798817,
+ "tokens_seen": 564527104
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698335005015046,
+ "loss": 1.4832,
+ "theoretical_loss": 3.892951002902006,
+ "tokens_seen": 564592640
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006698174523570712,
+ "loss": 1.5263,
+ "theoretical_loss": 3.892903697653174,
+ "tokens_seen": 564658176
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669801404212638,
+ "loss": 1.4369,
+ "theoretical_loss": 3.8928563994315266,
+ "tokens_seen": 564723712
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006697853560682047,
+ "loss": 1.3109,
+ "theoretical_loss": 3.8928091082352045,
+ "tokens_seen": 564789248
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006697693079237714,
+ "loss": 1.3746,
+ "theoretical_loss": 3.8927618240623496,
+ "tokens_seen": 564854784
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669753259779338,
+ "loss": 1.524,
+ "theoretical_loss": 3.8927145469111033,
+ "tokens_seen": 564920320
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006697372116349048,
+ "loss": 1.4817,
+ "theoretical_loss": 3.892667276779609,
+ "tokens_seen": 564985856
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006697211634904715,
+ "loss": 1.4604,
+ "theoretical_loss": 3.89262001366601,
+ "tokens_seen": 565051392
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006697051153460382,
+ "loss": 1.4828,
+ "theoretical_loss": 3.892572757568451,
+ "tokens_seen": 565116928
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006696890672016049,
+ "loss": 1.419,
+ "theoretical_loss": 3.892525508485077,
+ "tokens_seen": 565182464
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006696730190571716,
+ "loss": 1.4302,
+ "theoretical_loss": 3.892478266414033,
+ "tokens_seen": 565248000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006696569709127383,
+ "loss": 1.4737,
+ "theoretical_loss": 3.8924310313534667,
+ "tokens_seen": 565313536
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669640922768305,
+ "loss": 1.436,
+ "theoretical_loss": 3.892383803301524,
+ "tokens_seen": 565379072
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006696248746238717,
+ "loss": 1.385,
+ "theoretical_loss": 3.8923365822563536,
+ "tokens_seen": 565444608
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006696088264794384,
+ "loss": 1.3827,
+ "theoretical_loss": 3.8922893682161037,
+ "tokens_seen": 565510144
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695927783350051,
+ "loss": 1.4165,
+ "theoretical_loss": 3.892242161178924,
+ "tokens_seen": 565575680
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695767301905718,
+ "loss": 1.3998,
+ "theoretical_loss": 3.892194961142964,
+ "tokens_seen": 565641216
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695606820461385,
+ "loss": 1.4822,
+ "theoretical_loss": 3.8921477681063736,
+ "tokens_seen": 565706752
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695446339017052,
+ "loss": 1.5143,
+ "theoretical_loss": 3.892100582067306,
+ "tokens_seen": 565772288
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695285857572719,
+ "loss": 1.5652,
+ "theoretical_loss": 3.8920534030239122,
+ "tokens_seen": 565837824
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006695125376128386,
+ "loss": 1.4798,
+ "theoretical_loss": 3.892006230974345,
+ "tokens_seen": 565903360
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694964894684053,
+ "loss": 1.423,
+ "theoretical_loss": 3.8919590659167578,
+ "tokens_seen": 565968896
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669480441323972,
+ "loss": 1.4732,
+ "theoretical_loss": 3.8919119078493045,
+ "tokens_seen": 566034432
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694643931795387,
+ "loss": 1.4357,
+ "theoretical_loss": 3.8918647567701417,
+ "tokens_seen": 566099968
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694483450351053,
+ "loss": 1.481,
+ "theoretical_loss": 3.8918176126774227,
+ "tokens_seen": 566165504
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694322968906721,
+ "loss": 1.513,
+ "theoretical_loss": 3.891770475569305,
+ "tokens_seen": 566231040
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694162487462387,
+ "loss": 1.4065,
+ "theoretical_loss": 3.8917233454439453,
+ "tokens_seen": 566296576
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006694002006018055,
+ "loss": 1.4289,
+ "theoretical_loss": 3.8916762222995014,
+ "tokens_seen": 566362112
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693841524573721,
+ "loss": 1.4075,
+ "theoretical_loss": 3.8916291061341317,
+ "tokens_seen": 566427648
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693681043129389,
+ "loss": 1.3731,
+ "theoretical_loss": 3.8915819969459946,
+ "tokens_seen": 566493184
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693520561685055,
+ "loss": 1.4602,
+ "theoretical_loss": 3.891534894733251,
+ "tokens_seen": 566558720
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693360080240723,
+ "loss": 1.4231,
+ "theoretical_loss": 3.891487799494061,
+ "tokens_seen": 566624256
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693199598796389,
+ "loss": 1.3516,
+ "theoretical_loss": 3.891440711226585,
+ "tokens_seen": 566689792
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006693039117352056,
+ "loss": 1.4852,
+ "theoretical_loss": 3.891393629928986,
+ "tokens_seen": 566755328
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006692878635907723,
+ "loss": 1.5724,
+ "theoretical_loss": 3.8913465555994255,
+ "tokens_seen": 566820864
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 195104,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5167372226715088,
+ "objective/train/theoretical_loss": 3.8912994882360676,
+ "objective/train/tokens_used": 587346400,
+ "theoretical_loss": 3.8912994882360676,
+ "tokens_seen": 566886400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669271815446339,
+ "loss": 1.5297,
+ "theoretical_loss": 3.8912994882360676,
+ "tokens_seen": 566886400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006692557673019057,
+ "loss": 1.4265,
+ "theoretical_loss": 3.891252427837076,
+ "tokens_seen": 566951936
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006692397191574724,
+ "loss": 1.4251,
+ "theoretical_loss": 3.8912053744006156,
+ "tokens_seen": 567017472
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006692236710130391,
+ "loss": 1.5477,
+ "theoretical_loss": 3.8911583279248507,
+ "tokens_seen": 567083008
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006692076228686058,
+ "loss": 1.4827,
+ "theoretical_loss": 3.8911112884079486,
+ "tokens_seen": 567148544
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006691915747241725,
+ "loss": 1.3542,
+ "theoretical_loss": 3.8910642558480752,
+ "tokens_seen": 567214080
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006691755265797392,
+ "loss": 1.5154,
+ "theoretical_loss": 3.8910172302433987,
+ "tokens_seen": 567279616
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006691594784353059,
+ "loss": 1.3885,
+ "theoretical_loss": 3.8909702115920863,
+ "tokens_seen": 567345152
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006691434302908726,
+ "loss": 1.4874,
+ "theoretical_loss": 3.890923199892308,
+ "tokens_seen": 567410688
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006691273821464393,
+ "loss": 1.4088,
+ "theoretical_loss": 3.890876195142232,
+ "tokens_seen": 567476224
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000669111334002006,
+ "loss": 1.4489,
+ "theoretical_loss": 3.8908291973400297,
+ "tokens_seen": 567541760
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690952858575727,
+ "loss": 1.3928,
+ "theoretical_loss": 3.890782206483871,
+ "tokens_seen": 567607296
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690792377131394,
+ "loss": 1.4766,
+ "theoretical_loss": 3.8907352225719283,
+ "tokens_seen": 567672832
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690631895687061,
+ "loss": 1.3147,
+ "theoretical_loss": 3.890688245602373,
+ "tokens_seen": 567738368
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690471414242729,
+ "loss": 1.4016,
+ "theoretical_loss": 3.890641275573379,
+ "tokens_seen": 567803904
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690310932798395,
+ "loss": 1.3566,
+ "theoretical_loss": 3.8905943124831195,
+ "tokens_seen": 567869440
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006690150451354063,
+ "loss": 1.3658,
+ "theoretical_loss": 3.890547356329769,
+ "tokens_seen": 567934976
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689989969909729,
+ "loss": 1.4371,
+ "theoretical_loss": 3.890500407111502,
+ "tokens_seen": 568000512
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689829488465397,
+ "loss": 1.4648,
+ "theoretical_loss": 3.8904534648264955,
+ "tokens_seen": 568066048
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689669007021063,
+ "loss": 1.3354,
+ "theoretical_loss": 3.8904065294729246,
+ "tokens_seen": 568131584
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689508525576731,
+ "loss": 1.5189,
+ "theoretical_loss": 3.890359601048967,
+ "tokens_seen": 568197120
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689348044132397,
+ "loss": 1.5132,
+ "theoretical_loss": 3.8903126795528014,
+ "tokens_seen": 568262656
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689187562688065,
+ "loss": 1.4662,
+ "theoretical_loss": 3.8902657649826047,
+ "tokens_seen": 568328192
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006689027081243731,
+ "loss": 1.4783,
+ "theoretical_loss": 3.890218857336557,
+ "tokens_seen": 568393728
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688866599799399,
+ "loss": 1.3859,
+ "theoretical_loss": 3.890171956612838,
+ "tokens_seen": 568459264
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688706118355065,
+ "loss": 1.4601,
+ "theoretical_loss": 3.890125062809628,
+ "tokens_seen": 568524800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688545636910733,
+ "loss": 1.4535,
+ "theoretical_loss": 3.8900781759251086,
+ "tokens_seen": 568590336
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688385155466399,
+ "loss": 1.3373,
+ "theoretical_loss": 3.8900312959574617,
+ "tokens_seen": 568655872
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688224674022067,
+ "loss": 1.4347,
+ "theoretical_loss": 3.88998442290487,
+ "tokens_seen": 568721408
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006688064192577733,
+ "loss": 1.4231,
+ "theoretical_loss": 3.889937556765517,
+ "tokens_seen": 568786944
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687903711133401,
+ "loss": 1.3824,
+ "theoretical_loss": 3.8898906975375858,
+ "tokens_seen": 568852480
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687743229689067,
+ "loss": 1.4538,
+ "theoretical_loss": 3.8898438452192616,
+ "tokens_seen": 568918016
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687582748244735,
+ "loss": 1.4741,
+ "theoretical_loss": 3.8897969998087305,
+ "tokens_seen": 568983552
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687422266800401,
+ "loss": 1.4618,
+ "theoretical_loss": 3.8897501613041783,
+ "tokens_seen": 569049088
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687261785356069,
+ "loss": 1.4505,
+ "theoretical_loss": 3.8897033297037904,
+ "tokens_seen": 569114624
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006687101303911735,
+ "loss": 1.429,
+ "theoretical_loss": 3.889656505005756,
+ "tokens_seen": 569180160
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686940822467403,
+ "loss": 1.5221,
+ "theoretical_loss": 3.889609687208262,
+ "tokens_seen": 569245696
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686780341023069,
+ "loss": 1.4228,
+ "theoretical_loss": 3.8895628763094976,
+ "tokens_seen": 569311232
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686619859578737,
+ "loss": 1.47,
+ "theoretical_loss": 3.8895160723076527,
+ "tokens_seen": 569376768
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686459378134403,
+ "loss": 1.4073,
+ "theoretical_loss": 3.889469275200917,
+ "tokens_seen": 569442304
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686298896690071,
+ "loss": 1.4508,
+ "theoretical_loss": 3.889422484987481,
+ "tokens_seen": 569507840
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006686138415245737,
+ "loss": 1.3578,
+ "theoretical_loss": 3.889375701665537,
+ "tokens_seen": 569573376
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685977933801405,
+ "loss": 1.4829,
+ "theoretical_loss": 3.8893289252332766,
+ "tokens_seen": 569638912
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685817452357071,
+ "loss": 1.4044,
+ "theoretical_loss": 3.8892821556888935,
+ "tokens_seen": 569704448
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685656970912739,
+ "loss": 1.4279,
+ "theoretical_loss": 3.8892353930305807,
+ "tokens_seen": 569769984
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685496489468405,
+ "loss": 1.6804,
+ "theoretical_loss": 3.8891886372565327,
+ "tokens_seen": 569835520
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685336008024073,
+ "loss": 1.4291,
+ "theoretical_loss": 3.889141888364944,
+ "tokens_seen": 569901056
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000668517552657974,
+ "loss": 1.4354,
+ "theoretical_loss": 3.8890951463540104,
+ "tokens_seen": 569966592
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006685015045135407,
+ "loss": 1.4495,
+ "theoretical_loss": 3.8890484112219283,
+ "tokens_seen": 570032128
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684854563691073,
+ "loss": 1.4318,
+ "theoretical_loss": 3.889001682966895,
+ "tokens_seen": 570097664
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 195990,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5162267684936523,
+ "objective/train/theoretical_loss": 3.8889549615871077,
+ "objective/train/tokens_used": 590623200,
+ "theoretical_loss": 3.8889549615871077,
+ "tokens_seen": 570163200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684694082246741,
+ "loss": 1.4552,
+ "theoretical_loss": 3.8889549615871077,
+ "tokens_seen": 570163200
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684533600802407,
+ "loss": 1.4672,
+ "theoretical_loss": 3.8889082470807645,
+ "tokens_seen": 570228736
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684373119358075,
+ "loss": 1.4537,
+ "theoretical_loss": 3.888861539446066,
+ "tokens_seen": 570294272
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684212637913742,
+ "loss": 1.4429,
+ "theoretical_loss": 3.88881483868121,
+ "tokens_seen": 570359808
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006684052156469409,
+ "loss": 1.4702,
+ "theoretical_loss": 3.8887681447843976,
+ "tokens_seen": 570425344
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006683891675025076,
+ "loss": 1.4375,
+ "theoretical_loss": 3.88872145775383,
+ "tokens_seen": 570490880
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006683731193580743,
+ "loss": 1.4581,
+ "theoretical_loss": 3.8886747775877084,
+ "tokens_seen": 570556416
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000668357071213641,
+ "loss": 1.3596,
+ "theoretical_loss": 3.888628104284236,
+ "tokens_seen": 570621952
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006683410230692077,
+ "loss": 1.5164,
+ "theoretical_loss": 3.888581437841615,
+ "tokens_seen": 570687488
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006683249749247744,
+ "loss": 1.4334,
+ "theoretical_loss": 3.8885347782580504,
+ "tokens_seen": 570753024
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006683089267803411,
+ "loss": 1.4766,
+ "theoretical_loss": 3.8884881255317456,
+ "tokens_seen": 570818560
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682928786359078,
+ "loss": 1.4858,
+ "theoretical_loss": 3.888441479660906,
+ "tokens_seen": 570884096
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682768304914745,
+ "loss": 1.532,
+ "theoretical_loss": 3.8883948406437376,
+ "tokens_seen": 570949632
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682607823470412,
+ "loss": 1.4114,
+ "theoretical_loss": 3.8883482084784466,
+ "tokens_seen": 571015168
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682447342026079,
+ "loss": 1.4769,
+ "theoretical_loss": 3.8883015831632406,
+ "tokens_seen": 571080704
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682286860581746,
+ "loss": 1.5367,
+ "theoretical_loss": 3.8882549646963263,
+ "tokens_seen": 571146240
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006682126379137413,
+ "loss": 1.3905,
+ "theoretical_loss": 3.888208353075914,
+ "tokens_seen": 571211776
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000668196589769308,
+ "loss": 1.4432,
+ "theoretical_loss": 3.8881617483002113,
+ "tokens_seen": 571277312
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681805416248747,
+ "loss": 1.4893,
+ "theoretical_loss": 3.8881151503674283,
+ "tokens_seen": 571342848
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681644934804414,
+ "loss": 1.375,
+ "theoretical_loss": 3.888068559275776,
+ "tokens_seen": 571408384
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681484453360081,
+ "loss": 1.4334,
+ "theoretical_loss": 3.8880219750234657,
+ "tokens_seen": 571473920
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681323971915748,
+ "loss": 1.5079,
+ "theoretical_loss": 3.8879753976087086,
+ "tokens_seen": 571539456
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681163490471415,
+ "loss": 1.4816,
+ "theoretical_loss": 3.8879288270297176,
+ "tokens_seen": 571604992
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006681003009027082,
+ "loss": 1.4003,
+ "theoretical_loss": 3.8878822632847063,
+ "tokens_seen": 571670528
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006680842527582749,
+ "loss": 1.5134,
+ "theoretical_loss": 3.887835706371888,
+ "tokens_seen": 571736064
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006680682046138416,
+ "loss": 1.4662,
+ "theoretical_loss": 3.887789156289477,
+ "tokens_seen": 571801600
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006680521564694082,
+ "loss": 1.4893,
+ "theoretical_loss": 3.887742613035689,
+ "tokens_seen": 571867136
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000668036108324975,
+ "loss": 1.5036,
+ "theoretical_loss": 3.8876960766087407,
+ "tokens_seen": 571932672
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006680200601805416,
+ "loss": 1.4776,
+ "theoretical_loss": 3.8876495470068475,
+ "tokens_seen": 571998208
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006680040120361084,
+ "loss": 1.4827,
+ "theoretical_loss": 3.8876030242282265,
+ "tokens_seen": 572063744
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667987963891675,
+ "loss": 1.3667,
+ "theoretical_loss": 3.8875565082710963,
+ "tokens_seen": 572129280
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006679719157472418,
+ "loss": 1.436,
+ "theoretical_loss": 3.887509999133675,
+ "tokens_seen": 572194816
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006679558676028084,
+ "loss": 1.4718,
+ "theoretical_loss": 3.887463496814182,
+ "tokens_seen": 572260352
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006679398194583752,
+ "loss": 1.5012,
+ "theoretical_loss": 3.887417001310838,
+ "tokens_seen": 572325888
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006679237713139418,
+ "loss": 1.5284,
+ "theoretical_loss": 3.887370512621862,
+ "tokens_seen": 572391424
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006679077231695085,
+ "loss": 1.3893,
+ "theoretical_loss": 3.887324030745477,
+ "tokens_seen": 572456960
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006678916750250752,
+ "loss": 1.4374,
+ "theoretical_loss": 3.887277555679903,
+ "tokens_seen": 572522496
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006678756268806419,
+ "loss": 1.3893,
+ "theoretical_loss": 3.8872310874233644,
+ "tokens_seen": 572588032
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006678595787362086,
+ "loss": 1.4841,
+ "theoretical_loss": 3.887184625974083,
+ "tokens_seen": 572653568
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006678435305917754,
+ "loss": 1.4397,
+ "theoretical_loss": 3.8871381713302835,
+ "tokens_seen": 572719104
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667827482447342,
+ "loss": 1.4047,
+ "theoretical_loss": 3.8870917234901907,
+ "tokens_seen": 572784640
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006678114343029088,
+ "loss": 1.4294,
+ "theoretical_loss": 3.8870452824520294,
+ "tokens_seen": 572850176
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006677953861584754,
+ "loss": 1.6088,
+ "theoretical_loss": 3.8869988482140254,
+ "tokens_seen": 572915712
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006677793380140422,
+ "loss": 1.5765,
+ "theoretical_loss": 3.886952420774406,
+ "tokens_seen": 572981248
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006677632898696088,
+ "loss": 1.4694,
+ "theoretical_loss": 3.8869060001313973,
+ "tokens_seen": 573046784
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006677472417251756,
+ "loss": 1.4458,
+ "theoretical_loss": 3.8868595862832285,
+ "tokens_seen": 573112320
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006677311935807422,
+ "loss": 1.4668,
+ "theoretical_loss": 3.886813179228127,
+ "tokens_seen": 573177856
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667715145436309,
+ "loss": 1.3994,
+ "theoretical_loss": 3.886766778964323,
+ "tokens_seen": 573243392
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676990972918756,
+ "loss": 1.4859,
+ "theoretical_loss": 3.8867203854900456,
+ "tokens_seen": 573308928
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676830491474424,
+ "loss": 1.3269,
+ "theoretical_loss": 3.886673998803526,
+ "tokens_seen": 573374464
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 197322,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.453871488571167,
+ "objective/train/theoretical_loss": 3.886627618902995,
+ "objective/train/tokens_used": 593900000,
+ "theoretical_loss": 3.886627618902995,
+ "tokens_seen": 573440000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667667001003009,
+ "loss": 1.5145,
+ "theoretical_loss": 3.886627618902995,
+ "tokens_seen": 573440000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676509528585758,
+ "loss": 1.4646,
+ "theoretical_loss": 3.886581245786685,
+ "tokens_seen": 573505536
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676349047141424,
+ "loss": 1.252,
+ "theoretical_loss": 3.8865348794528276,
+ "tokens_seen": 573571072
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676188565697092,
+ "loss": 1.4531,
+ "theoretical_loss": 3.8864885198996575,
+ "tokens_seen": 573636608
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006676028084252758,
+ "loss": 1.4677,
+ "theoretical_loss": 3.8864421671254066,
+ "tokens_seen": 573702144
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006675867602808426,
+ "loss": 1.4749,
+ "theoretical_loss": 3.8863958211283114,
+ "tokens_seen": 573767680
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006675707121364092,
+ "loss": 1.471,
+ "theoretical_loss": 3.8863494819066062,
+ "tokens_seen": 573833216
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667554663991976,
+ "loss": 1.5011,
+ "theoretical_loss": 3.8863031494585267,
+ "tokens_seen": 573898752
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006675386158475426,
+ "loss": 1.5202,
+ "theoretical_loss": 3.8862568237823103,
+ "tokens_seen": 573964288
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006675225677031094,
+ "loss": 1.4638,
+ "theoretical_loss": 3.8862105048761926,
+ "tokens_seen": 574029824
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667506519558676,
+ "loss": 1.5591,
+ "theoretical_loss": 3.8861641927384127,
+ "tokens_seen": 574095360
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674904714142428,
+ "loss": 1.3434,
+ "theoretical_loss": 3.8861178873672095,
+ "tokens_seen": 574160896
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674744232698094,
+ "loss": 1.4476,
+ "theoretical_loss": 3.886071588760821,
+ "tokens_seen": 574226432
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674583751253762,
+ "loss": 1.4522,
+ "theoretical_loss": 3.886025296917487,
+ "tokens_seen": 574291968
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674423269809428,
+ "loss": 1.4525,
+ "theoretical_loss": 3.8859790118354494,
+ "tokens_seen": 574357504
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674262788365096,
+ "loss": 1.3981,
+ "theoretical_loss": 3.8859327335129477,
+ "tokens_seen": 574423040
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006674102306920762,
+ "loss": 1.4895,
+ "theoretical_loss": 3.8858864619482247,
+ "tokens_seen": 574488576
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667394182547643,
+ "loss": 1.4367,
+ "theoretical_loss": 3.8858401971395224,
+ "tokens_seen": 574554112
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006673781344032096,
+ "loss": 1.4269,
+ "theoretical_loss": 3.885793939085085,
+ "tokens_seen": 574619648
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006673620862587764,
+ "loss": 1.4373,
+ "theoretical_loss": 3.8857476877831543,
+ "tokens_seen": 574685184
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667346038114343,
+ "loss": 1.4741,
+ "theoretical_loss": 3.8857014432319765,
+ "tokens_seen": 574750720
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006673299899699098,
+ "loss": 1.5079,
+ "theoretical_loss": 3.8856552054297957,
+ "tokens_seen": 574816256
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006673139418254764,
+ "loss": 1.5025,
+ "theoretical_loss": 3.8856089743748576,
+ "tokens_seen": 574881792
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672978936810432,
+ "loss": 1.3655,
+ "theoretical_loss": 3.885562750065409,
+ "tokens_seen": 574947328
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672818455366098,
+ "loss": 1.6031,
+ "theoretical_loss": 3.8855165324996976,
+ "tokens_seen": 575012864
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672657973921766,
+ "loss": 1.432,
+ "theoretical_loss": 3.88547032167597,
+ "tokens_seen": 575078400
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672497492477432,
+ "loss": 1.5351,
+ "theoretical_loss": 3.885424117592475,
+ "tokens_seen": 575143936
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.00066723370110331,
+ "loss": 1.4087,
+ "theoretical_loss": 3.885377920247461,
+ "tokens_seen": 575209472
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672176529588766,
+ "loss": 1.5188,
+ "theoretical_loss": 3.885331729639179,
+ "tokens_seen": 575275008
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006672016048144434,
+ "loss": 1.5046,
+ "theoretical_loss": 3.8852855457658784,
+ "tokens_seen": 575340544
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.00066718555667001,
+ "loss": 1.4645,
+ "theoretical_loss": 3.88523936862581,
+ "tokens_seen": 575406080
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006671695085255768,
+ "loss": 1.4773,
+ "theoretical_loss": 3.885193198217226,
+ "tokens_seen": 575471616
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006671534603811434,
+ "loss": 1.5131,
+ "theoretical_loss": 3.8851470345383783,
+ "tokens_seen": 575537152
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006671374122367102,
+ "loss": 1.4086,
+ "theoretical_loss": 3.8851008775875204,
+ "tokens_seen": 575602688
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006671213640922768,
+ "loss": 1.4833,
+ "theoretical_loss": 3.8850547273629052,
+ "tokens_seen": 575668224
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006671053159478436,
+ "loss": 1.3597,
+ "theoretical_loss": 3.8850085838627875,
+ "tokens_seen": 575733760
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006670892678034102,
+ "loss": 1.4577,
+ "theoretical_loss": 3.8849624470854223,
+ "tokens_seen": 575799296
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667073219658977,
+ "loss": 1.3629,
+ "theoretical_loss": 3.8849163170290644,
+ "tokens_seen": 575864832
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006670571715145436,
+ "loss": 1.3028,
+ "theoretical_loss": 3.8848701936919703,
+ "tokens_seen": 575930368
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006670411233701104,
+ "loss": 1.54,
+ "theoretical_loss": 3.8848240770723974,
+ "tokens_seen": 575995904
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000667025075225677,
+ "loss": 1.4641,
+ "theoretical_loss": 3.8847779671686022,
+ "tokens_seen": 576061440
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006670090270812438,
+ "loss": 1.403,
+ "theoretical_loss": 3.8847318639788435,
+ "tokens_seen": 576126976
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006669929789368105,
+ "loss": 1.4338,
+ "theoretical_loss": 3.8846857675013804,
+ "tokens_seen": 576192512
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006669769307923772,
+ "loss": 1.5129,
+ "theoretical_loss": 3.884639677734471,
+ "tokens_seen": 576258048
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006669608826479439,
+ "loss": 1.3515,
+ "theoretical_loss": 3.884593594676377,
+ "tokens_seen": 576323584
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006669448345035106,
+ "loss": 1.3883,
+ "theoretical_loss": 3.884547518325359,
+ "tokens_seen": 576389120
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006669287863590773,
+ "loss": 1.4969,
+ "theoretical_loss": 3.8845014486796776,
+ "tokens_seen": 576454656
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000666912738214644,
+ "loss": 1.3977,
+ "theoretical_loss": 3.8844553857375947,
+ "tokens_seen": 576520192
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668966900702107,
+ "loss": 1.3918,
+ "theoretical_loss": 3.884409329497374,
+ "tokens_seen": 576585728
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668806419257774,
+ "loss": 1.3791,
+ "theoretical_loss": 3.884363279957278,
+ "tokens_seen": 576651264
+ },
+ {
+ "epoch": 0.17,
+ "objective/train/docs_used": 198260,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4905140399932861,
+ "objective/train/theoretical_loss": 3.884317237115571,
+ "objective/train/tokens_used": 597176800,
+ "theoretical_loss": 3.884317237115571,
+ "tokens_seen": 576716800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668645937813441,
+ "loss": 1.4027,
+ "theoretical_loss": 3.884317237115571,
+ "tokens_seen": 576716800
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668485456369108,
+ "loss": 1.4067,
+ "theoretical_loss": 3.884271200970517,
+ "tokens_seen": 576782336
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668324974924775,
+ "loss": 1.5032,
+ "theoretical_loss": 3.8842251715203826,
+ "tokens_seen": 576847872
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668164493480442,
+ "loss": 1.4525,
+ "theoretical_loss": 3.8841791487634327,
+ "tokens_seen": 576913408
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006668004012036109,
+ "loss": 1.5495,
+ "theoretical_loss": 3.8841331326979347,
+ "tokens_seen": 576978944
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006667843530591776,
+ "loss": 1.4502,
+ "theoretical_loss": 3.8840871233221548,
+ "tokens_seen": 577044480
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006667683049147443,
+ "loss": 1.49,
+ "theoretical_loss": 3.8840411206343615,
+ "tokens_seen": 577110016
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.000666752256770311,
+ "loss": 1.4969,
+ "theoretical_loss": 3.883995124632823,
+ "tokens_seen": 577175552
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006667362086258777,
+ "loss": 1.495,
+ "theoretical_loss": 3.8839491353158087,
+ "tokens_seen": 577241088
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006667201604814444,
+ "loss": 1.5002,
+ "theoretical_loss": 3.8839031526815884,
+ "tokens_seen": 577306624
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006667041123370111,
+ "loss": 1.4853,
+ "theoretical_loss": 3.8838571767284327,
+ "tokens_seen": 577372160
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 0.0006666880641925778,
+ "loss": 1.4603,
+ "theoretical_loss": 3.883811207454612,
+ "tokens_seen": 577437696
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006666720160481445,
+ "loss": 1.3529,
+ "theoretical_loss": 3.883765244858399,
+ "tokens_seen": 577503232
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006666559679037111,
+ "loss": 1.4831,
+ "theoretical_loss": 3.8837192889380647,
+ "tokens_seen": 577568768
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006666399197592779,
+ "loss": 1.4656,
+ "theoretical_loss": 3.8836733396918834,
+ "tokens_seen": 577634304
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006666238716148445,
+ "loss": 1.4363,
+ "theoretical_loss": 3.8836273971181283,
+ "tokens_seen": 577699840
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006666078234704113,
+ "loss": 1.44,
+ "theoretical_loss": 3.883581461215074,
+ "tokens_seen": 577765376
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665917753259779,
+ "loss": 1.4905,
+ "theoretical_loss": 3.8835355319809954,
+ "tokens_seen": 577830912
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665757271815447,
+ "loss": 1.5494,
+ "theoretical_loss": 3.883489609414167,
+ "tokens_seen": 577896448
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665596790371113,
+ "loss": 1.4153,
+ "theoretical_loss": 3.8834436935128664,
+ "tokens_seen": 577961984
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665436308926781,
+ "loss": 1.3772,
+ "theoretical_loss": 3.8833977842753704,
+ "tokens_seen": 578027520
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665275827482447,
+ "loss": 1.4353,
+ "theoretical_loss": 3.883351881699956,
+ "tokens_seen": 578093056
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006665115346038115,
+ "loss": 1.5437,
+ "theoretical_loss": 3.883305985784901,
+ "tokens_seen": 578158592
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664954864593781,
+ "loss": 1.4388,
+ "theoretical_loss": 3.8832600965284847,
+ "tokens_seen": 578224128
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664794383149449,
+ "loss": 1.4035,
+ "theoretical_loss": 3.8832142139289862,
+ "tokens_seen": 578289664
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664633901705115,
+ "loss": 1.614,
+ "theoretical_loss": 3.883168337984686,
+ "tokens_seen": 578355200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664473420260783,
+ "loss": 1.4377,
+ "theoretical_loss": 3.883122468693865,
+ "tokens_seen": 578420736
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664312938816449,
+ "loss": 1.4435,
+ "theoretical_loss": 3.8830766060548036,
+ "tokens_seen": 578486272
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006664152457372117,
+ "loss": 1.4986,
+ "theoretical_loss": 3.8830307500657844,
+ "tokens_seen": 578551808
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663991975927783,
+ "loss": 1.37,
+ "theoretical_loss": 3.8829849007250905,
+ "tokens_seen": 578617344
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663831494483451,
+ "loss": 1.489,
+ "theoretical_loss": 3.8829390580310044,
+ "tokens_seen": 578682880
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663671013039117,
+ "loss": 1.4244,
+ "theoretical_loss": 3.88289322198181,
+ "tokens_seen": 578748416
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663510531594785,
+ "loss": 1.4809,
+ "theoretical_loss": 3.8828473925757923,
+ "tokens_seen": 578813952
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663350050150451,
+ "loss": 1.5996,
+ "theoretical_loss": 3.882801569811236,
+ "tokens_seen": 578879488
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663189568706119,
+ "loss": 1.4475,
+ "theoretical_loss": 3.8827557536864274,
+ "tokens_seen": 578945024
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006663029087261785,
+ "loss": 1.5018,
+ "theoretical_loss": 3.882709944199653,
+ "tokens_seen": 579010560
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662868605817453,
+ "loss": 1.5429,
+ "theoretical_loss": 3.8826641413491987,
+ "tokens_seen": 579076096
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662708124373119,
+ "loss": 1.4644,
+ "theoretical_loss": 3.882618345133354,
+ "tokens_seen": 579141632
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662547642928787,
+ "loss": 1.4266,
+ "theoretical_loss": 3.882572555550406,
+ "tokens_seen": 579207168
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662387161484453,
+ "loss": 1.5529,
+ "theoretical_loss": 3.882526772598644,
+ "tokens_seen": 579272704
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662226680040121,
+ "loss": 1.4612,
+ "theoretical_loss": 3.8824809962763576,
+ "tokens_seen": 579338240
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006662066198595787,
+ "loss": 1.4295,
+ "theoretical_loss": 3.882435226581837,
+ "tokens_seen": 579403776
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661905717151455,
+ "loss": 1.3675,
+ "theoretical_loss": 3.8823894635133738,
+ "tokens_seen": 579469312
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661745235707121,
+ "loss": 1.4033,
+ "theoretical_loss": 3.8823437070692584,
+ "tokens_seen": 579534848
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661584754262789,
+ "loss": 1.3765,
+ "theoretical_loss": 3.882297957247784,
+ "tokens_seen": 579600384
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661424272818455,
+ "loss": 1.4589,
+ "theoretical_loss": 3.8822522140472424,
+ "tokens_seen": 579665920
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661263791374123,
+ "loss": 1.4647,
+ "theoretical_loss": 3.882206477465928,
+ "tokens_seen": 579731456
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006661103309929789,
+ "loss": 1.3975,
+ "theoretical_loss": 3.882160747502134,
+ "tokens_seen": 579796992
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660942828485457,
+ "loss": 1.5154,
+ "theoretical_loss": 3.8821150241541558,
+ "tokens_seen": 579862528
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660782347041123,
+ "loss": 1.5099,
+ "theoretical_loss": 3.8820693074202888,
+ "tokens_seen": 579928064
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 199439,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4916222095489502,
+ "objective/train/theoretical_loss": 3.8820235972988275,
+ "objective/train/tokens_used": 600453600,
+ "theoretical_loss": 3.8820235972988275,
+ "tokens_seen": 579993600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660621865596791,
+ "loss": 1.4725,
+ "theoretical_loss": 3.8820235972988275,
+ "tokens_seen": 579993600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660461384152457,
+ "loss": 1.5312,
+ "theoretical_loss": 3.8819778937880702,
+ "tokens_seen": 580059136
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660300902708125,
+ "loss": 1.481,
+ "theoretical_loss": 3.8819321968863134,
+ "tokens_seen": 580124672
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006660140421263791,
+ "loss": 1.4392,
+ "theoretical_loss": 3.8818865065918553,
+ "tokens_seen": 580190208
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659979939819459,
+ "loss": 1.282,
+ "theoretical_loss": 3.8818408229029937,
+ "tokens_seen": 580255744
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659819458375125,
+ "loss": 1.43,
+ "theoretical_loss": 3.881795145818028,
+ "tokens_seen": 580321280
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659658976930793,
+ "loss": 1.5045,
+ "theoretical_loss": 3.8817494753352584,
+ "tokens_seen": 580386816
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659498495486459,
+ "loss": 1.3841,
+ "theoretical_loss": 3.881703811452985,
+ "tokens_seen": 580452352
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659338014042127,
+ "loss": 1.4778,
+ "theoretical_loss": 3.8816581541695085,
+ "tokens_seen": 580517888
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659177532597793,
+ "loss": 1.4307,
+ "theoretical_loss": 3.8816125034831312,
+ "tokens_seen": 580583424
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006659017051153461,
+ "loss": 1.3677,
+ "theoretical_loss": 3.881566859392154,
+ "tokens_seen": 580648960
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658856569709127,
+ "loss": 1.4426,
+ "theoretical_loss": 3.8815212218948814,
+ "tokens_seen": 580714496
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658696088264795,
+ "loss": 1.3904,
+ "theoretical_loss": 3.8814755909896164,
+ "tokens_seen": 580780032
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658535606820461,
+ "loss": 1.4756,
+ "theoretical_loss": 3.881429966674663,
+ "tokens_seen": 580845568
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658375125376129,
+ "loss": 1.4131,
+ "theoretical_loss": 3.881384348948325,
+ "tokens_seen": 580911104
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658214643931795,
+ "loss": 1.3903,
+ "theoretical_loss": 3.8813387378089095,
+ "tokens_seen": 580976640
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006658054162487463,
+ "loss": 1.3678,
+ "theoretical_loss": 3.8812931332547214,
+ "tokens_seen": 581042176
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657893681043129,
+ "loss": 1.4463,
+ "theoretical_loss": 3.881247535284068,
+ "tokens_seen": 581107712
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657733199598797,
+ "loss": 1.3848,
+ "theoretical_loss": 3.881201943895256,
+ "tokens_seen": 581173248
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657572718154463,
+ "loss": 1.4144,
+ "theoretical_loss": 3.8811563590865936,
+ "tokens_seen": 581238784
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657412236710131,
+ "loss": 1.304,
+ "theoretical_loss": 3.8811107808563894,
+ "tokens_seen": 581304320
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657251755265797,
+ "loss": 1.3599,
+ "theoretical_loss": 3.8810652092029523,
+ "tokens_seen": 581369856
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006657091273821465,
+ "loss": 1.4867,
+ "theoretical_loss": 3.881019644124593,
+ "tokens_seen": 581435392
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006656930792377131,
+ "loss": 1.3985,
+ "theoretical_loss": 3.8809740856196204,
+ "tokens_seen": 581500928
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006656770310932799,
+ "loss": 1.4878,
+ "theoretical_loss": 3.880928533686346,
+ "tokens_seen": 581566464
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006656609829488466,
+ "loss": 1.5318,
+ "theoretical_loss": 3.8808829883230826,
+ "tokens_seen": 581632000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006656449348044133,
+ "loss": 1.4466,
+ "theoretical_loss": 3.880837449528141,
+ "tokens_seen": 581697536
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00066562888665998,
+ "loss": 1.4171,
+ "theoretical_loss": 3.8807919172998346,
+ "tokens_seen": 581763072
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006656128385155467,
+ "loss": 1.3872,
+ "theoretical_loss": 3.8807463916364773,
+ "tokens_seen": 581828608
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655967903711134,
+ "loss": 1.508,
+ "theoretical_loss": 3.8807008725363836,
+ "tokens_seen": 581894144
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655807422266801,
+ "loss": 1.506,
+ "theoretical_loss": 3.880655359997867,
+ "tokens_seen": 581959680
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655646940822468,
+ "loss": 1.2942,
+ "theoretical_loss": 3.8806098540192435,
+ "tokens_seen": 582025216
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655486459378135,
+ "loss": 1.5053,
+ "theoretical_loss": 3.880564354598829,
+ "tokens_seen": 582090752
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655325977933802,
+ "loss": 1.4198,
+ "theoretical_loss": 3.8805188617349406,
+ "tokens_seen": 582156288
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655165496489469,
+ "loss": 1.5112,
+ "theoretical_loss": 3.880473375425895,
+ "tokens_seen": 582221824
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006655005015045136,
+ "loss": 1.468,
+ "theoretical_loss": 3.880427895670011,
+ "tokens_seen": 582287360
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006654844533600803,
+ "loss": 1.4688,
+ "theoretical_loss": 3.8803824224656056,
+ "tokens_seen": 582352896
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000665468405215647,
+ "loss": 1.3934,
+ "theoretical_loss": 3.880336955810999,
+ "tokens_seen": 582418432
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006654523570712137,
+ "loss": 1.5792,
+ "theoretical_loss": 3.880291495704511,
+ "tokens_seen": 582483968
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006654363089267805,
+ "loss": 1.3663,
+ "theoretical_loss": 3.880246042144461,
+ "tokens_seen": 582549504
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006654202607823471,
+ "loss": 1.4836,
+ "theoretical_loss": 3.880200595129171,
+ "tokens_seen": 582615040
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006654042126379139,
+ "loss": 1.4792,
+ "theoretical_loss": 3.880155154656962,
+ "tokens_seen": 582680576
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006653881644934805,
+ "loss": 1.3691,
+ "theoretical_loss": 3.880109720726157,
+ "tokens_seen": 582746112
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006653721163490472,
+ "loss": 1.4147,
+ "theoretical_loss": 3.880064293335078,
+ "tokens_seen": 582811648
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006653560682046139,
+ "loss": 1.422,
+ "theoretical_loss": 3.880018872482049,
+ "tokens_seen": 582877184
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006653400200601806,
+ "loss": 1.4849,
+ "theoretical_loss": 3.8799734581653933,
+ "tokens_seen": 582942720
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006653239719157473,
+ "loss": 1.5962,
+ "theoretical_loss": 3.8799280503834366,
+ "tokens_seen": 583008256
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000665307923771314,
+ "loss": 1.4192,
+ "theoretical_loss": 3.879882649134503,
+ "tokens_seen": 583073792
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006652918756268807,
+ "loss": 1.4484,
+ "theoretical_loss": 3.8798372544169197,
+ "tokens_seen": 583139328
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006652758274824474,
+ "loss": 1.4309,
+ "theoretical_loss": 3.879791866229013,
+ "tokens_seen": 583204864
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 200344,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4507158994674683,
+ "objective/train/theoretical_loss": 3.879746484569109,
+ "objective/train/tokens_used": 603730400,
+ "theoretical_loss": 3.879746484569109,
+ "tokens_seen": 583270400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000665259779338014,
+ "loss": 1.5196,
+ "theoretical_loss": 3.879746484569109,
+ "tokens_seen": 583270400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006652437311935808,
+ "loss": 1.4805,
+ "theoretical_loss": 3.879701109435537,
+ "tokens_seen": 583335936
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006652276830491474,
+ "loss": 1.4719,
+ "theoretical_loss": 3.8796557408266237,
+ "tokens_seen": 583401472
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006652116349047142,
+ "loss": 1.5439,
+ "theoretical_loss": 3.8796103787406997,
+ "tokens_seen": 583467008
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006651955867602808,
+ "loss": 1.4825,
+ "theoretical_loss": 3.8795650231760934,
+ "tokens_seen": 583532544
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006651795386158476,
+ "loss": 1.4274,
+ "theoretical_loss": 3.879519674131136,
+ "tokens_seen": 583598080
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006651634904714142,
+ "loss": 1.3955,
+ "theoretical_loss": 3.8794743316041584,
+ "tokens_seen": 583663616
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000665147442326981,
+ "loss": 1.4537,
+ "theoretical_loss": 3.879428995593491,
+ "tokens_seen": 583729152
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006651313941825476,
+ "loss": 1.468,
+ "theoretical_loss": 3.8793836660974668,
+ "tokens_seen": 583794688
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006651153460381144,
+ "loss": 1.552,
+ "theoretical_loss": 3.8793383431144175,
+ "tokens_seen": 583860224
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000665099297893681,
+ "loss": 1.4288,
+ "theoretical_loss": 3.879293026642678,
+ "tokens_seen": 583925760
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650832497492478,
+ "loss": 1.3514,
+ "theoretical_loss": 3.879247716680581,
+ "tokens_seen": 583991296
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650672016048144,
+ "loss": 1.4898,
+ "theoretical_loss": 3.8792024132264613,
+ "tokens_seen": 584056832
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650511534603812,
+ "loss": 1.5033,
+ "theoretical_loss": 3.8791571162786544,
+ "tokens_seen": 584122368
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650351053159478,
+ "loss": 1.3391,
+ "theoretical_loss": 3.8791118258354955,
+ "tokens_seen": 584187904
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650190571715146,
+ "loss": 1.3986,
+ "theoretical_loss": 3.8790665418953214,
+ "tokens_seen": 584253440
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006650030090270812,
+ "loss": 1.4383,
+ "theoretical_loss": 3.879021264456469,
+ "tokens_seen": 584318976
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664986960882648,
+ "loss": 1.5654,
+ "theoretical_loss": 3.878975993517276,
+ "tokens_seen": 584384512
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006649709127382146,
+ "loss": 1.5445,
+ "theoretical_loss": 3.8789307290760804,
+ "tokens_seen": 584450048
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006649548645937814,
+ "loss": 1.4391,
+ "theoretical_loss": 3.8788854711312206,
+ "tokens_seen": 584515584
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664938816449348,
+ "loss": 1.4836,
+ "theoretical_loss": 3.878840219681037,
+ "tokens_seen": 584581120
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006649227683049148,
+ "loss": 1.5231,
+ "theoretical_loss": 3.8787949747238697,
+ "tokens_seen": 584646656
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006649067201604814,
+ "loss": 1.5907,
+ "theoretical_loss": 3.878749736258058,
+ "tokens_seen": 584712192
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006648906720160482,
+ "loss": 1.4663,
+ "theoretical_loss": 3.8787045042819446,
+ "tokens_seen": 584777728
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006648746238716148,
+ "loss": 1.4545,
+ "theoretical_loss": 3.8786592787938705,
+ "tokens_seen": 584843264
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006648585757271816,
+ "loss": 1.4241,
+ "theoretical_loss": 3.878614059792178,
+ "tokens_seen": 584908800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006648425275827482,
+ "loss": 1.5496,
+ "theoretical_loss": 3.878568847275212,
+ "tokens_seen": 584974336
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664826479438315,
+ "loss": 1.379,
+ "theoretical_loss": 3.8785236412413138,
+ "tokens_seen": 585039872
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006648104312938816,
+ "loss": 1.5821,
+ "theoretical_loss": 3.878478441688829,
+ "tokens_seen": 585105408
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006647943831494484,
+ "loss": 1.5253,
+ "theoretical_loss": 3.878433248616102,
+ "tokens_seen": 585170944
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664778335005015,
+ "loss": 1.4069,
+ "theoretical_loss": 3.878388062021479,
+ "tokens_seen": 585236480
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006647622868605818,
+ "loss": 1.3906,
+ "theoretical_loss": 3.8783428819033055,
+ "tokens_seen": 585302016
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006647462387161484,
+ "loss": 1.5373,
+ "theoretical_loss": 3.8782977082599284,
+ "tokens_seen": 585367552
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006647301905717152,
+ "loss": 1.4043,
+ "theoretical_loss": 3.8782525410896955,
+ "tokens_seen": 585433088
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006647141424272818,
+ "loss": 1.3978,
+ "theoretical_loss": 3.8782073803909545,
+ "tokens_seen": 585498624
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006646980942828486,
+ "loss": 1.443,
+ "theoretical_loss": 3.878162226162053,
+ "tokens_seen": 585564160
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006646820461384152,
+ "loss": 1.4232,
+ "theoretical_loss": 3.878117078401342,
+ "tokens_seen": 585629696
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664665997993982,
+ "loss": 1.4794,
+ "theoretical_loss": 3.87807193710717,
+ "tokens_seen": 585695232
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006646499498495486,
+ "loss": 1.4249,
+ "theoretical_loss": 3.8780268022778874,
+ "tokens_seen": 585760768
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006646339017051154,
+ "loss": 1.4592,
+ "theoretical_loss": 3.877981673911846,
+ "tokens_seen": 585826304
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664617853560682,
+ "loss": 1.4521,
+ "theoretical_loss": 3.8779365520073963,
+ "tokens_seen": 585891840
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006646018054162488,
+ "loss": 1.5234,
+ "theoretical_loss": 3.8778914365628916,
+ "tokens_seen": 585957376
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006645857572718154,
+ "loss": 1.5314,
+ "theoretical_loss": 3.877846327576684,
+ "tokens_seen": 586022912
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006645697091273822,
+ "loss": 1.4303,
+ "theoretical_loss": 3.8778012250471265,
+ "tokens_seen": 586088448
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006645536609829488,
+ "loss": 1.4924,
+ "theoretical_loss": 3.8777561289725746,
+ "tokens_seen": 586153984
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006645376128385156,
+ "loss": 1.3857,
+ "theoretical_loss": 3.8777110393513814,
+ "tokens_seen": 586219520
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006645215646940822,
+ "loss": 1.4557,
+ "theoretical_loss": 3.877665956181903,
+ "tokens_seen": 586285056
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664505516549649,
+ "loss": 1.4093,
+ "theoretical_loss": 3.8776208794624942,
+ "tokens_seen": 586350592
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006644894684052156,
+ "loss": 1.4833,
+ "theoretical_loss": 3.8775758091915127,
+ "tokens_seen": 586416128
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006644734202607824,
+ "loss": 1.6214,
+ "theoretical_loss": 3.8775307453673147,
+ "tokens_seen": 586481664
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 201447,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.557558298110962,
+ "objective/train/theoretical_loss": 3.8774856879882584,
+ "objective/train/tokens_used": 607007200,
+ "theoretical_loss": 3.8774856879882584,
+ "tokens_seen": 586547200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664457372116349,
+ "loss": 1.5159,
+ "theoretical_loss": 3.8774856879882584,
+ "tokens_seen": 586547200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006644413239719158,
+ "loss": 1.4372,
+ "theoretical_loss": 3.8774406370527013,
+ "tokens_seen": 586612736
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006644252758274824,
+ "loss": 1.5486,
+ "theoretical_loss": 3.8773955925590036,
+ "tokens_seen": 586678272
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006644092276830492,
+ "loss": 1.4192,
+ "theoretical_loss": 3.8773505545055227,
+ "tokens_seen": 586743808
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006643931795386158,
+ "loss": 1.4344,
+ "theoretical_loss": 3.87730552289062,
+ "tokens_seen": 586809344
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006643771313941826,
+ "loss": 1.5196,
+ "theoretical_loss": 3.8772604977126557,
+ "tokens_seen": 586874880
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006643610832497492,
+ "loss": 1.3887,
+ "theoretical_loss": 3.8772154789699913,
+ "tokens_seen": 586940416
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664345035105316,
+ "loss": 1.3559,
+ "theoretical_loss": 3.8771704666609885,
+ "tokens_seen": 587005952
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006643289869608826,
+ "loss": 1.3598,
+ "theoretical_loss": 3.8771254607840095,
+ "tokens_seen": 587071488
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006643129388164494,
+ "loss": 1.3921,
+ "theoretical_loss": 3.877080461337418,
+ "tokens_seen": 587137024
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664296890672016,
+ "loss": 1.4247,
+ "theoretical_loss": 3.877035468319577,
+ "tokens_seen": 587202560
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006642808425275828,
+ "loss": 1.4363,
+ "theoretical_loss": 3.8769904817288507,
+ "tokens_seen": 587268096
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006642647943831495,
+ "loss": 1.5142,
+ "theoretical_loss": 3.8769455015636045,
+ "tokens_seen": 587333632
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006642487462387162,
+ "loss": 1.4592,
+ "theoretical_loss": 3.8769005278222033,
+ "tokens_seen": 587399168
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664232698094283,
+ "loss": 1.5649,
+ "theoretical_loss": 3.876855560503013,
+ "tokens_seen": 587464704
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006642166499498496,
+ "loss": 1.435,
+ "theoretical_loss": 3.8768105996044007,
+ "tokens_seen": 587530240
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006642006018054164,
+ "loss": 1.3991,
+ "theoretical_loss": 3.876765645124734,
+ "tokens_seen": 587595776
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000664184553660983,
+ "loss": 1.4028,
+ "theoretical_loss": 3.876720697062379,
+ "tokens_seen": 587661312
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006641685055165498,
+ "loss": 1.4663,
+ "theoretical_loss": 3.876675755415706,
+ "tokens_seen": 587726848
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006641524573721164,
+ "loss": 1.4308,
+ "theoretical_loss": 3.8766308201830837,
+ "tokens_seen": 587792384
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006641364092276832,
+ "loss": 1.5021,
+ "theoretical_loss": 3.8765858913628803,
+ "tokens_seen": 587857920
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006641203610832498,
+ "loss": 1.5041,
+ "theoretical_loss": 3.8765409689534676,
+ "tokens_seen": 587923456
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006641043129388166,
+ "loss": 1.4928,
+ "theoretical_loss": 3.8764960529532155,
+ "tokens_seen": 587988992
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006640882647943832,
+ "loss": 1.4278,
+ "theoretical_loss": 3.876451143360496,
+ "tokens_seen": 588054528
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00066407221664995,
+ "loss": 1.534,
+ "theoretical_loss": 3.8764062401736803,
+ "tokens_seen": 588120064
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006640561685055166,
+ "loss": 1.4744,
+ "theoretical_loss": 3.8763613433911415,
+ "tokens_seen": 588185600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006640401203610834,
+ "loss": 1.4814,
+ "theoretical_loss": 3.8763164530112535,
+ "tokens_seen": 588251136
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00066402407221665,
+ "loss": 1.4645,
+ "theoretical_loss": 3.8762715690323883,
+ "tokens_seen": 588316672
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006640080240722168,
+ "loss": 1.4931,
+ "theoretical_loss": 3.876226691452922,
+ "tokens_seen": 588382208
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639919759277834,
+ "loss": 1.3968,
+ "theoretical_loss": 3.8761818202712286,
+ "tokens_seen": 588447744
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639759277833501,
+ "loss": 1.5262,
+ "theoretical_loss": 3.8761369554856837,
+ "tokens_seen": 588513280
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639598796389168,
+ "loss": 1.3893,
+ "theoretical_loss": 3.8760920970946637,
+ "tokens_seen": 588578816
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639438314944835,
+ "loss": 1.3683,
+ "theoretical_loss": 3.8760472450965455,
+ "tokens_seen": 588644352
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639277833500502,
+ "loss": 1.4467,
+ "theoretical_loss": 3.876002399489706,
+ "tokens_seen": 588709888
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006639117352056169,
+ "loss": 1.4814,
+ "theoretical_loss": 3.8759575602725236,
+ "tokens_seen": 588775424
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006638956870611836,
+ "loss": 1.4286,
+ "theoretical_loss": 3.8759127274433762,
+ "tokens_seen": 588840960
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006638796389167503,
+ "loss": 1.4914,
+ "theoretical_loss": 3.8758679010006434,
+ "tokens_seen": 588906496
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000663863590772317,
+ "loss": 1.38,
+ "theoretical_loss": 3.875823080942705,
+ "tokens_seen": 588972032
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006638475426278837,
+ "loss": 1.5693,
+ "theoretical_loss": 3.875778267267941,
+ "tokens_seen": 589037568
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006638314944834503,
+ "loss": 1.4688,
+ "theoretical_loss": 3.8757334599747324,
+ "tokens_seen": 589103104
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006638154463390171,
+ "loss": 1.3994,
+ "theoretical_loss": 3.8756886590614608,
+ "tokens_seen": 589168640
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637993981945837,
+ "loss": 1.4926,
+ "theoretical_loss": 3.8756438645265074,
+ "tokens_seen": 589234176
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637833500501505,
+ "loss": 1.4596,
+ "theoretical_loss": 3.8755990763682564,
+ "tokens_seen": 589299712
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637673019057171,
+ "loss": 1.4927,
+ "theoretical_loss": 3.87555429458509,
+ "tokens_seen": 589365248
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637512537612839,
+ "loss": 1.4579,
+ "theoretical_loss": 3.8755095191753917,
+ "tokens_seen": 589430784
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637352056168505,
+ "loss": 1.4657,
+ "theoretical_loss": 3.8754647501375468,
+ "tokens_seen": 589496320
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637191574724173,
+ "loss": 1.4161,
+ "theoretical_loss": 3.8754199874699404,
+ "tokens_seen": 589561856
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006637031093279839,
+ "loss": 1.5285,
+ "theoretical_loss": 3.875375231170957,
+ "tokens_seen": 589627392
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636870611835507,
+ "loss": 1.4242,
+ "theoretical_loss": 3.8753304812389837,
+ "tokens_seen": 589692928
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636710130391173,
+ "loss": 1.5375,
+ "theoretical_loss": 3.8752857376724075,
+ "tokens_seen": 589758464
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 202342,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.514643907546997,
+ "objective/train/theoretical_loss": 3.875241000469615,
+ "objective/train/tokens_used": 610284000,
+ "theoretical_loss": 3.875241000469615,
+ "tokens_seen": 589824000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636549648946841,
+ "loss": 1.5381,
+ "theoretical_loss": 3.875241000469615,
+ "tokens_seen": 589824000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636389167502507,
+ "loss": 1.4772,
+ "theoretical_loss": 3.8751962696289945,
+ "tokens_seen": 589889536
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636228686058175,
+ "loss": 1.4867,
+ "theoretical_loss": 3.8751515451489347,
+ "tokens_seen": 589955072
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006636068204613841,
+ "loss": 1.4871,
+ "theoretical_loss": 3.875106827027824,
+ "tokens_seen": 590020608
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635907723169509,
+ "loss": 1.4657,
+ "theoretical_loss": 3.875062115264053,
+ "tokens_seen": 590086144
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635747241725175,
+ "loss": 1.4551,
+ "theoretical_loss": 3.875017409856012,
+ "tokens_seen": 590151680
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635586760280843,
+ "loss": 1.4035,
+ "theoretical_loss": 3.8749727108020915,
+ "tokens_seen": 590217216
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635426278836509,
+ "loss": 1.4181,
+ "theoretical_loss": 3.8749280181006824,
+ "tokens_seen": 590282752
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635265797392177,
+ "loss": 1.5073,
+ "theoretical_loss": 3.874883331750178,
+ "tokens_seen": 590348288
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006635105315947843,
+ "loss": 1.537,
+ "theoretical_loss": 3.874838651748971,
+ "tokens_seen": 590413824
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634944834503511,
+ "loss": 1.5114,
+ "theoretical_loss": 3.874793978095453,
+ "tokens_seen": 590479360
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634784353059177,
+ "loss": 1.3964,
+ "theoretical_loss": 3.874749310788019,
+ "tokens_seen": 590544896
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634623871614845,
+ "loss": 1.4324,
+ "theoretical_loss": 3.8747046498250626,
+ "tokens_seen": 590610432
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634463390170511,
+ "loss": 1.474,
+ "theoretical_loss": 3.87465999520498,
+ "tokens_seen": 590675968
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634302908726179,
+ "loss": 1.5118,
+ "theoretical_loss": 3.8746153469261664,
+ "tokens_seen": 590741504
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006634142427281845,
+ "loss": 1.4831,
+ "theoretical_loss": 3.8745707049870175,
+ "tokens_seen": 590807040
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633981945837513,
+ "loss": 1.4809,
+ "theoretical_loss": 3.8745260693859302,
+ "tokens_seen": 590872576
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633821464393179,
+ "loss": 1.319,
+ "theoretical_loss": 3.8744814401213024,
+ "tokens_seen": 590938112
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633660982948847,
+ "loss": 1.4283,
+ "theoretical_loss": 3.8744368171915307,
+ "tokens_seen": 591003648
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633500501504513,
+ "loss": 1.4634,
+ "theoretical_loss": 3.8743922005950147,
+ "tokens_seen": 591069184
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633340020060181,
+ "loss": 1.4687,
+ "theoretical_loss": 3.8743475903301534,
+ "tokens_seen": 591134720
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633179538615847,
+ "loss": 1.451,
+ "theoretical_loss": 3.874302986395346,
+ "tokens_seen": 591200256
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006633019057171515,
+ "loss": 1.5493,
+ "theoretical_loss": 3.8742583887889928,
+ "tokens_seen": 591265792
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632858575727181,
+ "loss": 1.5102,
+ "theoretical_loss": 3.874213797509495,
+ "tokens_seen": 591331328
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632698094282849,
+ "loss": 1.41,
+ "theoretical_loss": 3.874169212555253,
+ "tokens_seen": 591396864
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632537612838515,
+ "loss": 1.3676,
+ "theoretical_loss": 3.87412463392467,
+ "tokens_seen": 591462400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632377131394183,
+ "loss": 1.4752,
+ "theoretical_loss": 3.8740800616161484,
+ "tokens_seen": 591527936
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632216649949849,
+ "loss": 1.4781,
+ "theoretical_loss": 3.8740354956280907,
+ "tokens_seen": 591593472
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006632056168505517,
+ "loss": 1.4841,
+ "theoretical_loss": 3.873990935958901,
+ "tokens_seen": 591659008
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631895687061183,
+ "loss": 1.5448,
+ "theoretical_loss": 3.8739463826069835,
+ "tokens_seen": 591724544
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631735205616851,
+ "loss": 1.3107,
+ "theoretical_loss": 3.8739018355707433,
+ "tokens_seen": 591790080
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631574724172517,
+ "loss": 1.4157,
+ "theoretical_loss": 3.8738572948485848,
+ "tokens_seen": 591855616
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631414242728185,
+ "loss": 1.4669,
+ "theoretical_loss": 3.873812760438916,
+ "tokens_seen": 591921152
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631253761283851,
+ "loss": 1.4281,
+ "theoretical_loss": 3.873768232340142,
+ "tokens_seen": 591986688
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006631093279839519,
+ "loss": 1.4568,
+ "theoretical_loss": 3.87372371055067,
+ "tokens_seen": 592052224
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006630932798395185,
+ "loss": 1.5104,
+ "theoretical_loss": 3.8736791950689087,
+ "tokens_seen": 592117760
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006630772316950853,
+ "loss": 1.4712,
+ "theoretical_loss": 3.8736346858932658,
+ "tokens_seen": 592183296
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000663061183550652,
+ "loss": 1.4616,
+ "theoretical_loss": 3.8735901830221495,
+ "tokens_seen": 592248832
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006630451354062187,
+ "loss": 1.4809,
+ "theoretical_loss": 3.873545686453971,
+ "tokens_seen": 592314368
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006630290872617855,
+ "loss": 1.4827,
+ "theoretical_loss": 3.8735011961871395,
+ "tokens_seen": 592379904
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006630130391173521,
+ "loss": 1.5461,
+ "theoretical_loss": 3.873456712220066,
+ "tokens_seen": 592445440
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629969909729189,
+ "loss": 1.4825,
+ "theoretical_loss": 3.873412234551161,
+ "tokens_seen": 592510976
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629809428284855,
+ "loss": 1.4295,
+ "theoretical_loss": 3.8733677631788366,
+ "tokens_seen": 592576512
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629648946840523,
+ "loss": 1.491,
+ "theoretical_loss": 3.8733232981015058,
+ "tokens_seen": 592642048
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629488465396189,
+ "loss": 1.5089,
+ "theoretical_loss": 3.8732788393175808,
+ "tokens_seen": 592707584
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629327983951857,
+ "loss": 1.5842,
+ "theoretical_loss": 3.8732343868254753,
+ "tokens_seen": 592773120
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629167502507523,
+ "loss": 1.384,
+ "theoretical_loss": 3.873189940623604,
+ "tokens_seen": 592838656
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006629007021063191,
+ "loss": 1.4859,
+ "theoretical_loss": 3.8731455007103808,
+ "tokens_seen": 592904192
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628846539618857,
+ "loss": 1.4207,
+ "theoretical_loss": 3.873101067084221,
+ "tokens_seen": 592969728
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628686058174525,
+ "loss": 1.2965,
+ "theoretical_loss": 3.873056639743541,
+ "tokens_seen": 593035264
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 203552,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4285786151885986,
+ "objective/train/theoretical_loss": 3.8730122186867573,
+ "objective/train/tokens_used": 613560800,
+ "theoretical_loss": 3.8730122186867573,
+ "tokens_seen": 593100800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628525576730191,
+ "loss": 1.394,
+ "theoretical_loss": 3.8730122186867573,
+ "tokens_seen": 593100800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628365095285859,
+ "loss": 1.6074,
+ "theoretical_loss": 3.872967803912286,
+ "tokens_seen": 593166336
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628204613841525,
+ "loss": 1.3419,
+ "theoretical_loss": 3.872923395418546,
+ "tokens_seen": 593231872
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006628044132397193,
+ "loss": 1.4308,
+ "theoretical_loss": 3.872878993203954,
+ "tokens_seen": 593297408
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627883650952859,
+ "loss": 1.4721,
+ "theoretical_loss": 3.8728345972669294,
+ "tokens_seen": 593362944
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627723169508527,
+ "loss": 1.4272,
+ "theoretical_loss": 3.872790207605891,
+ "tokens_seen": 593428480
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627562688064193,
+ "loss": 1.3887,
+ "theoretical_loss": 3.87274582421926,
+ "tokens_seen": 593494016
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627402206619861,
+ "loss": 1.4862,
+ "theoretical_loss": 3.872701447105455,
+ "tokens_seen": 593559552
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627241725175527,
+ "loss": 1.5783,
+ "theoretical_loss": 3.8726570762628985,
+ "tokens_seen": 593625088
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006627081243731195,
+ "loss": 1.4866,
+ "theoretical_loss": 3.872612711690011,
+ "tokens_seen": 593690624
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626920762286861,
+ "loss": 1.46,
+ "theoretical_loss": 3.8725683533852155,
+ "tokens_seen": 593756160
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626760280842529,
+ "loss": 1.4521,
+ "theoretical_loss": 3.8725240013469344,
+ "tokens_seen": 593821696
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626599799398195,
+ "loss": 1.5239,
+ "theoretical_loss": 3.8724796555735903,
+ "tokens_seen": 593887232
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626439317953863,
+ "loss": 1.5429,
+ "theoretical_loss": 3.8724353160636085,
+ "tokens_seen": 593952768
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626278836509529,
+ "loss": 1.4103,
+ "theoretical_loss": 3.872390982815412,
+ "tokens_seen": 594018304
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006626118355065197,
+ "loss": 1.3815,
+ "theoretical_loss": 3.8723466558274264,
+ "tokens_seen": 594083840
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006625957873620863,
+ "loss": 1.5167,
+ "theoretical_loss": 3.872302335098077,
+ "tokens_seen": 594149376
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000662579739217653,
+ "loss": 1.4964,
+ "theoretical_loss": 3.872258020625791,
+ "tokens_seen": 594214912
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006625636910732197,
+ "loss": 1.5179,
+ "theoretical_loss": 3.8722137124089935,
+ "tokens_seen": 594280448
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006625476429287864,
+ "loss": 1.4576,
+ "theoretical_loss": 3.8721694104461126,
+ "tokens_seen": 594345984
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006625315947843531,
+ "loss": 1.4038,
+ "theoretical_loss": 3.872125114735576,
+ "tokens_seen": 594411520
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006625155466399198,
+ "loss": 1.4124,
+ "theoretical_loss": 3.8720808252758125,
+ "tokens_seen": 594477056
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624994984954865,
+ "loss": 1.5148,
+ "theoretical_loss": 3.872036542065251,
+ "tokens_seen": 594542592
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624834503510532,
+ "loss": 1.4547,
+ "theoretical_loss": 3.87199226510232,
+ "tokens_seen": 594608128
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624674022066199,
+ "loss": 1.5369,
+ "theoretical_loss": 3.871947994385451,
+ "tokens_seen": 594673664
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624513540621866,
+ "loss": 1.5483,
+ "theoretical_loss": 3.871903729913074,
+ "tokens_seen": 594739200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624353059177532,
+ "loss": 1.4984,
+ "theoretical_loss": 3.8718594716836203,
+ "tokens_seen": 594804736
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00066241925777332,
+ "loss": 1.5162,
+ "theoretical_loss": 3.871815219695522,
+ "tokens_seen": 594870272
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006624032096288866,
+ "loss": 1.3593,
+ "theoretical_loss": 3.8717709739472106,
+ "tokens_seen": 594935808
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006623871614844534,
+ "loss": 1.4981,
+ "theoretical_loss": 3.87172673443712,
+ "tokens_seen": 595001344
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00066237111334002,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8716825011636837,
+ "tokens_seen": 595066880
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006623550651955868,
+ "loss": 1.5133,
+ "theoretical_loss": 3.871638274125335,
+ "tokens_seen": 595132416
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006623390170511534,
+ "loss": 1.3823,
+ "theoretical_loss": 3.871594053320509,
+ "tokens_seen": 595197952
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006623229689067202,
+ "loss": 1.4658,
+ "theoretical_loss": 3.871549838747641,
+ "tokens_seen": 595263488
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006623069207622868,
+ "loss": 1.4668,
+ "theoretical_loss": 3.871505630405167,
+ "tokens_seen": 595329024
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006622908726178536,
+ "loss": 1.5038,
+ "theoretical_loss": 3.8714614282915227,
+ "tokens_seen": 595394560
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006622748244734202,
+ "loss": 1.5095,
+ "theoretical_loss": 3.8714172324051455,
+ "tokens_seen": 595460096
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000662258776328987,
+ "loss": 1.4693,
+ "theoretical_loss": 3.8713730427444726,
+ "tokens_seen": 595525632
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006622427281845536,
+ "loss": 1.4081,
+ "theoretical_loss": 3.8713288593079422,
+ "tokens_seen": 595591168
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006622266800401204,
+ "loss": 1.3935,
+ "theoretical_loss": 3.871284682093993,
+ "tokens_seen": 595656704
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000662210631895687,
+ "loss": 1.4838,
+ "theoretical_loss": 3.871240511101064,
+ "tokens_seen": 595722240
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621945837512538,
+ "loss": 1.491,
+ "theoretical_loss": 3.8711963463275945,
+ "tokens_seen": 595787776
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621785356068204,
+ "loss": 1.4342,
+ "theoretical_loss": 3.871152187772026,
+ "tokens_seen": 595853312
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621624874623872,
+ "loss": 1.4372,
+ "theoretical_loss": 3.8711080354327976,
+ "tokens_seen": 595918848
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621464393179538,
+ "loss": 1.4873,
+ "theoretical_loss": 3.8710638893083518,
+ "tokens_seen": 595984384
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621303911735206,
+ "loss": 1.3888,
+ "theoretical_loss": 3.8710197493971306,
+ "tokens_seen": 596049920
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006621143430290872,
+ "loss": 1.5513,
+ "theoretical_loss": 3.870975615697576,
+ "tokens_seen": 596115456
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000662098294884654,
+ "loss": 1.3557,
+ "theoretical_loss": 3.8709314882081323,
+ "tokens_seen": 596180992
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006620822467402206,
+ "loss": 1.4748,
+ "theoretical_loss": 3.870887366927242,
+ "tokens_seen": 596246528
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006620661985957874,
+ "loss": 1.4486,
+ "theoretical_loss": 3.8708432518533495,
+ "tokens_seen": 596312064
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 204530,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5145456790924072,
+ "objective/train/theoretical_loss": 3.8707991429848994,
+ "objective/train/tokens_used": 616837600,
+ "theoretical_loss": 3.8707991429848994,
+ "tokens_seen": 596377600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000662050150451354,
+ "loss": 1.4601,
+ "theoretical_loss": 3.8707991429848994,
+ "tokens_seen": 596377600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006620341023069208,
+ "loss": 1.4813,
+ "theoretical_loss": 3.8707550403203377,
+ "tokens_seen": 596443136
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006620180541624874,
+ "loss": 1.4676,
+ "theoretical_loss": 3.8707109438581098,
+ "tokens_seen": 596508672
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006620020060180542,
+ "loss": 1.429,
+ "theoretical_loss": 3.8706668535966626,
+ "tokens_seen": 596574208
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006619859578736208,
+ "loss": 1.4921,
+ "theoretical_loss": 3.8706227695344424,
+ "tokens_seen": 596639744
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006619699097291876,
+ "loss": 1.4169,
+ "theoretical_loss": 3.8705786916698974,
+ "tokens_seen": 596705280
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006619538615847542,
+ "loss": 1.411,
+ "theoretical_loss": 3.870534620001476,
+ "tokens_seen": 596770816
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661937813440321,
+ "loss": 1.4635,
+ "theoretical_loss": 3.870490554527626,
+ "tokens_seen": 596836352
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006619217652958876,
+ "loss": 1.4659,
+ "theoretical_loss": 3.8704464952467967,
+ "tokens_seen": 596901888
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006619057171514544,
+ "loss": 1.3666,
+ "theoretical_loss": 3.870402442157439,
+ "tokens_seen": 596967424
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661889669007021,
+ "loss": 1.6077,
+ "theoretical_loss": 3.8703583952580023,
+ "tokens_seen": 597032960
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006618736208625878,
+ "loss": 1.4423,
+ "theoretical_loss": 3.8703143545469376,
+ "tokens_seen": 597098496
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006618575727181545,
+ "loss": 1.5461,
+ "theoretical_loss": 3.870270320022697,
+ "tokens_seen": 597164032
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006618415245737212,
+ "loss": 1.3763,
+ "theoretical_loss": 3.8702262916837316,
+ "tokens_seen": 597229568
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661825476429288,
+ "loss": 1.5186,
+ "theoretical_loss": 3.8701822695284953,
+ "tokens_seen": 597295104
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006618094282848546,
+ "loss": 1.4728,
+ "theoretical_loss": 3.8701382535554396,
+ "tokens_seen": 597360640
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006617933801404213,
+ "loss": 1.4291,
+ "theoretical_loss": 3.8700942437630195,
+ "tokens_seen": 597426176
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661777331995988,
+ "loss": 1.51,
+ "theoretical_loss": 3.870050240149689,
+ "tokens_seen": 597491712
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006617612838515548,
+ "loss": 1.4125,
+ "theoretical_loss": 3.870006242713903,
+ "tokens_seen": 597557248
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006617452357071214,
+ "loss": 1.5887,
+ "theoretical_loss": 3.8699622514541163,
+ "tokens_seen": 597622784
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006617291875626882,
+ "loss": 1.5154,
+ "theoretical_loss": 3.8699182663687854,
+ "tokens_seen": 597688320
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006617131394182548,
+ "loss": 1.4929,
+ "theoretical_loss": 3.8698742874563665,
+ "tokens_seen": 597753856
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006616970912738216,
+ "loss": 1.4577,
+ "theoretical_loss": 3.8698303147153172,
+ "tokens_seen": 597819392
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006616810431293882,
+ "loss": 1.3801,
+ "theoretical_loss": 3.8697863481440944,
+ "tokens_seen": 597884928
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661664994984955,
+ "loss": 1.5343,
+ "theoretical_loss": 3.8697423877411565,
+ "tokens_seen": 597950464
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006616489468405216,
+ "loss": 1.4261,
+ "theoretical_loss": 3.8696984335049622,
+ "tokens_seen": 598016000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006616328986960884,
+ "loss": 1.3522,
+ "theoretical_loss": 3.869654485433971,
+ "tokens_seen": 598081536
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661616850551655,
+ "loss": 1.4609,
+ "theoretical_loss": 3.869610543526643,
+ "tokens_seen": 598147072
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006616008024072218,
+ "loss": 1.4074,
+ "theoretical_loss": 3.8695666077814375,
+ "tokens_seen": 598212608
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006615847542627884,
+ "loss": 1.4669,
+ "theoretical_loss": 3.869522678196816,
+ "tokens_seen": 598278144
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006615687061183552,
+ "loss": 1.5038,
+ "theoretical_loss": 3.86947875477124,
+ "tokens_seen": 598343680
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006615526579739218,
+ "loss": 1.4047,
+ "theoretical_loss": 3.8694348375031717,
+ "tokens_seen": 598409216
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006615366098294886,
+ "loss": 1.3909,
+ "theoretical_loss": 3.8693909263910737,
+ "tokens_seen": 598474752
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006615205616850552,
+ "loss": 1.4707,
+ "theoretical_loss": 3.869347021433409,
+ "tokens_seen": 598540288
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661504513540622,
+ "loss": 1.4068,
+ "theoretical_loss": 3.869303122628641,
+ "tokens_seen": 598605824
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006614884653961886,
+ "loss": 1.378,
+ "theoretical_loss": 3.869259229975235,
+ "tokens_seen": 598671360
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006614724172517554,
+ "loss": 1.4765,
+ "theoretical_loss": 3.869215343471654,
+ "tokens_seen": 598736896
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661456369107322,
+ "loss": 1.5062,
+ "theoretical_loss": 3.8691714631163645,
+ "tokens_seen": 598802432
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006614403209628888,
+ "loss": 1.4873,
+ "theoretical_loss": 3.8691275889078325,
+ "tokens_seen": 598867968
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006614242728184554,
+ "loss": 1.5145,
+ "theoretical_loss": 3.8690837208445243,
+ "tokens_seen": 598933504
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006614082246740222,
+ "loss": 1.5306,
+ "theoretical_loss": 3.869039858924906,
+ "tokens_seen": 598999040
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006613921765295888,
+ "loss": 1.5118,
+ "theoretical_loss": 3.868996003147447,
+ "tokens_seen": 599064576
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006613761283851556,
+ "loss": 1.3758,
+ "theoretical_loss": 3.868952153510614,
+ "tokens_seen": 599130112
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006613600802407222,
+ "loss": 1.4249,
+ "theoretical_loss": 3.868908310012876,
+ "tokens_seen": 599195648
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661344032096289,
+ "loss": 1.481,
+ "theoretical_loss": 3.868864472652702,
+ "tokens_seen": 599261184
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006613279839518556,
+ "loss": 1.3103,
+ "theoretical_loss": 3.868820641428562,
+ "tokens_seen": 599326720
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006613119358074224,
+ "loss": 1.5188,
+ "theoretical_loss": 3.868776816338926,
+ "tokens_seen": 599392256
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661295887662989,
+ "loss": 1.5063,
+ "theoretical_loss": 3.8687329973822653,
+ "tokens_seen": 599457792
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006612798395185558,
+ "loss": 1.4514,
+ "theoretical_loss": 3.868689184557051,
+ "tokens_seen": 599523328
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006612637913741224,
+ "loss": 1.5114,
+ "theoretical_loss": 3.8686453778617547,
+ "tokens_seen": 599588864
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 205743,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4836275577545166,
+ "objective/train/theoretical_loss": 3.868601577294849,
+ "objective/train/tokens_used": 620114400,
+ "theoretical_loss": 3.868601577294849,
+ "tokens_seen": 599654400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006612477432296892,
+ "loss": 1.5095,
+ "theoretical_loss": 3.868601577294849,
+ "tokens_seen": 599654400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006612316950852558,
+ "loss": 1.2991,
+ "theoretical_loss": 3.8685577828548086,
+ "tokens_seen": 599719936
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006612156469408226,
+ "loss": 1.3201,
+ "theoretical_loss": 3.8685139945401046,
+ "tokens_seen": 599785472
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611995987963892,
+ "loss": 1.4232,
+ "theoretical_loss": 3.8684702123492123,
+ "tokens_seen": 599851008
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611835506519559,
+ "loss": 1.5699,
+ "theoretical_loss": 3.8684264362806062,
+ "tokens_seen": 599916544
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611675025075226,
+ "loss": 1.5524,
+ "theoretical_loss": 3.868382666332762,
+ "tokens_seen": 599982080
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611514543630893,
+ "loss": 1.4625,
+ "theoretical_loss": 3.8683389025041546,
+ "tokens_seen": 600047616
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000661135406218656,
+ "loss": 1.4177,
+ "theoretical_loss": 3.8682951447932608,
+ "tokens_seen": 600113152
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611193580742227,
+ "loss": 1.3461,
+ "theoretical_loss": 3.8682513931985576,
+ "tokens_seen": 600178688
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006611033099297894,
+ "loss": 1.3989,
+ "theoretical_loss": 3.8682076477185223,
+ "tokens_seen": 600244224
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610872617853561,
+ "loss": 1.4005,
+ "theoretical_loss": 3.868163908351632,
+ "tokens_seen": 600309760
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610712136409228,
+ "loss": 1.4583,
+ "theoretical_loss": 3.868120175096367,
+ "tokens_seen": 600375296
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610551654964895,
+ "loss": 1.5047,
+ "theoretical_loss": 3.868076447951205,
+ "tokens_seen": 600440832
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610391173520561,
+ "loss": 1.4884,
+ "theoretical_loss": 3.8680327269146257,
+ "tokens_seen": 600506368
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610230692076229,
+ "loss": 1.347,
+ "theoretical_loss": 3.8679890119851095,
+ "tokens_seen": 600571904
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006610070210631895,
+ "loss": 1.466,
+ "theoretical_loss": 3.867945303161137,
+ "tokens_seen": 600637440
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609909729187563,
+ "loss": 1.4438,
+ "theoretical_loss": 3.8679016004411895,
+ "tokens_seen": 600702976
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609749247743229,
+ "loss": 1.4553,
+ "theoretical_loss": 3.867857903823748,
+ "tokens_seen": 600768512
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609588766298897,
+ "loss": 1.4764,
+ "theoretical_loss": 3.8678142133072964,
+ "tokens_seen": 600834048
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609428284854563,
+ "loss": 1.4829,
+ "theoretical_loss": 3.8677705288903166,
+ "tokens_seen": 600899584
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609267803410231,
+ "loss": 1.5542,
+ "theoretical_loss": 3.867726850571291,
+ "tokens_seen": 600965120
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006609107321965897,
+ "loss": 1.3085,
+ "theoretical_loss": 3.867683178348705,
+ "tokens_seen": 601030656
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608946840521565,
+ "loss": 1.4878,
+ "theoretical_loss": 3.867639512221043,
+ "tokens_seen": 601096192
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608786359077231,
+ "loss": 1.4221,
+ "theoretical_loss": 3.8675958521867893,
+ "tokens_seen": 601161728
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608625877632899,
+ "loss": 1.4264,
+ "theoretical_loss": 3.8675521982444296,
+ "tokens_seen": 601227264
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608465396188565,
+ "loss": 1.5051,
+ "theoretical_loss": 3.8675085503924507,
+ "tokens_seen": 601292800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608304914744233,
+ "loss": 1.4914,
+ "theoretical_loss": 3.867464908629338,
+ "tokens_seen": 601358336
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006608144433299899,
+ "loss": 1.4987,
+ "theoretical_loss": 3.86742127295358,
+ "tokens_seen": 601423872
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607983951855567,
+ "loss": 1.3915,
+ "theoretical_loss": 3.8673776433636635,
+ "tokens_seen": 601489408
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607823470411233,
+ "loss": 1.5461,
+ "theoretical_loss": 3.8673340198580766,
+ "tokens_seen": 601554944
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607662988966901,
+ "loss": 1.3752,
+ "theoretical_loss": 3.867290402435309,
+ "tokens_seen": 601620480
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607502507522567,
+ "loss": 1.5197,
+ "theoretical_loss": 3.8672467910938497,
+ "tokens_seen": 601686016
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607342026078235,
+ "loss": 1.4824,
+ "theoretical_loss": 3.8672031858321887,
+ "tokens_seen": 601751552
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607181544633901,
+ "loss": 1.3691,
+ "theoretical_loss": 3.8671595866488158,
+ "tokens_seen": 601817088
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006607021063189569,
+ "loss": 1.4666,
+ "theoretical_loss": 3.867115993542222,
+ "tokens_seen": 601882624
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006606860581745235,
+ "loss": 1.5184,
+ "theoretical_loss": 3.8670724065109,
+ "tokens_seen": 601948160
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006606700100300903,
+ "loss": 1.4481,
+ "theoretical_loss": 3.8670288255533403,
+ "tokens_seen": 602013696
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000660653961885657,
+ "loss": 1.4513,
+ "theoretical_loss": 3.8669852506680362,
+ "tokens_seen": 602079232
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006606379137412237,
+ "loss": 1.4266,
+ "theoretical_loss": 3.866941681853481,
+ "tokens_seen": 602144768
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006606218655967904,
+ "loss": 1.4738,
+ "theoretical_loss": 3.866898119108168,
+ "tokens_seen": 602210304
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006606058174523571,
+ "loss": 1.4137,
+ "theoretical_loss": 3.866854562430592,
+ "tokens_seen": 602275840
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605897693079238,
+ "loss": 1.4907,
+ "theoretical_loss": 3.8668110118192462,
+ "tokens_seen": 602341376
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605737211634905,
+ "loss": 1.4957,
+ "theoretical_loss": 3.8667674672726275,
+ "tokens_seen": 602406912
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605576730190572,
+ "loss": 1.465,
+ "theoretical_loss": 3.866723928789231,
+ "tokens_seen": 602472448
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605416248746239,
+ "loss": 1.4293,
+ "theoretical_loss": 3.8666803963675536,
+ "tokens_seen": 602537984
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605255767301906,
+ "loss": 1.4752,
+ "theoretical_loss": 3.866636870006091,
+ "tokens_seen": 602603520
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006605095285857573,
+ "loss": 1.4357,
+ "theoretical_loss": 3.866593349703342,
+ "tokens_seen": 602669056
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000660493480441324,
+ "loss": 1.4409,
+ "theoretical_loss": 3.8665498354578034,
+ "tokens_seen": 602734592
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006604774322968907,
+ "loss": 1.5654,
+ "theoretical_loss": 3.866506327267974,
+ "tokens_seen": 602800128
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006604613841524574,
+ "loss": 1.4332,
+ "theoretical_loss": 3.866462825132353,
+ "tokens_seen": 602865664
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 206667,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4438321590423584,
+ "objective/train/theoretical_loss": 3.86641932904944,
+ "objective/train/tokens_used": 623391200,
+ "theoretical_loss": 3.86641932904944,
+ "tokens_seen": 602931200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006604453360080241,
+ "loss": 1.458,
+ "theoretical_loss": 3.86641932904944,
+ "tokens_seen": 602931200
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006604292878635908,
+ "loss": 1.3835,
+ "theoretical_loss": 3.866375839017735,
+ "tokens_seen": 602996736
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006604132397191575,
+ "loss": 1.3458,
+ "theoretical_loss": 3.8663323550357385,
+ "tokens_seen": 603062272
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603971915747243,
+ "loss": 1.3798,
+ "theoretical_loss": 3.866288877101952,
+ "tokens_seen": 603127808
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603811434302909,
+ "loss": 1.4673,
+ "theoretical_loss": 3.866245405214877,
+ "tokens_seen": 603193344
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603650952858577,
+ "loss": 1.4339,
+ "theoretical_loss": 3.8662019393730156,
+ "tokens_seen": 603258880
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603490471414243,
+ "loss": 1.4291,
+ "theoretical_loss": 3.86615847957487,
+ "tokens_seen": 603324416
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000660332998996991,
+ "loss": 1.4796,
+ "theoretical_loss": 3.866115025818945,
+ "tokens_seen": 603389952
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603169508525577,
+ "loss": 1.2908,
+ "theoretical_loss": 3.8660715781037434,
+ "tokens_seen": 603455488
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006603009027081245,
+ "loss": 1.6529,
+ "theoretical_loss": 3.8660281364277695,
+ "tokens_seen": 603521024
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602848545636911,
+ "loss": 1.4889,
+ "theoretical_loss": 3.865984700789528,
+ "tokens_seen": 603586560
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602688064192579,
+ "loss": 1.397,
+ "theoretical_loss": 3.865941271187525,
+ "tokens_seen": 603652096
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602527582748245,
+ "loss": 1.415,
+ "theoretical_loss": 3.865897847620266,
+ "tokens_seen": 603717632
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602367101303913,
+ "loss": 1.479,
+ "theoretical_loss": 3.865854430086258,
+ "tokens_seen": 603783168
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602206619859579,
+ "loss": 1.3538,
+ "theoretical_loss": 3.865811018584007,
+ "tokens_seen": 603848704
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006602046138415247,
+ "loss": 1.4362,
+ "theoretical_loss": 3.8657676131120224,
+ "tokens_seen": 603914240
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601885656970913,
+ "loss": 1.3837,
+ "theoretical_loss": 3.86572421366881,
+ "tokens_seen": 603979776
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601725175526581,
+ "loss": 1.4328,
+ "theoretical_loss": 3.86568082025288,
+ "tokens_seen": 604045312
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601564694082247,
+ "loss": 1.3513,
+ "theoretical_loss": 3.8656374328627408,
+ "tokens_seen": 604110848
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601404212637915,
+ "loss": 1.4515,
+ "theoretical_loss": 3.8655940514969025,
+ "tokens_seen": 604176384
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601243731193581,
+ "loss": 1.4263,
+ "theoretical_loss": 3.865550676153875,
+ "tokens_seen": 604241920
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006601083249749249,
+ "loss": 1.3591,
+ "theoretical_loss": 3.8655073068321695,
+ "tokens_seen": 604307456
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600922768304915,
+ "loss": 1.5243,
+ "theoretical_loss": 3.8654639435302967,
+ "tokens_seen": 604372992
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600762286860583,
+ "loss": 1.4241,
+ "theoretical_loss": 3.865420586246768,
+ "tokens_seen": 604438528
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600601805416249,
+ "loss": 1.4579,
+ "theoretical_loss": 3.865377234980097,
+ "tokens_seen": 604504064
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600441323971917,
+ "loss": 1.3609,
+ "theoretical_loss": 3.865333889728795,
+ "tokens_seen": 604569600
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600280842527583,
+ "loss": 1.392,
+ "theoretical_loss": 3.8652905504913777,
+ "tokens_seen": 604635136
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006600120361083251,
+ "loss": 1.3966,
+ "theoretical_loss": 3.865247217266356,
+ "tokens_seen": 604700672
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599959879638917,
+ "loss": 1.4724,
+ "theoretical_loss": 3.865203890052247,
+ "tokens_seen": 604766208
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599799398194585,
+ "loss": 1.3838,
+ "theoretical_loss": 3.865160568847564,
+ "tokens_seen": 604831744
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599638916750251,
+ "loss": 1.427,
+ "theoretical_loss": 3.865117253650823,
+ "tokens_seen": 604897280
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599478435305919,
+ "loss": 1.382,
+ "theoretical_loss": 3.8650739444605406,
+ "tokens_seen": 604962816
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599317953861585,
+ "loss": 1.4716,
+ "theoretical_loss": 3.8650306412752324,
+ "tokens_seen": 605028352
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006599157472417253,
+ "loss": 1.4328,
+ "theoretical_loss": 3.8649873440934157,
+ "tokens_seen": 605093888
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598996990972919,
+ "loss": 1.4793,
+ "theoretical_loss": 3.8649440529136085,
+ "tokens_seen": 605159424
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598836509528587,
+ "loss": 1.3659,
+ "theoretical_loss": 3.864900767734329,
+ "tokens_seen": 605224960
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598676028084253,
+ "loss": 1.4324,
+ "theoretical_loss": 3.8648574885540947,
+ "tokens_seen": 605290496
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598515546639921,
+ "loss": 1.3815,
+ "theoretical_loss": 3.8648142153714264,
+ "tokens_seen": 605356032
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598355065195587,
+ "loss": 1.4118,
+ "theoretical_loss": 3.8647709481848427,
+ "tokens_seen": 605421568
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598194583751255,
+ "loss": 1.4245,
+ "theoretical_loss": 3.8647276869928637,
+ "tokens_seen": 605487104
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006598034102306921,
+ "loss": 1.4302,
+ "theoretical_loss": 3.864684431794011,
+ "tokens_seen": 605552640
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597873620862588,
+ "loss": 1.4232,
+ "theoretical_loss": 3.864641182586806,
+ "tokens_seen": 605618176
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597713139418255,
+ "loss": 1.4132,
+ "theoretical_loss": 3.8645979393697694,
+ "tokens_seen": 605683712
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597552657973922,
+ "loss": 1.4686,
+ "theoretical_loss": 3.864554702141424,
+ "tokens_seen": 605749248
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597392176529589,
+ "loss": 1.3692,
+ "theoretical_loss": 3.8645114709002932,
+ "tokens_seen": 605814784
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597231695085256,
+ "loss": 1.4305,
+ "theoretical_loss": 3.8644682456448995,
+ "tokens_seen": 605880320
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006597071213640923,
+ "loss": 1.3826,
+ "theoretical_loss": 3.864425026373768,
+ "tokens_seen": 605945856
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659691073219659,
+ "loss": 1.464,
+ "theoretical_loss": 3.864381813085422,
+ "tokens_seen": 606011392
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006596750250752257,
+ "loss": 1.3892,
+ "theoretical_loss": 3.8643386057783866,
+ "tokens_seen": 606076928
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006596589769307924,
+ "loss": 1.4343,
+ "theoretical_loss": 3.8642954044511875,
+ "tokens_seen": 606142464
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 207838,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.6236754655838013,
+ "objective/train/theoretical_loss": 3.8642522091023506,
+ "objective/train/tokens_used": 626668000,
+ "theoretical_loss": 3.8642522091023506,
+ "tokens_seen": 606208000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659642928786359,
+ "loss": 1.5245,
+ "theoretical_loss": 3.8642522091023506,
+ "tokens_seen": 606208000
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006596268806419258,
+ "loss": 1.5009,
+ "theoretical_loss": 3.864209019730403,
+ "tokens_seen": 606273536
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006596108324974924,
+ "loss": 1.3572,
+ "theoretical_loss": 3.8641658363338705,
+ "tokens_seen": 606339072
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006595947843530592,
+ "loss": 1.4147,
+ "theoretical_loss": 3.8641226589112816,
+ "tokens_seen": 606404608
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006595787362086258,
+ "loss": 1.3962,
+ "theoretical_loss": 3.8640794874611646,
+ "tokens_seen": 606470144
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006595626880641926,
+ "loss": 1.4121,
+ "theoretical_loss": 3.8640363219820473,
+ "tokens_seen": 606535680
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006595466399197592,
+ "loss": 1.4212,
+ "theoretical_loss": 3.8639931624724593,
+ "tokens_seen": 606601216
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659530591775326,
+ "loss": 1.5439,
+ "theoretical_loss": 3.8639500089309307,
+ "tokens_seen": 606666752
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006595145436308926,
+ "loss": 1.4606,
+ "theoretical_loss": 3.86390686135599,
+ "tokens_seen": 606732288
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594984954864594,
+ "loss": 1.4124,
+ "theoretical_loss": 3.86386371974617,
+ "tokens_seen": 606797824
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659482447342026,
+ "loss": 1.4786,
+ "theoretical_loss": 3.863820584100001,
+ "tokens_seen": 606863360
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594663991975928,
+ "loss": 1.4325,
+ "theoretical_loss": 3.863777454416014,
+ "tokens_seen": 606928896
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594503510531595,
+ "loss": 1.4714,
+ "theoretical_loss": 3.863734330692742,
+ "tokens_seen": 606994432
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594343029087262,
+ "loss": 1.4178,
+ "theoretical_loss": 3.8636912129287175,
+ "tokens_seen": 607059968
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594182547642929,
+ "loss": 1.4254,
+ "theoretical_loss": 3.8636481011224744,
+ "tokens_seen": 607125504
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006594022066198596,
+ "loss": 1.5808,
+ "theoretical_loss": 3.8636049952725466,
+ "tokens_seen": 607191040
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006593861584754263,
+ "loss": 1.3405,
+ "theoretical_loss": 3.863561895377467,
+ "tokens_seen": 607256576
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659370110330993,
+ "loss": 1.4353,
+ "theoretical_loss": 3.8635188014357715,
+ "tokens_seen": 607322112
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006593540621865597,
+ "loss": 1.4427,
+ "theoretical_loss": 3.8634757134459954,
+ "tokens_seen": 607387648
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006593380140421264,
+ "loss": 1.5434,
+ "theoretical_loss": 3.8634326314066745,
+ "tokens_seen": 607453184
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006593219658976931,
+ "loss": 1.4577,
+ "theoretical_loss": 3.8633895553163455,
+ "tokens_seen": 607518720
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006593059177532598,
+ "loss": 1.4115,
+ "theoretical_loss": 3.8633464851735444,
+ "tokens_seen": 607584256
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006592898696088265,
+ "loss": 1.512,
+ "theoretical_loss": 3.8633034209768096,
+ "tokens_seen": 607649792
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006592738214643932,
+ "loss": 1.4788,
+ "theoretical_loss": 3.8632603627246787,
+ "tokens_seen": 607715328
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006592577733199599,
+ "loss": 1.3012,
+ "theoretical_loss": 3.86321731041569,
+ "tokens_seen": 607780864
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006592417251755266,
+ "loss": 1.341,
+ "theoretical_loss": 3.863174264048383,
+ "tokens_seen": 607846400
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006592256770310933,
+ "loss": 1.3701,
+ "theoretical_loss": 3.8631312236212967,
+ "tokens_seen": 607911936
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.00065920962888666,
+ "loss": 1.3869,
+ "theoretical_loss": 3.863088189132971,
+ "tokens_seen": 607977472
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591935807422267,
+ "loss": 1.4,
+ "theoretical_loss": 3.8630451605819474,
+ "tokens_seen": 608043008
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591775325977934,
+ "loss": 1.4417,
+ "theoretical_loss": 3.8630021379667654,
+ "tokens_seen": 608108544
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591614844533601,
+ "loss": 1.3807,
+ "theoretical_loss": 3.862959121285968,
+ "tokens_seen": 608174080
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591454363089268,
+ "loss": 1.4167,
+ "theoretical_loss": 3.8629161105380962,
+ "tokens_seen": 608239616
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591293881644935,
+ "loss": 1.367,
+ "theoretical_loss": 3.862873105721694,
+ "tokens_seen": 608305152
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006591133400200602,
+ "loss": 1.3104,
+ "theoretical_loss": 3.862830106835303,
+ "tokens_seen": 608370688
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659097291875627,
+ "loss": 1.4652,
+ "theoretical_loss": 3.8627871138774674,
+ "tokens_seen": 608436224
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006590812437311936,
+ "loss": 1.5179,
+ "theoretical_loss": 3.862744126846732,
+ "tokens_seen": 608501760
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006590651955867603,
+ "loss": 1.3854,
+ "theoretical_loss": 3.8627011457416405,
+ "tokens_seen": 608567296
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000659049147442327,
+ "loss": 1.5049,
+ "theoretical_loss": 3.862658170560738,
+ "tokens_seen": 608632832
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006590330992978938,
+ "loss": 1.4641,
+ "theoretical_loss": 3.862615201302571,
+ "tokens_seen": 608698368
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006590170511534604,
+ "loss": 1.4137,
+ "theoretical_loss": 3.8625722379656855,
+ "tokens_seen": 608763904
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006590010030090272,
+ "loss": 1.4818,
+ "theoretical_loss": 3.862529280548628,
+ "tokens_seen": 608829440
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006589849548645938,
+ "loss": 1.403,
+ "theoretical_loss": 3.8624863290499456,
+ "tokens_seen": 608894976
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006589689067201606,
+ "loss": 1.4422,
+ "theoretical_loss": 3.862443383468186,
+ "tokens_seen": 608960512
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006589528585757272,
+ "loss": 1.4226,
+ "theoretical_loss": 3.8624004438018984,
+ "tokens_seen": 609026048
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000658936810431294,
+ "loss": 1.3911,
+ "theoretical_loss": 3.8623575100496303,
+ "tokens_seen": 609091584
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006589207622868606,
+ "loss": 1.4543,
+ "theoretical_loss": 3.8623145822099314,
+ "tokens_seen": 609157120
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006589047141424274,
+ "loss": 1.4233,
+ "theoretical_loss": 3.8622716602813516,
+ "tokens_seen": 609222656
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000658888665997994,
+ "loss": 1.5068,
+ "theoretical_loss": 3.862228744262441,
+ "tokens_seen": 609288192
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006588726178535608,
+ "loss": 1.3251,
+ "theoretical_loss": 3.8621858341517514,
+ "tokens_seen": 609353728
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006588565697091274,
+ "loss": 1.4483,
+ "theoretical_loss": 3.8621429299478325,
+ "tokens_seen": 609419264
+ },
+ {
+ "epoch": 0.18,
+ "objective/train/docs_used": 208904,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2397621870040894,
+ "objective/train/theoretical_loss": 3.862100031649237,
+ "objective/train/tokens_used": 629944800,
+ "theoretical_loss": 3.862100031649237,
+ "tokens_seen": 609484800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006588405215646942,
+ "loss": 1.297,
+ "theoretical_loss": 3.862100031649237,
+ "tokens_seen": 609484800
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006588244734202608,
+ "loss": 1.3689,
+ "theoretical_loss": 3.8620571392545173,
+ "tokens_seen": 609550336
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006588084252758276,
+ "loss": 1.5308,
+ "theoretical_loss": 3.8620142527622265,
+ "tokens_seen": 609615872
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006587923771313942,
+ "loss": 1.3965,
+ "theoretical_loss": 3.8619713721709172,
+ "tokens_seen": 609681408
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000658776328986961,
+ "loss": 1.3237,
+ "theoretical_loss": 3.861928497479144,
+ "tokens_seen": 609746944
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006587602808425276,
+ "loss": 1.4447,
+ "theoretical_loss": 3.86188562868546,
+ "tokens_seen": 609812480
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006587442326980944,
+ "loss": 1.4676,
+ "theoretical_loss": 3.8618427657884222,
+ "tokens_seen": 609878016
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000658728184553661,
+ "loss": 1.4838,
+ "theoretical_loss": 3.861799908786584,
+ "tokens_seen": 609943552
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006587121364092278,
+ "loss": 1.5459,
+ "theoretical_loss": 3.8617570576785027,
+ "tokens_seen": 610009088
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006586960882647944,
+ "loss": 1.315,
+ "theoretical_loss": 3.8617142124627346,
+ "tokens_seen": 610074624
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006586800401203612,
+ "loss": 1.5473,
+ "theoretical_loss": 3.8616713731378356,
+ "tokens_seen": 610140160
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006586639919759278,
+ "loss": 1.4366,
+ "theoretical_loss": 3.8616285397023637,
+ "tokens_seen": 610205696
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006586479438314946,
+ "loss": 1.4457,
+ "theoretical_loss": 3.861585712154877,
+ "tokens_seen": 610271232
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006586318956870612,
+ "loss": 1.4372,
+ "theoretical_loss": 3.8615428904939337,
+ "tokens_seen": 610336768
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.000658615847542628,
+ "loss": 1.4573,
+ "theoretical_loss": 3.861500074718093,
+ "tokens_seen": 610402304
+ },
+ {
+ "epoch": 0.18,
+ "learning_rate": 0.0006585997993981946,
+ "loss": 1.4509,
+ "theoretical_loss": 3.8614572648259142,
+ "tokens_seen": 610467840
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006585837512537614,
+ "loss": 1.4814,
+ "theoretical_loss": 3.8614144608159577,
+ "tokens_seen": 610533376
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000658567703109328,
+ "loss": 1.4871,
+ "theoretical_loss": 3.8613716626867833,
+ "tokens_seen": 610598912
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006585516549648948,
+ "loss": 1.5432,
+ "theoretical_loss": 3.861328870436952,
+ "tokens_seen": 610664448
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006585356068204614,
+ "loss": 1.502,
+ "theoretical_loss": 3.8612860840650267,
+ "tokens_seen": 610729984
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006585195586760282,
+ "loss": 1.391,
+ "theoretical_loss": 3.8612433035695672,
+ "tokens_seen": 610795520
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006585035105315948,
+ "loss": 1.4072,
+ "theoretical_loss": 3.861200528949138,
+ "tokens_seen": 610861056
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006584874623871616,
+ "loss": 1.378,
+ "theoretical_loss": 3.8611577602023006,
+ "tokens_seen": 610926592
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006584714142427282,
+ "loss": 1.4051,
+ "theoretical_loss": 3.8611149973276193,
+ "tokens_seen": 610992128
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000658455366098295,
+ "loss": 1.3886,
+ "theoretical_loss": 3.861072240323658,
+ "tokens_seen": 611057664
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006584393179538616,
+ "loss": 1.4285,
+ "theoretical_loss": 3.861029489188981,
+ "tokens_seen": 611123200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006584232698094284,
+ "loss": 1.3538,
+ "theoretical_loss": 3.8609867439221537,
+ "tokens_seen": 611188736
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000658407221664995,
+ "loss": 1.4264,
+ "theoretical_loss": 3.860944004521742,
+ "tokens_seen": 611254272
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583911735205618,
+ "loss": 1.4412,
+ "theoretical_loss": 3.860901270986311,
+ "tokens_seen": 611319808
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583751253761284,
+ "loss": 1.4681,
+ "theoretical_loss": 3.8608585433144276,
+ "tokens_seen": 611385344
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583590772316951,
+ "loss": 1.4123,
+ "theoretical_loss": 3.860815821504659,
+ "tokens_seen": 611450880
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583430290872618,
+ "loss": 1.3016,
+ "theoretical_loss": 3.860773105555573,
+ "tokens_seen": 611516416
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583269809428285,
+ "loss": 1.4889,
+ "theoretical_loss": 3.860730395465737,
+ "tokens_seen": 611581952
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006583109327983952,
+ "loss": 1.3215,
+ "theoretical_loss": 3.86068769123372,
+ "tokens_seen": 611647488
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006582948846539619,
+ "loss": 1.4208,
+ "theoretical_loss": 3.860644992858091,
+ "tokens_seen": 611713024
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006582788365095286,
+ "loss": 1.459,
+ "theoretical_loss": 3.8606023003374195,
+ "tokens_seen": 611778560
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006582627883650953,
+ "loss": 1.3311,
+ "theoretical_loss": 3.860559613670276,
+ "tokens_seen": 611844096
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000658246740220662,
+ "loss": 1.4441,
+ "theoretical_loss": 3.86051693285523,
+ "tokens_seen": 611909632
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006582306920762287,
+ "loss": 1.3947,
+ "theoretical_loss": 3.8604742578908535,
+ "tokens_seen": 611975168
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006582146439317954,
+ "loss": 1.4688,
+ "theoretical_loss": 3.860431588775718,
+ "tokens_seen": 612040704
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581985957873621,
+ "loss": 1.4368,
+ "theoretical_loss": 3.860388925508395,
+ "tokens_seen": 612106240
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581825476429288,
+ "loss": 1.397,
+ "theoretical_loss": 3.8603462680874583,
+ "tokens_seen": 612171776
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581664994984955,
+ "loss": 1.4094,
+ "theoretical_loss": 3.8603036165114797,
+ "tokens_seen": 612237312
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581504513540622,
+ "loss": 1.5308,
+ "theoretical_loss": 3.8602609707790334,
+ "tokens_seen": 612302848
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581344032096289,
+ "loss": 1.4203,
+ "theoretical_loss": 3.860218330888693,
+ "tokens_seen": 612368384
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581183550651956,
+ "loss": 1.4991,
+ "theoretical_loss": 3.8601756968390335,
+ "tokens_seen": 612433920
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006581023069207623,
+ "loss": 1.4507,
+ "theoretical_loss": 3.86013306862863,
+ "tokens_seen": 612499456
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000658086258776329,
+ "loss": 1.3386,
+ "theoretical_loss": 3.8600904462560583,
+ "tokens_seen": 612564992
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006580702106318957,
+ "loss": 1.5372,
+ "theoretical_loss": 3.8600478297198944,
+ "tokens_seen": 612630528
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006580541624874624,
+ "loss": 1.551,
+ "theoretical_loss": 3.860005219018714,
+ "tokens_seen": 612696064
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 210170,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.420013427734375,
+ "objective/train/theoretical_loss": 3.8599626141510948,
+ "objective/train/tokens_used": 633221600,
+ "theoretical_loss": 3.8599626141510948,
+ "tokens_seen": 612761600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006580381143430291,
+ "loss": 1.4537,
+ "theoretical_loss": 3.8599626141510948,
+ "tokens_seen": 612761600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006580220661985958,
+ "loss": 1.4975,
+ "theoretical_loss": 3.859920015115615,
+ "tokens_seen": 612827136
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006580060180541625,
+ "loss": 1.5048,
+ "theoretical_loss": 3.8598774219108516,
+ "tokens_seen": 612892672
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006579899699097292,
+ "loss": 1.4598,
+ "theoretical_loss": 3.859834834535384,
+ "tokens_seen": 612958208
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006579739217652959,
+ "loss": 1.5072,
+ "theoretical_loss": 3.859792252987791,
+ "tokens_seen": 613023744
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006579578736208626,
+ "loss": 1.5103,
+ "theoretical_loss": 3.859749677266652,
+ "tokens_seen": 613089280
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006579418254764293,
+ "loss": 1.3005,
+ "theoretical_loss": 3.8597071073705473,
+ "tokens_seen": 613154816
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000657925777331996,
+ "loss": 1.4161,
+ "theoretical_loss": 3.8596645432980567,
+ "tokens_seen": 613220352
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006579097291875627,
+ "loss": 1.415,
+ "theoretical_loss": 3.859621985047763,
+ "tokens_seen": 613285888
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578936810431294,
+ "loss": 1.4236,
+ "theoretical_loss": 3.859579432618246,
+ "tokens_seen": 613351424
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578776328986961,
+ "loss": 1.4362,
+ "theoretical_loss": 3.8595368860080894,
+ "tokens_seen": 613416960
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578615847542628,
+ "loss": 1.3682,
+ "theoretical_loss": 3.8594943452158743,
+ "tokens_seen": 613482496
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578455366098295,
+ "loss": 1.3953,
+ "theoretical_loss": 3.8594518102401842,
+ "tokens_seen": 613548032
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578294884653962,
+ "loss": 1.3524,
+ "theoretical_loss": 3.8594092810796026,
+ "tokens_seen": 613613568
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006578134403209629,
+ "loss": 1.4545,
+ "theoretical_loss": 3.859366757732714,
+ "tokens_seen": 613679104
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577973921765296,
+ "loss": 1.3618,
+ "theoretical_loss": 3.8593242401981027,
+ "tokens_seen": 613744640
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577813440320963,
+ "loss": 1.4137,
+ "theoretical_loss": 3.859281728474354,
+ "tokens_seen": 613810176
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000657765295887663,
+ "loss": 1.4826,
+ "theoretical_loss": 3.8592392225600527,
+ "tokens_seen": 613875712
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577492477432297,
+ "loss": 1.5189,
+ "theoretical_loss": 3.859196722453785,
+ "tokens_seen": 613941248
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577331995987964,
+ "loss": 1.4141,
+ "theoretical_loss": 3.859154228154138,
+ "tokens_seen": 614006784
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577171514543631,
+ "loss": 1.3756,
+ "theoretical_loss": 3.8591117396596983,
+ "tokens_seen": 614072320
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006577011033099298,
+ "loss": 1.3612,
+ "theoretical_loss": 3.859069256969054,
+ "tokens_seen": 614137856
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006576850551654965,
+ "loss": 1.3814,
+ "theoretical_loss": 3.859026780080792,
+ "tokens_seen": 614203392
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006576690070210633,
+ "loss": 1.4247,
+ "theoretical_loss": 3.8589843089935014,
+ "tokens_seen": 614268928
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006576529588766299,
+ "loss": 1.4333,
+ "theoretical_loss": 3.8589418437057716,
+ "tokens_seen": 614334464
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006576369107321967,
+ "loss": 1.4784,
+ "theoretical_loss": 3.8588993842161923,
+ "tokens_seen": 614400000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006576208625877633,
+ "loss": 1.4326,
+ "theoretical_loss": 3.858856930523352,
+ "tokens_seen": 614465536
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.00065760481444333,
+ "loss": 1.5097,
+ "theoretical_loss": 3.8588144826258426,
+ "tokens_seen": 614531072
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575887662988967,
+ "loss": 1.3983,
+ "theoretical_loss": 3.8587720405222545,
+ "tokens_seen": 614596608
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575727181544635,
+ "loss": 1.3622,
+ "theoretical_loss": 3.8587296042111796,
+ "tokens_seen": 614662144
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575566700100301,
+ "loss": 1.5156,
+ "theoretical_loss": 3.8586871736912087,
+ "tokens_seen": 614727680
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575406218655969,
+ "loss": 1.3586,
+ "theoretical_loss": 3.858644748960936,
+ "tokens_seen": 614793216
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575245737211635,
+ "loss": 1.5279,
+ "theoretical_loss": 3.858602330018953,
+ "tokens_seen": 614858752
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006575085255767303,
+ "loss": 1.5332,
+ "theoretical_loss": 3.858559916863853,
+ "tokens_seen": 614924288
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574924774322969,
+ "loss": 1.4379,
+ "theoretical_loss": 3.8585175094942317,
+ "tokens_seen": 614989824
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574764292878637,
+ "loss": 1.5811,
+ "theoretical_loss": 3.8584751079086814,
+ "tokens_seen": 615055360
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574603811434303,
+ "loss": 1.5058,
+ "theoretical_loss": 3.8584327121057984,
+ "tokens_seen": 615120896
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574443329989971,
+ "loss": 1.4936,
+ "theoretical_loss": 3.858390322084178,
+ "tokens_seen": 615186432
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574282848545637,
+ "loss": 1.4568,
+ "theoretical_loss": 3.8583479378424155,
+ "tokens_seen": 615251968
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006574122367101305,
+ "loss": 1.4072,
+ "theoretical_loss": 3.858305559379107,
+ "tokens_seen": 615317504
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573961885656971,
+ "loss": 1.5034,
+ "theoretical_loss": 3.8582631866928505,
+ "tokens_seen": 615383040
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573801404212639,
+ "loss": 1.4569,
+ "theoretical_loss": 3.858220819782243,
+ "tokens_seen": 615448576
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573640922768305,
+ "loss": 1.4396,
+ "theoretical_loss": 3.858178458645882,
+ "tokens_seen": 615514112
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573480441323973,
+ "loss": 1.4184,
+ "theoretical_loss": 3.858136103282366,
+ "tokens_seen": 615579648
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573319959879639,
+ "loss": 1.4304,
+ "theoretical_loss": 3.8580937536902935,
+ "tokens_seen": 615645184
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006573159478435307,
+ "loss": 1.4899,
+ "theoretical_loss": 3.8580514098682643,
+ "tokens_seen": 615710720
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572998996990973,
+ "loss": 1.5058,
+ "theoretical_loss": 3.8580090718148776,
+ "tokens_seen": 615776256
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572838515546641,
+ "loss": 1.3675,
+ "theoretical_loss": 3.8579667395287345,
+ "tokens_seen": 615841792
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572678034102307,
+ "loss": 1.5073,
+ "theoretical_loss": 3.8579244130084356,
+ "tokens_seen": 615907328
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572517552657975,
+ "loss": 1.3504,
+ "theoretical_loss": 3.8578820922525816,
+ "tokens_seen": 615972864
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 211365,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4504954814910889,
+ "objective/train/theoretical_loss": 3.857839777259775,
+ "objective/train/tokens_used": 636498400,
+ "theoretical_loss": 3.857839777259775,
+ "tokens_seen": 616038400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572357071213641,
+ "loss": 1.4745,
+ "theoretical_loss": 3.857839777259775,
+ "tokens_seen": 616038400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572196589769309,
+ "loss": 1.4646,
+ "theoretical_loss": 3.857797468028618,
+ "tokens_seen": 616103936
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006572036108324975,
+ "loss": 1.4291,
+ "theoretical_loss": 3.8577551645577124,
+ "tokens_seen": 616169472
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571875626880643,
+ "loss": 1.4413,
+ "theoretical_loss": 3.857712866845662,
+ "tokens_seen": 616235008
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571715145436309,
+ "loss": 1.417,
+ "theoretical_loss": 3.857670574891071,
+ "tokens_seen": 616300544
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571554663991977,
+ "loss": 1.469,
+ "theoretical_loss": 3.857628288692543,
+ "tokens_seen": 616366080
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571394182547643,
+ "loss": 1.4186,
+ "theoretical_loss": 3.857586008248683,
+ "tokens_seen": 616431616
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571233701103311,
+ "loss": 1.5126,
+ "theoretical_loss": 3.8575437335580958,
+ "tokens_seen": 616497152
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006571073219658977,
+ "loss": 1.418,
+ "theoretical_loss": 3.8575014646193875,
+ "tokens_seen": 616562688
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570912738214645,
+ "loss": 1.4638,
+ "theoretical_loss": 3.8574592014311637,
+ "tokens_seen": 616628224
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570752256770311,
+ "loss": 1.4218,
+ "theoretical_loss": 3.857416943992032,
+ "tokens_seen": 616693760
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570591775325979,
+ "loss": 1.5737,
+ "theoretical_loss": 3.8573746923005983,
+ "tokens_seen": 616759296
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570431293881645,
+ "loss": 1.5094,
+ "theoretical_loss": 3.857332446355471,
+ "tokens_seen": 616824832
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570270812437313,
+ "loss": 1.3835,
+ "theoretical_loss": 3.857290206155258,
+ "tokens_seen": 616890368
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006570110330992979,
+ "loss": 1.3138,
+ "theoretical_loss": 3.8572479716985675,
+ "tokens_seen": 616955904
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006569949849548647,
+ "loss": 1.416,
+ "theoretical_loss": 3.857205742984009,
+ "tokens_seen": 617021440
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006569789368104313,
+ "loss": 1.3857,
+ "theoretical_loss": 3.857163520010192,
+ "tokens_seen": 617086976
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656962888665998,
+ "loss": 1.4585,
+ "theoretical_loss": 3.8571213027757274,
+ "tokens_seen": 617152512
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006569468405215647,
+ "loss": 1.3681,
+ "theoretical_loss": 3.857079091279224,
+ "tokens_seen": 617218048
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006569307923771314,
+ "loss": 1.4211,
+ "theoretical_loss": 3.857036885519293,
+ "tokens_seen": 617283584
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006569147442326981,
+ "loss": 1.4748,
+ "theoretical_loss": 3.8569946854945467,
+ "tokens_seen": 617349120
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568986960882648,
+ "loss": 1.4468,
+ "theoretical_loss": 3.8569524912035975,
+ "tokens_seen": 617414656
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568826479438315,
+ "loss": 1.4331,
+ "theoretical_loss": 3.8569103026450566,
+ "tokens_seen": 617480192
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568665997993982,
+ "loss": 1.4306,
+ "theoretical_loss": 3.8568681198175376,
+ "tokens_seen": 617545728
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568505516549649,
+ "loss": 1.5051,
+ "theoretical_loss": 3.856825942719654,
+ "tokens_seen": 617611264
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568345035105316,
+ "loss": 1.4273,
+ "theoretical_loss": 3.856783771350019,
+ "tokens_seen": 617676800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006568184553660983,
+ "loss": 1.4189,
+ "theoretical_loss": 3.8567416057072474,
+ "tokens_seen": 617742336
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656802407221665,
+ "loss": 1.39,
+ "theoretical_loss": 3.8566994457899546,
+ "tokens_seen": 617807872
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567863590772317,
+ "loss": 1.4286,
+ "theoretical_loss": 3.8566572915967554,
+ "tokens_seen": 617873408
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567703109327984,
+ "loss": 1.449,
+ "theoretical_loss": 3.8566151431262656,
+ "tokens_seen": 617938944
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567542627883651,
+ "loss": 1.4484,
+ "theoretical_loss": 3.856573000377101,
+ "tokens_seen": 618004480
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567382146439318,
+ "loss": 1.4595,
+ "theoretical_loss": 3.8565308633478796,
+ "tokens_seen": 618070016
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567221664994985,
+ "loss": 1.543,
+ "theoretical_loss": 3.856488732037218,
+ "tokens_seen": 618135552
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006567061183550652,
+ "loss": 1.4479,
+ "theoretical_loss": 3.856446606443734,
+ "tokens_seen": 618201088
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006566900702106319,
+ "loss": 1.404,
+ "theoretical_loss": 3.8564044865660456,
+ "tokens_seen": 618266624
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006566740220661986,
+ "loss": 1.3884,
+ "theoretical_loss": 3.8563623724027716,
+ "tokens_seen": 618332160
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006566579739217653,
+ "loss": 1.5086,
+ "theoretical_loss": 3.856320263952531,
+ "tokens_seen": 618397696
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656641925777332,
+ "loss": 1.4269,
+ "theoretical_loss": 3.856278161213944,
+ "tokens_seen": 618463232
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006566258776328987,
+ "loss": 1.4405,
+ "theoretical_loss": 3.856236064185631,
+ "tokens_seen": 618528768
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006566098294884654,
+ "loss": 1.4679,
+ "theoretical_loss": 3.856193972866212,
+ "tokens_seen": 618594304
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565937813440321,
+ "loss": 1.4031,
+ "theoretical_loss": 3.8561518872543075,
+ "tokens_seen": 618659840
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565777331995988,
+ "loss": 1.3028,
+ "theoretical_loss": 3.85610980734854,
+ "tokens_seen": 618725376
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565616850551655,
+ "loss": 1.4489,
+ "theoretical_loss": 3.8560677331475315,
+ "tokens_seen": 618790912
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565456369107322,
+ "loss": 1.5921,
+ "theoretical_loss": 3.8560256646499047,
+ "tokens_seen": 618856448
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565295887662989,
+ "loss": 1.418,
+ "theoretical_loss": 3.855983601854282,
+ "tokens_seen": 618921984
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006565135406218656,
+ "loss": 1.5715,
+ "theoretical_loss": 3.8559415447592875,
+ "tokens_seen": 618987520
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564974924774323,
+ "loss": 1.4975,
+ "theoretical_loss": 3.8558994933635447,
+ "tokens_seen": 619053056
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656481444332999,
+ "loss": 1.4568,
+ "theoretical_loss": 3.855857447665678,
+ "tokens_seen": 619118592
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564653961885657,
+ "loss": 1.4231,
+ "theoretical_loss": 3.855815407664313,
+ "tokens_seen": 619184128
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564493480441324,
+ "loss": 1.4654,
+ "theoretical_loss": 3.8557733733580744,
+ "tokens_seen": 619249664
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 212287,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3307487964630127,
+ "objective/train/theoretical_loss": 3.855731344745589,
+ "objective/train/tokens_used": 639775200,
+ "theoretical_loss": 3.855731344745589,
+ "tokens_seen": 619315200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564332998996991,
+ "loss": 1.3697,
+ "theoretical_loss": 3.855731344745589,
+ "tokens_seen": 619315200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564172517552658,
+ "loss": 1.3096,
+ "theoretical_loss": 3.8556893218254817,
+ "tokens_seen": 619380736
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006564012036108325,
+ "loss": 1.4491,
+ "theoretical_loss": 3.855647304596381,
+ "tokens_seen": 619446272
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006563851554663992,
+ "loss": 1.5931,
+ "theoretical_loss": 3.8556052930569127,
+ "tokens_seen": 619511808
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656369107321966,
+ "loss": 1.4521,
+ "theoretical_loss": 3.8555632872057055,
+ "tokens_seen": 619577344
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006563530591775326,
+ "loss": 1.4756,
+ "theoretical_loss": 3.8555212870413875,
+ "tokens_seen": 619642880
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006563370110330993,
+ "loss": 1.3477,
+ "theoretical_loss": 3.8554792925625874,
+ "tokens_seen": 619708416
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656320962888666,
+ "loss": 1.4574,
+ "theoretical_loss": 3.855437303767934,
+ "tokens_seen": 619773952
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006563049147442327,
+ "loss": 1.4968,
+ "theoretical_loss": 3.855395320656058,
+ "tokens_seen": 619839488
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006562888665997994,
+ "loss": 1.4919,
+ "theoretical_loss": 3.8553533432255884,
+ "tokens_seen": 619905024
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006562728184553662,
+ "loss": 1.4488,
+ "theoretical_loss": 3.855311371475157,
+ "tokens_seen": 619970560
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006562567703109328,
+ "loss": 1.4551,
+ "theoretical_loss": 3.8552694054033942,
+ "tokens_seen": 620036096
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006562407221664996,
+ "loss": 1.5415,
+ "theoretical_loss": 3.855227445008931,
+ "tokens_seen": 620101632
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006562246740220662,
+ "loss": 1.4233,
+ "theoretical_loss": 3.8551854902904017,
+ "tokens_seen": 620167168
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656208625877633,
+ "loss": 1.44,
+ "theoretical_loss": 3.855143541246436,
+ "tokens_seen": 620232704
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006561925777331996,
+ "loss": 1.3843,
+ "theoretical_loss": 3.855101597875669,
+ "tokens_seen": 620298240
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006561765295887664,
+ "loss": 1.511,
+ "theoretical_loss": 3.8550596601767326,
+ "tokens_seen": 620363776
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656160481444333,
+ "loss": 1.4461,
+ "theoretical_loss": 3.8550177281482627,
+ "tokens_seen": 620429312
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006561444332998998,
+ "loss": 1.3921,
+ "theoretical_loss": 3.8549758017888918,
+ "tokens_seen": 620494848
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006561283851554664,
+ "loss": 1.3798,
+ "theoretical_loss": 3.8549338810972564,
+ "tokens_seen": 620560384
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006561123370110332,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8548919660719907,
+ "tokens_seen": 620625920
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560962888665998,
+ "loss": 1.333,
+ "theoretical_loss": 3.854850056711731,
+ "tokens_seen": 620691456
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560802407221666,
+ "loss": 1.5228,
+ "theoretical_loss": 3.8548081530151137,
+ "tokens_seen": 620756992
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560641925777332,
+ "loss": 1.5282,
+ "theoretical_loss": 3.8547662549807757,
+ "tokens_seen": 620822528
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560481444333,
+ "loss": 1.3841,
+ "theoretical_loss": 3.854724362607354,
+ "tokens_seen": 620888064
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560320962888666,
+ "loss": 1.4249,
+ "theoretical_loss": 3.854682475893486,
+ "tokens_seen": 620953600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006560160481444334,
+ "loss": 1.4984,
+ "theoretical_loss": 3.854640594837811,
+ "tokens_seen": 621019136
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000656,
+ "loss": 1.5547,
+ "theoretical_loss": 3.8545987194389664,
+ "tokens_seen": 621084672
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559839518555668,
+ "loss": 1.4241,
+ "theoretical_loss": 3.8545568496955926,
+ "tokens_seen": 621150208
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559679037111334,
+ "loss": 1.6195,
+ "theoretical_loss": 3.854514985606328,
+ "tokens_seen": 621215744
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559518555667002,
+ "loss": 1.3812,
+ "theoretical_loss": 3.854473127169814,
+ "tokens_seen": 621281280
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559358074222668,
+ "loss": 1.3515,
+ "theoretical_loss": 3.85443127438469,
+ "tokens_seen": 621346816
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559197592778336,
+ "loss": 1.4354,
+ "theoretical_loss": 3.8543894272495978,
+ "tokens_seen": 621412352
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006559037111334002,
+ "loss": 1.4532,
+ "theoretical_loss": 3.8543475857631786,
+ "tokens_seen": 621477888
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655887662988967,
+ "loss": 1.4251,
+ "theoretical_loss": 3.854305749924075,
+ "tokens_seen": 621543424
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006558716148445336,
+ "loss": 1.5207,
+ "theoretical_loss": 3.854263919730928,
+ "tokens_seen": 621608960
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006558555667001004,
+ "loss": 1.4856,
+ "theoretical_loss": 3.8542220951823816,
+ "tokens_seen": 621674496
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006558395185556671,
+ "loss": 1.5301,
+ "theoretical_loss": 3.854180276277079,
+ "tokens_seen": 621740032
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006558234704112338,
+ "loss": 1.4044,
+ "theoretical_loss": 3.8541384630136646,
+ "tokens_seen": 621805568
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006558074222668005,
+ "loss": 1.3659,
+ "theoretical_loss": 3.854096655390781,
+ "tokens_seen": 621871104
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006557913741223672,
+ "loss": 1.4789,
+ "theoretical_loss": 3.8540548534070753,
+ "tokens_seen": 621936640
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006557753259779339,
+ "loss": 1.3902,
+ "theoretical_loss": 3.854013057061191,
+ "tokens_seen": 622002176
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006557592778335006,
+ "loss": 1.5189,
+ "theoretical_loss": 3.853971266351775,
+ "tokens_seen": 622067712
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006557432296890673,
+ "loss": 1.4863,
+ "theoretical_loss": 3.853929481277472,
+ "tokens_seen": 622133248
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655727181544634,
+ "loss": 1.5079,
+ "theoretical_loss": 3.8538877018369306,
+ "tokens_seen": 622198784
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006557111334002006,
+ "loss": 1.387,
+ "theoretical_loss": 3.8538459280287967,
+ "tokens_seen": 622264320
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006556950852557674,
+ "loss": 1.4602,
+ "theoretical_loss": 3.853804159851718,
+ "tokens_seen": 622329856
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655679037111334,
+ "loss": 1.5986,
+ "theoretical_loss": 3.853762397304343,
+ "tokens_seen": 622395392
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006556629889669008,
+ "loss": 1.4307,
+ "theoretical_loss": 3.853720640385319,
+ "tokens_seen": 622460928
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006556469408224674,
+ "loss": 1.4352,
+ "theoretical_loss": 3.853678889093297,
+ "tokens_seen": 622526464
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 213453,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5051805973052979,
+ "objective/train/theoretical_loss": 3.8536371434269254,
+ "objective/train/tokens_used": 643052000,
+ "theoretical_loss": 3.8536371434269254,
+ "tokens_seen": 622592000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006556308926780342,
+ "loss": 1.5377,
+ "theoretical_loss": 3.8536371434269254,
+ "tokens_seen": 622592000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006556148445336008,
+ "loss": 1.4414,
+ "theoretical_loss": 3.853595403384854,
+ "tokens_seen": 622657536
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555987963891676,
+ "loss": 1.5108,
+ "theoretical_loss": 3.853553668965733,
+ "tokens_seen": 622723072
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555827482447342,
+ "loss": 1.486,
+ "theoretical_loss": 3.8535119401682136,
+ "tokens_seen": 622788608
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555667001003009,
+ "loss": 1.4433,
+ "theoretical_loss": 3.8534702169909476,
+ "tokens_seen": 622854144
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555506519558676,
+ "loss": 1.4521,
+ "theoretical_loss": 3.853428499432586,
+ "tokens_seen": 622919680
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555346038114343,
+ "loss": 1.4889,
+ "theoretical_loss": 3.8533867874917815,
+ "tokens_seen": 622985216
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655518555667001,
+ "loss": 1.395,
+ "theoretical_loss": 3.8533450811671877,
+ "tokens_seen": 623050752
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006555025075225677,
+ "loss": 1.3459,
+ "theoretical_loss": 3.8533033804574557,
+ "tokens_seen": 623116288
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554864593781344,
+ "loss": 1.3917,
+ "theoretical_loss": 3.8532616853612405,
+ "tokens_seen": 623181824
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554704112337011,
+ "loss": 1.5538,
+ "theoretical_loss": 3.853219995877196,
+ "tokens_seen": 623247360
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554543630892678,
+ "loss": 1.418,
+ "theoretical_loss": 3.853178312003977,
+ "tokens_seen": 623312896
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554383149448345,
+ "loss": 1.491,
+ "theoretical_loss": 3.853136633740239,
+ "tokens_seen": 623378432
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554222668004012,
+ "loss": 1.5011,
+ "theoretical_loss": 3.853094961084636,
+ "tokens_seen": 623443968
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006554062186559679,
+ "loss": 1.503,
+ "theoretical_loss": 3.853053294035825,
+ "tokens_seen": 623509504
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006553901705115346,
+ "loss": 1.4062,
+ "theoretical_loss": 3.8530116325924624,
+ "tokens_seen": 623575040
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006553741223671013,
+ "loss": 1.479,
+ "theoretical_loss": 3.8529699767532053,
+ "tokens_seen": 623640576
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655358074222668,
+ "loss": 1.406,
+ "theoretical_loss": 3.85292832651671,
+ "tokens_seen": 623706112
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006553420260782347,
+ "loss": 1.4656,
+ "theoretical_loss": 3.8528866818816354,
+ "tokens_seen": 623771648
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006553259779338014,
+ "loss": 1.4556,
+ "theoretical_loss": 3.8528450428466403,
+ "tokens_seen": 623837184
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006553099297893681,
+ "loss": 1.3951,
+ "theoretical_loss": 3.852803409410382,
+ "tokens_seen": 623902720
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552938816449348,
+ "loss": 1.6086,
+ "theoretical_loss": 3.85276178157152,
+ "tokens_seen": 623968256
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552778335005015,
+ "loss": 1.5793,
+ "theoretical_loss": 3.852720159328715,
+ "tokens_seen": 624033792
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552617853560682,
+ "loss": 1.5246,
+ "theoretical_loss": 3.852678542680626,
+ "tokens_seen": 624099328
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552457372116349,
+ "loss": 1.4312,
+ "theoretical_loss": 3.8526369316259137,
+ "tokens_seen": 624164864
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552296890672016,
+ "loss": 1.3875,
+ "theoretical_loss": 3.8525953261632404,
+ "tokens_seen": 624230400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006552136409227683,
+ "loss": 1.5285,
+ "theoretical_loss": 3.852553726291266,
+ "tokens_seen": 624295936
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655197592778335,
+ "loss": 1.4572,
+ "theoretical_loss": 3.8525121320086537,
+ "tokens_seen": 624361472
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551815446339017,
+ "loss": 1.4146,
+ "theoretical_loss": 3.8524705433140656,
+ "tokens_seen": 624427008
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551654964894684,
+ "loss": 1.4862,
+ "theoretical_loss": 3.8524289602061645,
+ "tokens_seen": 624492544
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551494483450351,
+ "loss": 1.4295,
+ "theoretical_loss": 3.8523873826836135,
+ "tokens_seen": 624558080
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551334002006018,
+ "loss": 1.3939,
+ "theoretical_loss": 3.8523458107450765,
+ "tokens_seen": 624623616
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551173520561685,
+ "loss": 1.5243,
+ "theoretical_loss": 3.8523042443892184,
+ "tokens_seen": 624689152
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006551013039117352,
+ "loss": 1.4286,
+ "theoretical_loss": 3.852262683614703,
+ "tokens_seen": 624754688
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006550852557673019,
+ "loss": 1.5194,
+ "theoretical_loss": 3.8522211284201964,
+ "tokens_seen": 624820224
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006550692076228686,
+ "loss": 1.4788,
+ "theoretical_loss": 3.852179578804364,
+ "tokens_seen": 624885760
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006550531594784353,
+ "loss": 1.404,
+ "theoretical_loss": 3.852138034765872,
+ "tokens_seen": 624951296
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000655037111334002,
+ "loss": 1.3126,
+ "theoretical_loss": 3.8520964963033864,
+ "tokens_seen": 625016832
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006550210631895687,
+ "loss": 1.3926,
+ "theoretical_loss": 3.8520549634155747,
+ "tokens_seen": 625082368
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006550050150451354,
+ "loss": 1.4594,
+ "theoretical_loss": 3.852013436101104,
+ "tokens_seen": 625147904
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549889669007021,
+ "loss": 1.5262,
+ "theoretical_loss": 3.851971914358643,
+ "tokens_seen": 625213440
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549729187562688,
+ "loss": 1.4859,
+ "theoretical_loss": 3.85193039818686,
+ "tokens_seen": 625278976
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549568706118355,
+ "loss": 1.4388,
+ "theoretical_loss": 3.8518888875844226,
+ "tokens_seen": 625344512
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549408224674022,
+ "loss": 1.4641,
+ "theoretical_loss": 3.8518473825500017,
+ "tokens_seen": 625410048
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549247743229689,
+ "loss": 1.4174,
+ "theoretical_loss": 3.8518058830822666,
+ "tokens_seen": 625475584
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006549087261785357,
+ "loss": 1.4281,
+ "theoretical_loss": 3.8517643891798867,
+ "tokens_seen": 625541120
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006548926780341023,
+ "loss": 1.4682,
+ "theoretical_loss": 3.8517229008415343,
+ "tokens_seen": 625606656
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000654876629889669,
+ "loss": 1.4707,
+ "theoretical_loss": 3.8516814180658794,
+ "tokens_seen": 625672192
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006548605817452357,
+ "loss": 1.5598,
+ "theoretical_loss": 3.8516399408515936,
+ "tokens_seen": 625737728
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006548445336008025,
+ "loss": 1.549,
+ "theoretical_loss": 3.8515984691973495,
+ "tokens_seen": 625803264
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 214373,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3765876293182373,
+ "objective/train/theoretical_loss": 3.85155700310182,
+ "objective/train/tokens_used": 646328800,
+ "theoretical_loss": 3.85155700310182,
+ "tokens_seen": 625868800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006548284854563691,
+ "loss": 1.4324,
+ "theoretical_loss": 3.85155700310182,
+ "tokens_seen": 625868800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006548124373119359,
+ "loss": 1.4596,
+ "theoretical_loss": 3.8515155425636767,
+ "tokens_seen": 625934336
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547963891675025,
+ "loss": 1.373,
+ "theoretical_loss": 3.851474087581594,
+ "tokens_seen": 625999872
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547803410230693,
+ "loss": 1.5331,
+ "theoretical_loss": 3.8514326381542454,
+ "tokens_seen": 626065408
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547642928786359,
+ "loss": 1.4168,
+ "theoretical_loss": 3.851391194280305,
+ "tokens_seen": 626130944
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547482447342027,
+ "loss": 1.4006,
+ "theoretical_loss": 3.8513497559584495,
+ "tokens_seen": 626196480
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547321965897693,
+ "loss": 1.4852,
+ "theoretical_loss": 3.851308323187352,
+ "tokens_seen": 626262016
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547161484453361,
+ "loss": 1.4906,
+ "theoretical_loss": 3.851266895965689,
+ "tokens_seen": 626327552
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006547001003009027,
+ "loss": 1.4673,
+ "theoretical_loss": 3.8512254742921357,
+ "tokens_seen": 626393088
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006546840521564695,
+ "loss": 1.4863,
+ "theoretical_loss": 3.8511840581653707,
+ "tokens_seen": 626458624
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006546680040120361,
+ "loss": 1.3973,
+ "theoretical_loss": 3.85114264758407,
+ "tokens_seen": 626524160
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006546519558676029,
+ "loss": 1.3883,
+ "theoretical_loss": 3.8511012425469104,
+ "tokens_seen": 626589696
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006546359077231696,
+ "loss": 1.4545,
+ "theoretical_loss": 3.851059843052571,
+ "tokens_seen": 626655232
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006546198595787363,
+ "loss": 1.3686,
+ "theoretical_loss": 3.85101844909973,
+ "tokens_seen": 626720768
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000654603811434303,
+ "loss": 1.4696,
+ "theoretical_loss": 3.8509770606870664,
+ "tokens_seen": 626786304
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545877632898697,
+ "loss": 1.4343,
+ "theoretical_loss": 3.8509356778132586,
+ "tokens_seen": 626851840
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545717151454364,
+ "loss": 1.442,
+ "theoretical_loss": 3.8508943004769876,
+ "tokens_seen": 626917376
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545556670010031,
+ "loss": 1.3759,
+ "theoretical_loss": 3.8508529286769324,
+ "tokens_seen": 626982912
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545396188565698,
+ "loss": 1.4723,
+ "theoretical_loss": 3.850811562411775,
+ "tokens_seen": 627048448
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545235707121365,
+ "loss": 1.3319,
+ "theoretical_loss": 3.8507702016801963,
+ "tokens_seen": 627113984
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006545075225677032,
+ "loss": 1.4292,
+ "theoretical_loss": 3.850728846480877,
+ "tokens_seen": 627179520
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006544914744232699,
+ "loss": 1.4316,
+ "theoretical_loss": 3.8506874968125,
+ "tokens_seen": 627245056
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006544754262788366,
+ "loss": 1.3812,
+ "theoretical_loss": 3.8506461526737477,
+ "tokens_seen": 627310592
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006544593781344033,
+ "loss": 1.4015,
+ "theoretical_loss": 3.8506048140633027,
+ "tokens_seen": 627376128
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.00065444332998997,
+ "loss": 1.4431,
+ "theoretical_loss": 3.850563480979849,
+ "tokens_seen": 627441664
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006544272818455367,
+ "loss": 1.4909,
+ "theoretical_loss": 3.8505221534220695,
+ "tokens_seen": 627507200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006544112337011034,
+ "loss": 1.395,
+ "theoretical_loss": 3.8504808313886496,
+ "tokens_seen": 627572736
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543951855566701,
+ "loss": 1.4019,
+ "theoretical_loss": 3.850439514878273,
+ "tokens_seen": 627638272
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543791374122368,
+ "loss": 1.5165,
+ "theoretical_loss": 3.8503982038896263,
+ "tokens_seen": 627703808
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543630892678035,
+ "loss": 1.3704,
+ "theoretical_loss": 3.850356898421394,
+ "tokens_seen": 627769344
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543470411233702,
+ "loss": 1.3597,
+ "theoretical_loss": 3.850315598472262,
+ "tokens_seen": 627834880
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543309929789369,
+ "loss": 1.4165,
+ "theoretical_loss": 3.8502743040409184,
+ "tokens_seen": 627900416
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006543149448345035,
+ "loss": 1.4233,
+ "theoretical_loss": 3.850233015126048,
+ "tokens_seen": 627965952
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542988966900703,
+ "loss": 1.3924,
+ "theoretical_loss": 3.8501917317263405,
+ "tokens_seen": 628031488
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542828485456369,
+ "loss": 1.4651,
+ "theoretical_loss": 3.8501504538404827,
+ "tokens_seen": 628097024
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542668004012037,
+ "loss": 1.4985,
+ "theoretical_loss": 3.850109181467163,
+ "tokens_seen": 628162560
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542507522567703,
+ "loss": 1.4034,
+ "theoretical_loss": 3.85006791460507,
+ "tokens_seen": 628228096
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542347041123371,
+ "loss": 1.4475,
+ "theoretical_loss": 3.850026653252894,
+ "tokens_seen": 628293632
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542186559679037,
+ "loss": 1.4138,
+ "theoretical_loss": 3.8499853974093234,
+ "tokens_seen": 628359168
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006542026078234705,
+ "loss": 1.3889,
+ "theoretical_loss": 3.8499441470730487,
+ "tokens_seen": 628424704
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541865596790371,
+ "loss": 1.3595,
+ "theoretical_loss": 3.849902902242761,
+ "tokens_seen": 628490240
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541705115346038,
+ "loss": 1.4252,
+ "theoretical_loss": 3.849861662917151,
+ "tokens_seen": 628555776
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541544633901705,
+ "loss": 1.5239,
+ "theoretical_loss": 3.8498204290949105,
+ "tokens_seen": 628621312
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541384152457372,
+ "loss": 1.3816,
+ "theoretical_loss": 3.849779200774731,
+ "tokens_seen": 628686848
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541223671013039,
+ "loss": 1.5023,
+ "theoretical_loss": 3.8497379779553054,
+ "tokens_seen": 628752384
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006541063189568706,
+ "loss": 1.518,
+ "theoretical_loss": 3.849696760635326,
+ "tokens_seen": 628817920
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006540902708124373,
+ "loss": 1.4259,
+ "theoretical_loss": 3.8496555488134865,
+ "tokens_seen": 628883456
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000654074222668004,
+ "loss": 1.534,
+ "theoretical_loss": 3.84961434248848,
+ "tokens_seen": 628948992
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006540581745235707,
+ "loss": 1.4677,
+ "theoretical_loss": 3.8495731416590018,
+ "tokens_seen": 629014528
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006540421263791374,
+ "loss": 1.4557,
+ "theoretical_loss": 3.8495319463237454,
+ "tokens_seen": 629080064
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 215608,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4194252490997314,
+ "objective/train/theoretical_loss": 3.8494907564814067,
+ "objective/train/tokens_used": 649605600,
+ "theoretical_loss": 3.8494907564814067,
+ "tokens_seen": 629145600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006540260782347041,
+ "loss": 1.4274,
+ "theoretical_loss": 3.8494907564814067,
+ "tokens_seen": 629145600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006540100300902708,
+ "loss": 1.487,
+ "theoretical_loss": 3.8494495721306805,
+ "tokens_seen": 629211136
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006539939819458375,
+ "loss": 1.5867,
+ "theoretical_loss": 3.849408393270264,
+ "tokens_seen": 629276672
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006539779338014042,
+ "loss": 1.3724,
+ "theoretical_loss": 3.849367219898852,
+ "tokens_seen": 629342208
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006539618856569709,
+ "loss": 1.4578,
+ "theoretical_loss": 3.849326052015142,
+ "tokens_seen": 629407744
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006539458375125376,
+ "loss": 1.5947,
+ "theoretical_loss": 3.8492848896178318,
+ "tokens_seen": 629473280
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006539297893681043,
+ "loss": 1.4659,
+ "theoretical_loss": 3.8492437327056184,
+ "tokens_seen": 629538816
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653913741223671,
+ "loss": 1.4527,
+ "theoretical_loss": 3.8492025812772006,
+ "tokens_seen": 629604352
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538976930792377,
+ "loss": 1.4258,
+ "theoretical_loss": 3.849161435331277,
+ "tokens_seen": 629669888
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538816449348044,
+ "loss": 1.429,
+ "theoretical_loss": 3.849120294866546,
+ "tokens_seen": 629735424
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538655967903711,
+ "loss": 1.4194,
+ "theoretical_loss": 3.8490791598817076,
+ "tokens_seen": 629800960
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538495486459378,
+ "loss": 1.5264,
+ "theoretical_loss": 3.8490380303754623,
+ "tokens_seen": 629866496
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538335005015045,
+ "loss": 1.3783,
+ "theoretical_loss": 3.8489969063465095,
+ "tokens_seen": 629932032
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538174523570712,
+ "loss": 1.4903,
+ "theoretical_loss": 3.8489557877935505,
+ "tokens_seen": 629997568
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006538014042126379,
+ "loss": 1.5286,
+ "theoretical_loss": 3.848914674715287,
+ "tokens_seen": 630063104
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006537853560682046,
+ "loss": 1.4062,
+ "theoretical_loss": 3.84887356711042,
+ "tokens_seen": 630128640
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006537693079237713,
+ "loss": 1.4923,
+ "theoretical_loss": 3.8488324649776517,
+ "tokens_seen": 630194176
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653753259779338,
+ "loss": 1.3488,
+ "theoretical_loss": 3.848791368315686,
+ "tokens_seen": 630259712
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006537372116349047,
+ "loss": 1.5031,
+ "theoretical_loss": 3.848750277123224,
+ "tokens_seen": 630325248
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006537211634904714,
+ "loss": 1.4806,
+ "theoretical_loss": 3.848709191398971,
+ "tokens_seen": 630390784
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006537051153460381,
+ "loss": 1.3267,
+ "theoretical_loss": 3.8486681111416297,
+ "tokens_seen": 630456320
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006536890672016048,
+ "loss": 1.4047,
+ "theoretical_loss": 3.848627036349905,
+ "tokens_seen": 630521856
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006536730190571715,
+ "loss": 1.4194,
+ "theoretical_loss": 3.848585967022502,
+ "tokens_seen": 630587392
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006536569709127382,
+ "loss": 1.408,
+ "theoretical_loss": 3.8485449031581256,
+ "tokens_seen": 630652928
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653640922768305,
+ "loss": 1.3323,
+ "theoretical_loss": 3.8485038447554816,
+ "tokens_seen": 630718464
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006536248746238716,
+ "loss": 1.5932,
+ "theoretical_loss": 3.848462791813276,
+ "tokens_seen": 630784000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006536088264794383,
+ "loss": 1.4501,
+ "theoretical_loss": 3.8484217443302153,
+ "tokens_seen": 630849536
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653592778335005,
+ "loss": 1.5799,
+ "theoretical_loss": 3.8483807023050076,
+ "tokens_seen": 630915072
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006535767301905717,
+ "loss": 1.4477,
+ "theoretical_loss": 3.848339665736359,
+ "tokens_seen": 630980608
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006535606820461384,
+ "loss": 1.3818,
+ "theoretical_loss": 3.8482986346229784,
+ "tokens_seen": 631046144
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006535446339017052,
+ "loss": 1.4074,
+ "theoretical_loss": 3.8482576089635736,
+ "tokens_seen": 631111680
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006535285857572718,
+ "loss": 1.5077,
+ "theoretical_loss": 3.848216588756854,
+ "tokens_seen": 631177216
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006535125376128386,
+ "loss": 1.4928,
+ "theoretical_loss": 3.8481755740015275,
+ "tokens_seen": 631242752
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534964894684052,
+ "loss": 1.4117,
+ "theoretical_loss": 3.848134564696305,
+ "tokens_seen": 631308288
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653480441323972,
+ "loss": 1.4229,
+ "theoretical_loss": 3.8480935608398967,
+ "tokens_seen": 631373824
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534643931795386,
+ "loss": 1.3704,
+ "theoretical_loss": 3.8480525624310125,
+ "tokens_seen": 631439360
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534483450351054,
+ "loss": 1.4856,
+ "theoretical_loss": 3.8480115694683636,
+ "tokens_seen": 631504896
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534322968906721,
+ "loss": 1.4555,
+ "theoretical_loss": 3.847970581950662,
+ "tokens_seen": 631570432
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534162487462388,
+ "loss": 1.5628,
+ "theoretical_loss": 3.8479295998766188,
+ "tokens_seen": 631635968
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006534002006018055,
+ "loss": 1.393,
+ "theoretical_loss": 3.8478886232449465,
+ "tokens_seen": 631701504
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006533841524573722,
+ "loss": 1.4871,
+ "theoretical_loss": 3.847847652054358,
+ "tokens_seen": 631767040
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006533681043129389,
+ "loss": 1.4766,
+ "theoretical_loss": 3.8478066863035663,
+ "tokens_seen": 631832576
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006533520561685056,
+ "loss": 1.4443,
+ "theoretical_loss": 3.8477657259912856,
+ "tokens_seen": 631898112
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006533360080240723,
+ "loss": 1.4973,
+ "theoretical_loss": 3.847724771116229,
+ "tokens_seen": 631963648
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653319959879639,
+ "loss": 1.4456,
+ "theoretical_loss": 3.847683821677112,
+ "tokens_seen": 632029184
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006533039117352057,
+ "loss": 1.4088,
+ "theoretical_loss": 3.847642877672649,
+ "tokens_seen": 632094720
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532878635907724,
+ "loss": 1.4612,
+ "theoretical_loss": 3.847601939101555,
+ "tokens_seen": 632160256
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532718154463391,
+ "loss": 1.5592,
+ "theoretical_loss": 3.8475610059625467,
+ "tokens_seen": 632225792
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532557673019058,
+ "loss": 1.484,
+ "theoretical_loss": 3.84752007825434,
+ "tokens_seen": 632291328
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532397191574725,
+ "loss": 1.384,
+ "theoretical_loss": 3.8474791559756514,
+ "tokens_seen": 632356864
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 216545,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.624903678894043,
+ "objective/train/theoretical_loss": 3.8474382391251982,
+ "objective/train/tokens_used": 652882400,
+ "theoretical_loss": 3.8474382391251982,
+ "tokens_seen": 632422400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532236710130392,
+ "loss": 1.5181,
+ "theoretical_loss": 3.8474382391251982,
+ "tokens_seen": 632422400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006532076228686059,
+ "loss": 1.3753,
+ "theoretical_loss": 3.8473973277016977,
+ "tokens_seen": 632487936
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006531915747241726,
+ "loss": 1.4691,
+ "theoretical_loss": 3.847356421703868,
+ "tokens_seen": 632553472
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006531755265797393,
+ "loss": 1.4561,
+ "theoretical_loss": 3.8473155211304277,
+ "tokens_seen": 632619008
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000653159478435306,
+ "loss": 1.507,
+ "theoretical_loss": 3.847274625980096,
+ "tokens_seen": 632684544
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006531434302908727,
+ "loss": 1.4212,
+ "theoretical_loss": 3.8472337362515914,
+ "tokens_seen": 632750080
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006531273821464394,
+ "loss": 1.4166,
+ "theoretical_loss": 3.8471928519436345,
+ "tokens_seen": 632815616
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006531113340020061,
+ "loss": 1.5466,
+ "theoretical_loss": 3.8471519730549444,
+ "tokens_seen": 632881152
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530952858575728,
+ "loss": 1.5033,
+ "theoretical_loss": 3.847111099584242,
+ "tokens_seen": 632946688
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530792377131395,
+ "loss": 1.443,
+ "theoretical_loss": 3.8470702315302496,
+ "tokens_seen": 633012224
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530631895687062,
+ "loss": 1.3511,
+ "theoretical_loss": 3.847029368891687,
+ "tokens_seen": 633077760
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530471414242729,
+ "loss": 1.4817,
+ "theoretical_loss": 3.846988511667277,
+ "tokens_seen": 633143296
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530310932798396,
+ "loss": 1.3648,
+ "theoretical_loss": 3.8469476598557417,
+ "tokens_seen": 633208832
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006530150451354063,
+ "loss": 1.353,
+ "theoretical_loss": 3.8469068134558038,
+ "tokens_seen": 633274368
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652998996990973,
+ "loss": 1.3605,
+ "theoretical_loss": 3.846865972466187,
+ "tokens_seen": 633339904
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529829488465397,
+ "loss": 1.4737,
+ "theoretical_loss": 3.846825136885615,
+ "tokens_seen": 633405440
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529669007021064,
+ "loss": 1.4733,
+ "theoretical_loss": 3.8467843067128102,
+ "tokens_seen": 633470976
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529508525576731,
+ "loss": 1.4118,
+ "theoretical_loss": 3.8467434819464996,
+ "tokens_seen": 633536512
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529348044132398,
+ "loss": 1.3678,
+ "theoretical_loss": 3.8467026625854066,
+ "tokens_seen": 633602048
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529187562688064,
+ "loss": 1.4033,
+ "theoretical_loss": 3.8466618486282567,
+ "tokens_seen": 633667584
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529187562688064,
+ "loss": 1.4708,
+ "theoretical_loss": 3.8466210400737757,
+ "tokens_seen": 633733120
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006529027081243732,
+ "loss": 1.4761,
+ "theoretical_loss": 3.8465802369206905,
+ "tokens_seen": 633798656
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006528866599799398,
+ "loss": 1.4281,
+ "theoretical_loss": 3.846539439167727,
+ "tokens_seen": 633864192
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006528706118355066,
+ "loss": 1.5256,
+ "theoretical_loss": 3.846498646813613,
+ "tokens_seen": 633929728
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006528545636910732,
+ "loss": 1.5862,
+ "theoretical_loss": 3.8464578598570753,
+ "tokens_seen": 633995264
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.00065283851554664,
+ "loss": 1.442,
+ "theoretical_loss": 3.8464170782968417,
+ "tokens_seen": 634060800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006528224674022066,
+ "loss": 1.5478,
+ "theoretical_loss": 3.8463763021316417,
+ "tokens_seen": 634126336
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006528064192577734,
+ "loss": 1.432,
+ "theoretical_loss": 3.8463355313602037,
+ "tokens_seen": 634191872
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.00065279037111334,
+ "loss": 1.4755,
+ "theoretical_loss": 3.8462947659812565,
+ "tokens_seen": 634257408
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006527743229689067,
+ "loss": 1.6248,
+ "theoretical_loss": 3.84625400599353,
+ "tokens_seen": 634322944
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006527582748244734,
+ "loss": 1.3956,
+ "theoretical_loss": 3.846213251395755,
+ "tokens_seen": 634388480
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006527422266800401,
+ "loss": 1.5417,
+ "theoretical_loss": 3.846172502186661,
+ "tokens_seen": 634454016
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006527261785356068,
+ "loss": 1.459,
+ "theoretical_loss": 3.8461317583649794,
+ "tokens_seen": 634519552
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006527101303911735,
+ "loss": 1.4051,
+ "theoretical_loss": 3.8460910199294416,
+ "tokens_seen": 634585088
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006526940822467402,
+ "loss": 1.447,
+ "theoretical_loss": 3.8460502868787794,
+ "tokens_seen": 634650624
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006526780341023069,
+ "loss": 1.4647,
+ "theoretical_loss": 3.8460095592117254,
+ "tokens_seen": 634716160
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006526619859578736,
+ "loss": 1.4076,
+ "theoretical_loss": 3.845968836927012,
+ "tokens_seen": 634781696
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006526459378134403,
+ "loss": 1.4685,
+ "theoretical_loss": 3.8459281200233724,
+ "tokens_seen": 634847232
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652629889669007,
+ "loss": 1.5148,
+ "theoretical_loss": 3.8458874084995402,
+ "tokens_seen": 634912768
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006526138415245737,
+ "loss": 1.3509,
+ "theoretical_loss": 3.8458467023542493,
+ "tokens_seen": 634978304
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525977933801404,
+ "loss": 1.4074,
+ "theoretical_loss": 3.845806001586234,
+ "tokens_seen": 635043840
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525817452357071,
+ "loss": 1.3696,
+ "theoretical_loss": 3.8457653061942296,
+ "tokens_seen": 635109376
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525656970912738,
+ "loss": 1.5072,
+ "theoretical_loss": 3.8457246161769705,
+ "tokens_seen": 635174912
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525496489468405,
+ "loss": 1.5341,
+ "theoretical_loss": 3.8456839315331934,
+ "tokens_seen": 635240448
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525336008024072,
+ "loss": 1.4519,
+ "theoretical_loss": 3.8456432522616337,
+ "tokens_seen": 635305984
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525175526579739,
+ "loss": 1.4432,
+ "theoretical_loss": 3.845602578361029,
+ "tokens_seen": 635371520
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006525015045135406,
+ "loss": 1.5209,
+ "theoretical_loss": 3.8455619098301144,
+ "tokens_seen": 635437056
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006524854563691073,
+ "loss": 1.3726,
+ "theoretical_loss": 3.8455212466676287,
+ "tokens_seen": 635502592
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652469408224674,
+ "loss": 1.4938,
+ "theoretical_loss": 3.8454805888723094,
+ "tokens_seen": 635568128
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006524533600802407,
+ "loss": 1.4569,
+ "theoretical_loss": 3.845439936442895,
+ "tokens_seen": 635633664
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 217757,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.47322678565979,
+ "objective/train/theoretical_loss": 3.8453992893781246,
+ "objective/train/tokens_used": 656159200,
+ "theoretical_loss": 3.8453992893781246,
+ "tokens_seen": 635699200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006524373119358074,
+ "loss": 1.4815,
+ "theoretical_loss": 3.8453992893781246,
+ "tokens_seen": 635699200
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006524212637913741,
+ "loss": 1.4739,
+ "theoretical_loss": 3.845358647676736,
+ "tokens_seen": 635764736
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006524052156469408,
+ "loss": 1.4726,
+ "theoretical_loss": 3.8453180113374694,
+ "tokens_seen": 635830272
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006523891675025075,
+ "loss": 1.434,
+ "theoretical_loss": 3.845277380359065,
+ "tokens_seen": 635895808
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006523731193580742,
+ "loss": 1.5007,
+ "theoretical_loss": 3.845236754740263,
+ "tokens_seen": 635961344
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006523570712136409,
+ "loss": 1.4894,
+ "theoretical_loss": 3.8451961344798042,
+ "tokens_seen": 636026880
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006523410230692076,
+ "loss": 1.3507,
+ "theoretical_loss": 3.84515551957643,
+ "tokens_seen": 636092416
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006523249749247743,
+ "loss": 1.4423,
+ "theoretical_loss": 3.845114910028882,
+ "tokens_seen": 636157952
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652308926780341,
+ "loss": 1.5087,
+ "theoretical_loss": 3.8450743058359023,
+ "tokens_seen": 636223488
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522928786359077,
+ "loss": 1.4043,
+ "theoretical_loss": 3.845033706996233,
+ "tokens_seen": 636289024
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522768304914744,
+ "loss": 1.3376,
+ "theoretical_loss": 3.8449931135086177,
+ "tokens_seen": 636354560
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522607823470411,
+ "loss": 1.4942,
+ "theoretical_loss": 3.8449525253717995,
+ "tokens_seen": 636420096
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522447342026078,
+ "loss": 1.354,
+ "theoretical_loss": 3.844911942584522,
+ "tokens_seen": 636485632
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522286860581746,
+ "loss": 1.4262,
+ "theoretical_loss": 3.84487136514553,
+ "tokens_seen": 636551168
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006522126379137412,
+ "loss": 1.5158,
+ "theoretical_loss": 3.8448307930535672,
+ "tokens_seen": 636616704
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652196589769308,
+ "loss": 1.5942,
+ "theoretical_loss": 3.8447902263073797,
+ "tokens_seen": 636682240
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006521805416248747,
+ "loss": 1.4652,
+ "theoretical_loss": 3.8447496649057125,
+ "tokens_seen": 636747776
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006521644934804414,
+ "loss": 1.5467,
+ "theoretical_loss": 3.844709108847311,
+ "tokens_seen": 636813312
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652148445336008,
+ "loss": 1.5092,
+ "theoretical_loss": 3.8446685581309223,
+ "tokens_seen": 636878848
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006521323971915748,
+ "loss": 1.3187,
+ "theoretical_loss": 3.844628012755293,
+ "tokens_seen": 636944384
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006521163490471415,
+ "loss": 1.4752,
+ "theoretical_loss": 3.8445874727191702,
+ "tokens_seen": 637009920
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006521003009027082,
+ "loss": 1.3897,
+ "theoretical_loss": 3.8445469380213013,
+ "tokens_seen": 637075456
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006520842527582749,
+ "loss": 1.5081,
+ "theoretical_loss": 3.844506408660435,
+ "tokens_seen": 637140992
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006520682046138416,
+ "loss": 1.519,
+ "theoretical_loss": 3.844465884635319,
+ "tokens_seen": 637206528
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006520521564694083,
+ "loss": 1.4463,
+ "theoretical_loss": 3.8444253659447023,
+ "tokens_seen": 637272064
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000652036108324975,
+ "loss": 1.4525,
+ "theoretical_loss": 3.8443848525873343,
+ "tokens_seen": 637337600
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006520200601805417,
+ "loss": 1.4409,
+ "theoretical_loss": 3.844344344561965,
+ "tokens_seen": 637403136
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006520040120361084,
+ "loss": 1.4772,
+ "theoretical_loss": 3.844303841867344,
+ "tokens_seen": 637468672
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519879638916751,
+ "loss": 1.4162,
+ "theoretical_loss": 3.8442633445022225,
+ "tokens_seen": 637534208
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519719157472418,
+ "loss": 1.399,
+ "theoretical_loss": 3.844222852465351,
+ "tokens_seen": 637599744
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519558676028085,
+ "loss": 1.4368,
+ "theoretical_loss": 3.844182365755481,
+ "tokens_seen": 637665280
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519398194583752,
+ "loss": 1.4406,
+ "theoretical_loss": 3.844141884371364,
+ "tokens_seen": 637730816
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519237713139419,
+ "loss": 1.5643,
+ "theoretical_loss": 3.8441014083117535,
+ "tokens_seen": 637796352
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006519077231695086,
+ "loss": 1.5508,
+ "theoretical_loss": 3.8440609375754007,
+ "tokens_seen": 637861888
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006518916750250753,
+ "loss": 1.3945,
+ "theoretical_loss": 3.844020472161059,
+ "tokens_seen": 637927424
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000651875626880642,
+ "loss": 1.3856,
+ "theoretical_loss": 3.8439800120674827,
+ "tokens_seen": 637992960
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006518595787362087,
+ "loss": 1.4326,
+ "theoretical_loss": 3.843939557293425,
+ "tokens_seen": 638058496
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006518435305917754,
+ "loss": 1.5456,
+ "theoretical_loss": 3.8438991078376405,
+ "tokens_seen": 638124032
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006518274824473421,
+ "loss": 1.4321,
+ "theoretical_loss": 3.843858663698884,
+ "tokens_seen": 638189568
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006518114343029088,
+ "loss": 1.3336,
+ "theoretical_loss": 3.84381822487591,
+ "tokens_seen": 638255104
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006517953861584755,
+ "loss": 1.3373,
+ "theoretical_loss": 3.843777791367475,
+ "tokens_seen": 638320640
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006517793380140422,
+ "loss": 1.4302,
+ "theoretical_loss": 3.843737363172335,
+ "tokens_seen": 638386176
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006517632898696089,
+ "loss": 1.4137,
+ "theoretical_loss": 3.843696940289246,
+ "tokens_seen": 638451712
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006517472417251756,
+ "loss": 1.396,
+ "theoretical_loss": 3.843656522716965,
+ "tokens_seen": 638517248
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006517311935807423,
+ "loss": 1.4537,
+ "theoretical_loss": 3.843616110454249,
+ "tokens_seen": 638582784
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000651715145436309,
+ "loss": 1.4281,
+ "theoretical_loss": 3.843575703499856,
+ "tokens_seen": 638648320
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516990972918757,
+ "loss": 1.3999,
+ "theoretical_loss": 3.843535301852544,
+ "tokens_seen": 638713856
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516830491474424,
+ "loss": 1.3964,
+ "theoretical_loss": 3.843494905511072,
+ "tokens_seen": 638779392
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516670010030091,
+ "loss": 1.3862,
+ "theoretical_loss": 3.8434545144741983,
+ "tokens_seen": 638844928
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516509528585758,
+ "loss": 1.5054,
+ "theoretical_loss": 3.8434141287406827,
+ "tokens_seen": 638910464
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 218641,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4308626651763916,
+ "objective/train/theoretical_loss": 3.8433737483092845,
+ "objective/train/tokens_used": 659436000,
+ "theoretical_loss": 3.8433737483092845,
+ "tokens_seen": 638976000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516349047141425,
+ "loss": 1.4943,
+ "theoretical_loss": 3.8433737483092845,
+ "tokens_seen": 638976000
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516188565697092,
+ "loss": 1.4438,
+ "theoretical_loss": 3.8433333731787647,
+ "tokens_seen": 639041536
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006516028084252759,
+ "loss": 1.5739,
+ "theoretical_loss": 3.843293003347883,
+ "tokens_seen": 639107072
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006515867602808426,
+ "loss": 1.3414,
+ "theoretical_loss": 3.843252638815401,
+ "tokens_seen": 639172608
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006515707121364093,
+ "loss": 1.4464,
+ "theoretical_loss": 3.8432122795800803,
+ "tokens_seen": 639238144
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000651554663991976,
+ "loss": 1.51,
+ "theoretical_loss": 3.843171925640682,
+ "tokens_seen": 639303680
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006515386158475427,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8431315769959697,
+ "tokens_seen": 639369216
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006515225677031094,
+ "loss": 1.4204,
+ "theoretical_loss": 3.8430912336447047,
+ "tokens_seen": 639434752
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006515065195586761,
+ "loss": 1.5045,
+ "theoretical_loss": 3.8430508955856504,
+ "tokens_seen": 639500288
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514904714142427,
+ "loss": 1.4237,
+ "theoretical_loss": 3.843010562817571,
+ "tokens_seen": 639565824
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514744232698095,
+ "loss": 1.4702,
+ "theoretical_loss": 3.84297023533923,
+ "tokens_seen": 639631360
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514583751253761,
+ "loss": 1.4988,
+ "theoretical_loss": 3.842929913149392,
+ "tokens_seen": 639696896
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514423269809429,
+ "loss": 1.4406,
+ "theoretical_loss": 3.8428895962468212,
+ "tokens_seen": 639762432
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514262788365095,
+ "loss": 1.4651,
+ "theoretical_loss": 3.842849284630284,
+ "tokens_seen": 639827968
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006514102306920763,
+ "loss": 1.4929,
+ "theoretical_loss": 3.8428089782985446,
+ "tokens_seen": 639893504
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006513941825476429,
+ "loss": 1.4309,
+ "theoretical_loss": 3.84276867725037,
+ "tokens_seen": 639959040
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006513781344032097,
+ "loss": 1.4411,
+ "theoretical_loss": 3.842728381484526,
+ "tokens_seen": 640024576
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006513620862587763,
+ "loss": 1.3425,
+ "theoretical_loss": 3.8426880909997796,
+ "tokens_seen": 640090112
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000651346038114343,
+ "loss": 1.5662,
+ "theoretical_loss": 3.8426478057948983,
+ "tokens_seen": 640155648
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006513299899699097,
+ "loss": 1.5147,
+ "theoretical_loss": 3.8426075258686496,
+ "tokens_seen": 640221184
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006513139418254764,
+ "loss": 1.4709,
+ "theoretical_loss": 3.842567251219802,
+ "tokens_seen": 640286720
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512978936810431,
+ "loss": 1.4913,
+ "theoretical_loss": 3.8425269818471235,
+ "tokens_seen": 640352256
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512818455366098,
+ "loss": 1.4533,
+ "theoretical_loss": 3.8424867177493836,
+ "tokens_seen": 640417792
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512657973921765,
+ "loss": 1.3171,
+ "theoretical_loss": 3.8424464589253504,
+ "tokens_seen": 640483328
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512497492477432,
+ "loss": 1.42,
+ "theoretical_loss": 3.842406205373795,
+ "tokens_seen": 640548864
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512337011033099,
+ "loss": 1.474,
+ "theoretical_loss": 3.8423659570934867,
+ "tokens_seen": 640614400
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512176529588766,
+ "loss": 1.3579,
+ "theoretical_loss": 3.842325714083197,
+ "tokens_seen": 640679936
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006512016048144433,
+ "loss": 1.4301,
+ "theoretical_loss": 3.842285476341696,
+ "tokens_seen": 640745472
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.00065118555667001,
+ "loss": 1.3874,
+ "theoretical_loss": 3.842245243867755,
+ "tokens_seen": 640811008
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006511695085255767,
+ "loss": 1.4085,
+ "theoretical_loss": 3.842205016660147,
+ "tokens_seen": 640876544
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006511534603811434,
+ "loss": 1.4344,
+ "theoretical_loss": 3.8421647947176423,
+ "tokens_seen": 640942080
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006511374122367101,
+ "loss": 1.438,
+ "theoretical_loss": 3.842124578039016,
+ "tokens_seen": 641007616
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006511213640922768,
+ "loss": 1.5147,
+ "theoretical_loss": 3.842084366623039,
+ "tokens_seen": 641073152
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006511053159478435,
+ "loss": 1.4703,
+ "theoretical_loss": 3.842044160468485,
+ "tokens_seen": 641138688
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510892678034102,
+ "loss": 1.4338,
+ "theoretical_loss": 3.8420039595741295,
+ "tokens_seen": 641204224
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510732196589769,
+ "loss": 1.3835,
+ "theoretical_loss": 3.841963763938746,
+ "tokens_seen": 641269760
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510571715145436,
+ "loss": 1.4765,
+ "theoretical_loss": 3.8419235735611075,
+ "tokens_seen": 641335296
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510411233701103,
+ "loss": 1.5916,
+ "theoretical_loss": 3.8418833884399914,
+ "tokens_seen": 641400832
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510250752256771,
+ "loss": 1.4546,
+ "theoretical_loss": 3.841843208574172,
+ "tokens_seen": 641466368
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006510090270812437,
+ "loss": 1.5065,
+ "theoretical_loss": 3.8418030339624254,
+ "tokens_seen": 641531904
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006509929789368105,
+ "loss": 1.4235,
+ "theoretical_loss": 3.8417628646035284,
+ "tokens_seen": 641597440
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006509769307923771,
+ "loss": 1.413,
+ "theoretical_loss": 3.8417227004962573,
+ "tokens_seen": 641662976
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006509608826479439,
+ "loss": 1.4879,
+ "theoretical_loss": 3.8416825416393894,
+ "tokens_seen": 641728512
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006509448345035105,
+ "loss": 1.531,
+ "theoretical_loss": 3.841642388031702,
+ "tokens_seen": 641794048
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006509287863590773,
+ "loss": 1.4144,
+ "theoretical_loss": 3.8416022396719733,
+ "tokens_seen": 641859584
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000650912738214644,
+ "loss": 1.4167,
+ "theoretical_loss": 3.841562096558982,
+ "tokens_seen": 641925120
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508966900702107,
+ "loss": 1.363,
+ "theoretical_loss": 3.841521958691506,
+ "tokens_seen": 641990656
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508806419257773,
+ "loss": 1.3944,
+ "theoretical_loss": 3.841481826068325,
+ "tokens_seen": 642056192
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508645937813441,
+ "loss": 1.3854,
+ "theoretical_loss": 3.841441698688219,
+ "tokens_seen": 642121728
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508485456369107,
+ "loss": 1.439,
+ "theoretical_loss": 3.8414015765499676,
+ "tokens_seen": 642187264
+ },
+ {
+ "epoch": 0.19,
+ "objective/train/docs_used": 219942,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3430824279785156,
+ "objective/train/theoretical_loss": 3.841361459652351,
+ "objective/train/tokens_used": 662712800,
+ "theoretical_loss": 3.841361459652351,
+ "tokens_seen": 642252800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508324974924775,
+ "loss": 1.3155,
+ "theoretical_loss": 3.841361459652351,
+ "tokens_seen": 642252800
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508164493480441,
+ "loss": 1.4824,
+ "theoretical_loss": 3.8413213479941506,
+ "tokens_seen": 642318336
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006508004012036109,
+ "loss": 1.3852,
+ "theoretical_loss": 3.841281241574147,
+ "tokens_seen": 642383872
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006507843530591776,
+ "loss": 1.5093,
+ "theoretical_loss": 3.841241140391123,
+ "tokens_seen": 642449408
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006507683049147443,
+ "loss": 1.4804,
+ "theoretical_loss": 3.8412010444438587,
+ "tokens_seen": 642514944
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000650752256770311,
+ "loss": 1.4636,
+ "theoretical_loss": 3.8411609537311384,
+ "tokens_seen": 642580480
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006507362086258777,
+ "loss": 1.3214,
+ "theoretical_loss": 3.841120868251744,
+ "tokens_seen": 642646016
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006507201604814444,
+ "loss": 1.4039,
+ "theoretical_loss": 3.8410807880044593,
+ "tokens_seen": 642711552
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006507041123370111,
+ "loss": 1.5107,
+ "theoretical_loss": 3.8410407129880673,
+ "tokens_seen": 642777088
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506880641925778,
+ "loss": 1.4625,
+ "theoretical_loss": 3.841000643201353,
+ "tokens_seen": 642842624
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506720160481445,
+ "loss": 1.4997,
+ "theoretical_loss": 3.8409605786431,
+ "tokens_seen": 642908160
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506559679037112,
+ "loss": 1.2775,
+ "theoretical_loss": 3.840920519312094,
+ "tokens_seen": 642973696
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506399197592779,
+ "loss": 1.4313,
+ "theoretical_loss": 3.8408804652071193,
+ "tokens_seen": 643039232
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506238716148446,
+ "loss": 1.4374,
+ "theoretical_loss": 3.8408404163269627,
+ "tokens_seen": 643104768
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006506078234704113,
+ "loss": 1.419,
+ "theoretical_loss": 3.84080037267041,
+ "tokens_seen": 643170304
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.000650591775325978,
+ "loss": 1.4437,
+ "theoretical_loss": 3.840760334236247,
+ "tokens_seen": 643235840
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006505757271815447,
+ "loss": 1.4948,
+ "theoretical_loss": 3.8407203010232616,
+ "tokens_seen": 643301376
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006505596790371114,
+ "loss": 1.4523,
+ "theoretical_loss": 3.840680273030241,
+ "tokens_seen": 643366912
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006505436308926781,
+ "loss": 1.5176,
+ "theoretical_loss": 3.840640250255972,
+ "tokens_seen": 643432448
+ },
+ {
+ "epoch": 0.19,
+ "learning_rate": 0.0006505275827482448,
+ "loss": 1.4363,
+ "theoretical_loss": 3.840600232699244,
+ "tokens_seen": 643497984
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006505115346038115,
+ "loss": 1.4636,
+ "theoretical_loss": 3.8405602203588445,
+ "tokens_seen": 643563520
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006504954864593782,
+ "loss": 1.4343,
+ "theoretical_loss": 3.8405202132335634,
+ "tokens_seen": 643629056
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006504794383149449,
+ "loss": 1.3896,
+ "theoretical_loss": 3.8404802113221894,
+ "tokens_seen": 643694592
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006504633901705116,
+ "loss": 1.3487,
+ "theoretical_loss": 3.8404402146235124,
+ "tokens_seen": 643760128
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006504473420260783,
+ "loss": 1.3639,
+ "theoretical_loss": 3.8404002231363226,
+ "tokens_seen": 643825664
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000650431293881645,
+ "loss": 1.398,
+ "theoretical_loss": 3.8403602368594107,
+ "tokens_seen": 643891200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006504152457372117,
+ "loss": 1.4373,
+ "theoretical_loss": 3.8403202557915677,
+ "tokens_seen": 643956736
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503991975927784,
+ "loss": 1.3552,
+ "theoretical_loss": 3.8402802799315845,
+ "tokens_seen": 644022272
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503831494483451,
+ "loss": 1.3018,
+ "theoretical_loss": 3.8402403092782533,
+ "tokens_seen": 644087808
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503671013039118,
+ "loss": 1.4913,
+ "theoretical_loss": 3.8402003438303662,
+ "tokens_seen": 644153344
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503510531594785,
+ "loss": 1.5308,
+ "theoretical_loss": 3.840160383586716,
+ "tokens_seen": 644218880
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503350050150452,
+ "loss": 1.5119,
+ "theoretical_loss": 3.8401204285460957,
+ "tokens_seen": 644284416
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503189568706119,
+ "loss": 1.3809,
+ "theoretical_loss": 3.840080478707298,
+ "tokens_seen": 644349952
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006503029087261786,
+ "loss": 1.4947,
+ "theoretical_loss": 3.840040534069118,
+ "tokens_seen": 644415488
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006502868605817453,
+ "loss": 1.4467,
+ "theoretical_loss": 3.8400005946303484,
+ "tokens_seen": 644481024
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000650270812437312,
+ "loss": 1.4591,
+ "theoretical_loss": 3.8399606603897847,
+ "tokens_seen": 644546560
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006502547642928787,
+ "loss": 1.4746,
+ "theoretical_loss": 3.8399207313462216,
+ "tokens_seen": 644612096
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006502387161484454,
+ "loss": 1.4329,
+ "theoretical_loss": 3.8398808074984547,
+ "tokens_seen": 644677632
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006502226680040121,
+ "loss": 1.3211,
+ "theoretical_loss": 3.8398408888452797,
+ "tokens_seen": 644743168
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006502066198595788,
+ "loss": 1.4165,
+ "theoretical_loss": 3.8398009753854927,
+ "tokens_seen": 644808704
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006501905717151455,
+ "loss": 1.3759,
+ "theoretical_loss": 3.839761067117891,
+ "tokens_seen": 644874240
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006501745235707122,
+ "loss": 1.4048,
+ "theoretical_loss": 3.8397211640412703,
+ "tokens_seen": 644939776
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006501584754262789,
+ "loss": 1.4058,
+ "theoretical_loss": 3.839681266154429,
+ "tokens_seen": 645005312
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006501424272818456,
+ "loss": 1.3807,
+ "theoretical_loss": 3.839641373456165,
+ "tokens_seen": 645070848
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006501263791374123,
+ "loss": 1.535,
+ "theoretical_loss": 3.8396014859452756,
+ "tokens_seen": 645136384
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000650110330992979,
+ "loss": 1.3981,
+ "theoretical_loss": 3.8395616036205604,
+ "tokens_seen": 645201920
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006500942828485456,
+ "loss": 1.52,
+ "theoretical_loss": 3.839521726480818,
+ "tokens_seen": 645267456
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006500782347041124,
+ "loss": 1.5095,
+ "theoretical_loss": 3.8394818545248475,
+ "tokens_seen": 645332992
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000650062186559679,
+ "loss": 1.4019,
+ "theoretical_loss": 3.8394419877514494,
+ "tokens_seen": 645398528
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006500461384152458,
+ "loss": 1.4695,
+ "theoretical_loss": 3.8394021261594236,
+ "tokens_seen": 645464064
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 220851,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.384150505065918,
+ "objective/train/theoretical_loss": 3.8393622697475704,
+ "objective/train/tokens_used": 665989600,
+ "theoretical_loss": 3.8393622697475704,
+ "tokens_seen": 645529600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006500300902708124,
+ "loss": 1.34,
+ "theoretical_loss": 3.8393622697475704,
+ "tokens_seen": 645529600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006500140421263792,
+ "loss": 1.4082,
+ "theoretical_loss": 3.839322418514691,
+ "tokens_seen": 645595136
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499979939819458,
+ "loss": 1.4156,
+ "theoretical_loss": 3.839282572459587,
+ "tokens_seen": 645660672
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499819458375126,
+ "loss": 1.4836,
+ "theoretical_loss": 3.83924273158106,
+ "tokens_seen": 645726208
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499658976930792,
+ "loss": 1.5027,
+ "theoretical_loss": 3.8392028958779125,
+ "tokens_seen": 645791744
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499498495486459,
+ "loss": 1.4867,
+ "theoretical_loss": 3.8391630653489472,
+ "tokens_seen": 645857280
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499338014042126,
+ "loss": 1.4544,
+ "theoretical_loss": 3.8391232399929667,
+ "tokens_seen": 645922816
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006499177532597793,
+ "loss": 1.4677,
+ "theoretical_loss": 3.839083419808774,
+ "tokens_seen": 645988352
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649901705115346,
+ "loss": 1.4005,
+ "theoretical_loss": 3.839043604795174,
+ "tokens_seen": 646053888
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498856569709127,
+ "loss": 1.3997,
+ "theoretical_loss": 3.83900379495097,
+ "tokens_seen": 646119424
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498696088264794,
+ "loss": 1.3995,
+ "theoretical_loss": 3.8389639902749675,
+ "tokens_seen": 646184960
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498535606820461,
+ "loss": 1.4303,
+ "theoretical_loss": 3.83892419076597,
+ "tokens_seen": 646250496
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498375125376128,
+ "loss": 1.4978,
+ "theoretical_loss": 3.8388843964227846,
+ "tokens_seen": 646316032
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498214643931796,
+ "loss": 1.473,
+ "theoretical_loss": 3.8388446072442157,
+ "tokens_seen": 646381568
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006498054162487462,
+ "loss": 1.5414,
+ "theoretical_loss": 3.8388048232290704,
+ "tokens_seen": 646447104
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649789368104313,
+ "loss": 1.4878,
+ "theoretical_loss": 3.8387650443761556,
+ "tokens_seen": 646512640
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006497733199598796,
+ "loss": 1.434,
+ "theoretical_loss": 3.838725270684277,
+ "tokens_seen": 646578176
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006497572718154464,
+ "loss": 1.5353,
+ "theoretical_loss": 3.8386855021522424,
+ "tokens_seen": 646643712
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649741223671013,
+ "loss": 1.3599,
+ "theoretical_loss": 3.83864573877886,
+ "tokens_seen": 646709248
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006497251755265798,
+ "loss": 1.4096,
+ "theoretical_loss": 3.8386059805629382,
+ "tokens_seen": 646774784
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006497091273821464,
+ "loss": 1.4407,
+ "theoretical_loss": 3.838566227503285,
+ "tokens_seen": 646840320
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006496930792377132,
+ "loss": 1.4622,
+ "theoretical_loss": 3.838526479598709,
+ "tokens_seen": 646905856
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006496770310932798,
+ "loss": 1.3577,
+ "theoretical_loss": 3.838486736848021,
+ "tokens_seen": 646971392
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006496609829488466,
+ "loss": 1.4514,
+ "theoretical_loss": 3.8384469992500296,
+ "tokens_seen": 647036928
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006496449348044132,
+ "loss": 1.4493,
+ "theoretical_loss": 3.8384072668035447,
+ "tokens_seen": 647102464
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064962888665998,
+ "loss": 1.5102,
+ "theoretical_loss": 3.8383675395073777,
+ "tokens_seen": 647168000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006496128385155466,
+ "loss": 1.3916,
+ "theoretical_loss": 3.838327817360339,
+ "tokens_seen": 647233536
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495967903711134,
+ "loss": 1.3993,
+ "theoretical_loss": 3.8382881003612406,
+ "tokens_seen": 647299072
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064958074222668,
+ "loss": 1.4586,
+ "theoretical_loss": 3.8382483885088927,
+ "tokens_seen": 647364608
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495646940822468,
+ "loss": 1.5188,
+ "theoretical_loss": 3.8382086818021097,
+ "tokens_seen": 647430144
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495486459378134,
+ "loss": 1.485,
+ "theoretical_loss": 3.838168980239702,
+ "tokens_seen": 647495680
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495325977933802,
+ "loss": 1.5177,
+ "theoretical_loss": 3.838129283820484,
+ "tokens_seen": 647561216
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495165496489468,
+ "loss": 1.4066,
+ "theoretical_loss": 3.838089592543268,
+ "tokens_seen": 647626752
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006495005015045136,
+ "loss": 1.3561,
+ "theoretical_loss": 3.8380499064068685,
+ "tokens_seen": 647692288
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006494844533600802,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8380102254100983,
+ "tokens_seen": 647757824
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649468405215647,
+ "loss": 1.3284,
+ "theoretical_loss": 3.8379705495517733,
+ "tokens_seen": 647823360
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006494523570712136,
+ "loss": 1.4504,
+ "theoretical_loss": 3.8379308788307083,
+ "tokens_seen": 647888896
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006494363089267804,
+ "loss": 1.3618,
+ "theoretical_loss": 3.837891213245718,
+ "tokens_seen": 647954432
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649420260782347,
+ "loss": 1.4628,
+ "theoretical_loss": 3.837851552795618,
+ "tokens_seen": 648019968
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006494042126379138,
+ "loss": 1.5271,
+ "theoretical_loss": 3.8378118974792246,
+ "tokens_seen": 648085504
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006493881644934805,
+ "loss": 1.4865,
+ "theoretical_loss": 3.837772247295354,
+ "tokens_seen": 648151040
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006493721163490472,
+ "loss": 1.5861,
+ "theoretical_loss": 3.8377326022428235,
+ "tokens_seen": 648216576
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006493560682046139,
+ "loss": 1.4137,
+ "theoretical_loss": 3.8376929623204505,
+ "tokens_seen": 648282112
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006493400200601806,
+ "loss": 1.4333,
+ "theoretical_loss": 3.8376533275270517,
+ "tokens_seen": 648347648
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006493239719157473,
+ "loss": 1.453,
+ "theoretical_loss": 3.837613697861446,
+ "tokens_seen": 648413184
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649307923771314,
+ "loss": 1.5266,
+ "theoretical_loss": 3.8375740733224513,
+ "tokens_seen": 648478720
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492918756268807,
+ "loss": 1.4287,
+ "theoretical_loss": 3.8375344539088867,
+ "tokens_seen": 648544256
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492758274824474,
+ "loss": 1.495,
+ "theoretical_loss": 3.837494839619571,
+ "tokens_seen": 648609792
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492597793380141,
+ "loss": 1.3091,
+ "theoretical_loss": 3.8374552304533243,
+ "tokens_seen": 648675328
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492437311935808,
+ "loss": 1.4309,
+ "theoretical_loss": 3.8374156264089665,
+ "tokens_seen": 648740864
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 222090,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4386017322540283,
+ "objective/train/theoretical_loss": 3.8373760274853175,
+ "objective/train/tokens_used": 669266400,
+ "theoretical_loss": 3.8373760274853175,
+ "tokens_seen": 648806400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492276830491475,
+ "loss": 1.4186,
+ "theoretical_loss": 3.8373760274853175,
+ "tokens_seen": 648806400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006492116349047142,
+ "loss": 1.5391,
+ "theoretical_loss": 3.8373364336811986,
+ "tokens_seen": 648871936
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006491955867602809,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8372968449954308,
+ "tokens_seen": 648937472
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006491795386158476,
+ "loss": 1.3458,
+ "theoretical_loss": 3.8372572614268354,
+ "tokens_seen": 649003008
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006491634904714143,
+ "loss": 1.4029,
+ "theoretical_loss": 3.8372176829742344,
+ "tokens_seen": 649068544
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000649147442326981,
+ "loss": 1.4743,
+ "theoretical_loss": 3.837178109636451,
+ "tokens_seen": 649134080
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006491313941825477,
+ "loss": 1.5013,
+ "theoretical_loss": 3.837138541412306,
+ "tokens_seen": 649199616
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006491153460381144,
+ "loss": 1.3991,
+ "theoretical_loss": 3.837098978300624,
+ "tokens_seen": 649265152
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490992978936811,
+ "loss": 1.533,
+ "theoretical_loss": 3.8370594203002284,
+ "tokens_seen": 649330688
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490832497492478,
+ "loss": 1.4891,
+ "theoretical_loss": 3.8370198674099427,
+ "tokens_seen": 649396224
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490672016048145,
+ "loss": 1.4534,
+ "theoretical_loss": 3.8369803196285917,
+ "tokens_seen": 649461760
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490511534603812,
+ "loss": 1.469,
+ "theoretical_loss": 3.8369407769549992,
+ "tokens_seen": 649527296
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490351053159479,
+ "loss": 1.3051,
+ "theoretical_loss": 3.8369012393879904,
+ "tokens_seen": 649592832
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490190571715146,
+ "loss": 1.443,
+ "theoretical_loss": 3.836861706926392,
+ "tokens_seen": 649658368
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006490030090270813,
+ "loss": 1.463,
+ "theoretical_loss": 3.836822179569028,
+ "tokens_seen": 649723904
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000648986960882648,
+ "loss": 1.3889,
+ "theoretical_loss": 3.8367826573147257,
+ "tokens_seen": 649789440
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006489709127382147,
+ "loss": 1.5045,
+ "theoretical_loss": 3.836743140162312,
+ "tokens_seen": 649854976
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006489548645937814,
+ "loss": 1.3668,
+ "theoretical_loss": 3.8367036281106124,
+ "tokens_seen": 649920512
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006489388164493481,
+ "loss": 1.4691,
+ "theoretical_loss": 3.836664121158456,
+ "tokens_seen": 649986048
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006489227683049148,
+ "loss": 1.4015,
+ "theoretical_loss": 3.83662461930467,
+ "tokens_seen": 650051584
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006489067201604815,
+ "loss": 1.473,
+ "theoretical_loss": 3.836585122548082,
+ "tokens_seen": 650117120
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006488906720160482,
+ "loss": 1.3815,
+ "theoretical_loss": 3.836545630887521,
+ "tokens_seen": 650182656
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006488746238716149,
+ "loss": 1.4541,
+ "theoretical_loss": 3.836506144321816,
+ "tokens_seen": 650248192
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006488585757271816,
+ "loss": 1.3895,
+ "theoretical_loss": 3.836466662849795,
+ "tokens_seen": 650313728
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006488425275827483,
+ "loss": 1.4039,
+ "theoretical_loss": 3.8364271864702904,
+ "tokens_seen": 650379264
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000648826479438315,
+ "loss": 1.4246,
+ "theoretical_loss": 3.8363877151821297,
+ "tokens_seen": 650444800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006488104312938817,
+ "loss": 1.4037,
+ "theoretical_loss": 3.8363482489841454,
+ "tokens_seen": 650510336
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487943831494484,
+ "loss": 1.4236,
+ "theoretical_loss": 3.8363087878751667,
+ "tokens_seen": 650575872
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487783350050151,
+ "loss": 1.4738,
+ "theoretical_loss": 3.8362693318540257,
+ "tokens_seen": 650641408
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487622868605818,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8362298809195545,
+ "tokens_seen": 650706944
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487462387161485,
+ "loss": 1.4915,
+ "theoretical_loss": 3.8361904350705833,
+ "tokens_seen": 650772480
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487301905717152,
+ "loss": 1.3687,
+ "theoretical_loss": 3.836150994305947,
+ "tokens_seen": 650838016
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006487141424272819,
+ "loss": 1.4345,
+ "theoretical_loss": 3.8361115586244763,
+ "tokens_seen": 650903552
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486980942828485,
+ "loss": 1.4897,
+ "theoretical_loss": 3.8360721280250054,
+ "tokens_seen": 650969088
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486820461384153,
+ "loss": 1.3686,
+ "theoretical_loss": 3.836032702506367,
+ "tokens_seen": 651034624
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486659979939819,
+ "loss": 1.4817,
+ "theoretical_loss": 3.8359932820673968,
+ "tokens_seen": 651100160
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486499498495487,
+ "loss": 1.5604,
+ "theoretical_loss": 3.8359538667069275,
+ "tokens_seen": 651165696
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486339017051153,
+ "loss": 1.4949,
+ "theoretical_loss": 3.8359144564237946,
+ "tokens_seen": 651231232
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486178535606821,
+ "loss": 1.4686,
+ "theoretical_loss": 3.8358750512168323,
+ "tokens_seen": 651296768
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006486018054162487,
+ "loss": 1.4227,
+ "theoretical_loss": 3.8358356510848775,
+ "tokens_seen": 651362304
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485857572718155,
+ "loss": 1.4537,
+ "theoretical_loss": 3.8357962560267644,
+ "tokens_seen": 651427840
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485697091273821,
+ "loss": 1.4057,
+ "theoretical_loss": 3.835756866041331,
+ "tokens_seen": 651493376
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485536609829489,
+ "loss": 1.3665,
+ "theoretical_loss": 3.8357174811274133,
+ "tokens_seen": 651558912
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485376128385155,
+ "loss": 1.4087,
+ "theoretical_loss": 3.8356781012838472,
+ "tokens_seen": 651624448
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485215646940823,
+ "loss": 1.4461,
+ "theoretical_loss": 3.8356387265094716,
+ "tokens_seen": 651689984
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006485055165496489,
+ "loss": 1.4452,
+ "theoretical_loss": 3.835599356803124,
+ "tokens_seen": 651755520
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484894684052157,
+ "loss": 1.3589,
+ "theoretical_loss": 3.835559992163642,
+ "tokens_seen": 651821056
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484734202607823,
+ "loss": 1.4137,
+ "theoretical_loss": 3.835520632589864,
+ "tokens_seen": 651886592
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484573721163491,
+ "loss": 1.4278,
+ "theoretical_loss": 3.83548127808063,
+ "tokens_seen": 651952128
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484413239719157,
+ "loss": 1.5026,
+ "theoretical_loss": 3.8354419286347783,
+ "tokens_seen": 652017664
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 222990,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3786110877990723,
+ "objective/train/theoretical_loss": 3.8354025842511494,
+ "objective/train/tokens_used": 672543200,
+ "theoretical_loss": 3.8354025842511494,
+ "tokens_seen": 652083200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484252758274825,
+ "loss": 1.5055,
+ "theoretical_loss": 3.8354025842511494,
+ "tokens_seen": 652083200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006484092276830491,
+ "loss": 1.4173,
+ "theoretical_loss": 3.8353632449285833,
+ "tokens_seen": 652148736
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483931795386159,
+ "loss": 1.3441,
+ "theoretical_loss": 3.8353239106659194,
+ "tokens_seen": 652214272
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483771313941825,
+ "loss": 1.5339,
+ "theoretical_loss": 3.835284581462,
+ "tokens_seen": 652279808
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483610832497493,
+ "loss": 1.5046,
+ "theoretical_loss": 3.8352452573156652,
+ "tokens_seen": 652345344
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483450351053159,
+ "loss": 1.5097,
+ "theoretical_loss": 3.835205938225757,
+ "tokens_seen": 652410880
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483289869608827,
+ "loss": 1.4368,
+ "theoretical_loss": 3.8351666241911175,
+ "tokens_seen": 652476416
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006483129388164493,
+ "loss": 1.4001,
+ "theoretical_loss": 3.8351273152105896,
+ "tokens_seen": 652541952
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482968906720161,
+ "loss": 1.4079,
+ "theoretical_loss": 3.8350880112830152,
+ "tokens_seen": 652607488
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482808425275827,
+ "loss": 1.4434,
+ "theoretical_loss": 3.8350487124072377,
+ "tokens_seen": 652673024
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482647943831495,
+ "loss": 1.5073,
+ "theoretical_loss": 3.835009418582101,
+ "tokens_seen": 652738560
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482487462387161,
+ "loss": 1.3897,
+ "theoretical_loss": 3.8349701298064485,
+ "tokens_seen": 652804096
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482326980942829,
+ "loss": 1.4405,
+ "theoretical_loss": 3.834930846079125,
+ "tokens_seen": 652869632
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482166499498495,
+ "loss": 1.4859,
+ "theoretical_loss": 3.8348915673989743,
+ "tokens_seen": 652935168
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006482006018054163,
+ "loss": 1.4749,
+ "theoretical_loss": 3.8348522937648424,
+ "tokens_seen": 653000704
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481845536609829,
+ "loss": 1.4313,
+ "theoretical_loss": 3.834813025175574,
+ "tokens_seen": 653066240
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481685055165497,
+ "loss": 1.4317,
+ "theoretical_loss": 3.8347737616300153,
+ "tokens_seen": 653131776
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481524573721163,
+ "loss": 1.4701,
+ "theoretical_loss": 3.8347345031270126,
+ "tokens_seen": 653197312
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481364092276831,
+ "loss": 1.4103,
+ "theoretical_loss": 3.834695249665412,
+ "tokens_seen": 653262848
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481203610832497,
+ "loss": 1.5055,
+ "theoretical_loss": 3.8346560012440607,
+ "tokens_seen": 653328384
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006481043129388165,
+ "loss": 1.3859,
+ "theoretical_loss": 3.8346167578618062,
+ "tokens_seen": 653393920
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006480882647943831,
+ "loss": 1.4491,
+ "theoretical_loss": 3.834577519517496,
+ "tokens_seen": 653459456
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006480722166499499,
+ "loss": 1.4329,
+ "theoretical_loss": 3.8345382862099786,
+ "tokens_seen": 653524992
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006480561685055166,
+ "loss": 1.4701,
+ "theoretical_loss": 3.8344990579381015,
+ "tokens_seen": 653590528
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006480401203610833,
+ "loss": 1.5133,
+ "theoretical_loss": 3.8344598347007146,
+ "tokens_seen": 653656064
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064802407221665,
+ "loss": 1.4859,
+ "theoretical_loss": 3.834420616496666,
+ "tokens_seen": 653721600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006480080240722167,
+ "loss": 1.4683,
+ "theoretical_loss": 3.8343814033248065,
+ "tokens_seen": 653787136
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479919759277834,
+ "loss": 1.4837,
+ "theoretical_loss": 3.8343421951839853,
+ "tokens_seen": 653852672
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479759277833501,
+ "loss": 1.46,
+ "theoretical_loss": 3.834302992073053,
+ "tokens_seen": 653918208
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479598796389168,
+ "loss": 1.3495,
+ "theoretical_loss": 3.8342637939908606,
+ "tokens_seen": 653983744
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479438314944835,
+ "loss": 1.4656,
+ "theoretical_loss": 3.8342246009362584,
+ "tokens_seen": 654049280
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479277833500502,
+ "loss": 1.4019,
+ "theoretical_loss": 3.8341854129080986,
+ "tokens_seen": 654114816
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006479117352056169,
+ "loss": 1.3941,
+ "theoretical_loss": 3.8341462299052327,
+ "tokens_seen": 654180352
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006478956870611836,
+ "loss": 1.4368,
+ "theoretical_loss": 3.8341070519265132,
+ "tokens_seen": 654245888
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006478796389167503,
+ "loss": 1.434,
+ "theoretical_loss": 3.8340678789707927,
+ "tokens_seen": 654311424
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647863590772317,
+ "loss": 1.2979,
+ "theoretical_loss": 3.834028711036924,
+ "tokens_seen": 654376960
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006478475426278837,
+ "loss": 1.4776,
+ "theoretical_loss": 3.8339895481237605,
+ "tokens_seen": 654442496
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006478314944834504,
+ "loss": 1.4851,
+ "theoretical_loss": 3.8339503902301564,
+ "tokens_seen": 654508032
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006478154463390171,
+ "loss": 1.4354,
+ "theoretical_loss": 3.833911237354965,
+ "tokens_seen": 654573568
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477993981945838,
+ "loss": 1.5246,
+ "theoretical_loss": 3.8338720894970413,
+ "tokens_seen": 654639104
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477833500501505,
+ "loss": 1.4338,
+ "theoretical_loss": 3.8338329466552405,
+ "tokens_seen": 654704640
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477673019057172,
+ "loss": 1.458,
+ "theoretical_loss": 3.833793808828417,
+ "tokens_seen": 654770176
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477512537612839,
+ "loss": 1.3989,
+ "theoretical_loss": 3.8337546760154275,
+ "tokens_seen": 654835712
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477352056168506,
+ "loss": 1.396,
+ "theoretical_loss": 3.8337155482151264,
+ "tokens_seen": 654901248
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006477191574724173,
+ "loss": 1.4821,
+ "theoretical_loss": 3.833676425426372,
+ "tokens_seen": 654966784
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647703109327984,
+ "loss": 1.3428,
+ "theoretical_loss": 3.8336373076480204,
+ "tokens_seen": 655032320
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476870611835507,
+ "loss": 1.2759,
+ "theoretical_loss": 3.8335981948789275,
+ "tokens_seen": 655097856
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476710130391174,
+ "loss": 1.3351,
+ "theoretical_loss": 3.833559087117952,
+ "tokens_seen": 655163392
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476549648946841,
+ "loss": 1.4337,
+ "theoretical_loss": 3.833519984363952,
+ "tokens_seen": 655228928
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476389167502508,
+ "loss": 1.521,
+ "theoretical_loss": 3.833480886615785,
+ "tokens_seen": 655294464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.6755372198767522,
+ "debugging/entropy-1-grams": 4.8915199782364125,
+ "debugging/length": 606.5,
+ "debugging/num_segments": 6,
+ "debugging/score": 0.054560030918185586,
+ "debugging/score_std": 0.025575581829586438,
+ "epoch": 0.2,
+ "objective/train/docs_used": 224121,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3385496139526367,
+ "objective/train/theoretical_loss": 3.83344179387231,
+ "objective/train/tokens_used": 675820000,
+ "theoretical_loss": 3.83344179387231,
+ "tokens_seen": 655360000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476228686058175,
+ "loss": 1.35,
+ "theoretical_loss": 3.83344179387231,
+ "tokens_seen": 655360000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006476068204613842,
+ "loss": 1.5245,
+ "theoretical_loss": 3.8334027061323863,
+ "tokens_seen": 655425536
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006475907723169509,
+ "loss": 1.4587,
+ "theoretical_loss": 3.8333636233948725,
+ "tokens_seen": 655491072
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006475747241725176,
+ "loss": 1.4165,
+ "theoretical_loss": 3.8333245456586287,
+ "tokens_seen": 655556608
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006475586760280843,
+ "loss": 1.5065,
+ "theoretical_loss": 3.8332854729225154,
+ "tokens_seen": 655622144
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647542627883651,
+ "loss": 1.5041,
+ "theoretical_loss": 3.8332464051853927,
+ "tokens_seen": 655687680
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006475265797392177,
+ "loss": 1.4827,
+ "theoretical_loss": 3.8332073424461215,
+ "tokens_seen": 655753216
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006475105315947844,
+ "loss": 1.4264,
+ "theoretical_loss": 3.8331682847035635,
+ "tokens_seen": 655818752
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474944834503511,
+ "loss": 1.3716,
+ "theoretical_loss": 3.8331292319565793,
+ "tokens_seen": 655884288
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474784353059178,
+ "loss": 1.4567,
+ "theoretical_loss": 3.833090184204032,
+ "tokens_seen": 655949824
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474623871614845,
+ "loss": 1.399,
+ "theoretical_loss": 3.833051141444783,
+ "tokens_seen": 656015360
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474463390170512,
+ "loss": 1.3809,
+ "theoretical_loss": 3.833012103677696,
+ "tokens_seen": 656080896
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474302908726179,
+ "loss": 1.3911,
+ "theoretical_loss": 3.8329730709016334,
+ "tokens_seen": 656146432
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006474142427281846,
+ "loss": 1.3959,
+ "theoretical_loss": 3.832934043115459,
+ "tokens_seen": 656211968
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473981945837513,
+ "loss": 1.4487,
+ "theoretical_loss": 3.8328950203180367,
+ "tokens_seen": 656277504
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647382146439318,
+ "loss": 1.439,
+ "theoretical_loss": 3.8328560025082306,
+ "tokens_seen": 656343040
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473660982948847,
+ "loss": 1.5272,
+ "theoretical_loss": 3.832816989684905,
+ "tokens_seen": 656408576
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473500501504514,
+ "loss": 1.4425,
+ "theoretical_loss": 3.8327779818469256,
+ "tokens_seen": 656474112
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473340020060181,
+ "loss": 1.5045,
+ "theoretical_loss": 3.8327389789931567,
+ "tokens_seen": 656539648
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473179538615848,
+ "loss": 1.4973,
+ "theoretical_loss": 3.8326999811224653,
+ "tokens_seen": 656605184
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006473019057171514,
+ "loss": 1.4604,
+ "theoretical_loss": 3.8326609882337164,
+ "tokens_seen": 656670720
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006472858575727182,
+ "loss": 1.4072,
+ "theoretical_loss": 3.8326220003257774,
+ "tokens_seen": 656736256
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006472698094282848,
+ "loss": 1.4249,
+ "theoretical_loss": 3.832583017397514,
+ "tokens_seen": 656801792
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006472537612838516,
+ "loss": 1.3675,
+ "theoretical_loss": 3.8325440394477943,
+ "tokens_seen": 656867328
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006472377131394182,
+ "loss": 1.4295,
+ "theoretical_loss": 3.8325050664754854,
+ "tokens_seen": 656932864
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647221664994985,
+ "loss": 1.461,
+ "theoretical_loss": 3.8324660984794554,
+ "tokens_seen": 656998400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006472056168505516,
+ "loss": 1.3973,
+ "theoretical_loss": 3.832427135458573,
+ "tokens_seen": 657063936
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006471895687061184,
+ "loss": 1.4038,
+ "theoretical_loss": 3.832388177411706,
+ "tokens_seen": 657129472
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647173520561685,
+ "loss": 1.4553,
+ "theoretical_loss": 3.832349224337724,
+ "tokens_seen": 657195008
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006471574724172518,
+ "loss": 1.5236,
+ "theoretical_loss": 3.832310276235497,
+ "tokens_seen": 657260544
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006471414242728184,
+ "loss": 1.3477,
+ "theoretical_loss": 3.8322713331038933,
+ "tokens_seen": 657326080
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006471253761283852,
+ "loss": 1.417,
+ "theoretical_loss": 3.8322323949417845,
+ "tokens_seen": 657391616
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006471093279839518,
+ "loss": 1.4558,
+ "theoretical_loss": 3.8321934617480404,
+ "tokens_seen": 657457152
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006470932798395186,
+ "loss": 1.4935,
+ "theoretical_loss": 3.832154533521532,
+ "tokens_seen": 657522688
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006470772316950852,
+ "loss": 1.4223,
+ "theoretical_loss": 3.8321156102611305,
+ "tokens_seen": 657588224
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647061183550652,
+ "loss": 1.4744,
+ "theoretical_loss": 3.832076691965708,
+ "tokens_seen": 657653760
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006470451354062186,
+ "loss": 1.4337,
+ "theoretical_loss": 3.832037778634136,
+ "tokens_seen": 657719296
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006470290872617854,
+ "loss": 1.3169,
+ "theoretical_loss": 3.831998870265287,
+ "tokens_seen": 657784832
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000647013039117352,
+ "loss": 1.3151,
+ "theoretical_loss": 3.831959966858034,
+ "tokens_seen": 657850368
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469969909729188,
+ "loss": 1.4623,
+ "theoretical_loss": 3.8319210684112495,
+ "tokens_seen": 657915904
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469809428284854,
+ "loss": 1.6412,
+ "theoretical_loss": 3.8318821749238077,
+ "tokens_seen": 657981440
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469648946840522,
+ "loss": 1.4437,
+ "theoretical_loss": 3.831843286394582,
+ "tokens_seen": 658046976
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469488465396188,
+ "loss": 1.388,
+ "theoretical_loss": 3.8318044028224465,
+ "tokens_seen": 658112512
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469327983951856,
+ "loss": 1.4706,
+ "theoretical_loss": 3.831765524206276,
+ "tokens_seen": 658178048
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006469167502507522,
+ "loss": 1.351,
+ "theoretical_loss": 3.8317266505449457,
+ "tokens_seen": 658243584
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646900702106319,
+ "loss": 1.4297,
+ "theoretical_loss": 3.8316877818373305,
+ "tokens_seen": 658309120
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006468846539618856,
+ "loss": 1.4541,
+ "theoretical_loss": 3.8316489180823066,
+ "tokens_seen": 658374656
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006468686058174524,
+ "loss": 1.4291,
+ "theoretical_loss": 3.8316100592787494,
+ "tokens_seen": 658440192
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646852557673019,
+ "loss": 1.3943,
+ "theoretical_loss": 3.8315712054255355,
+ "tokens_seen": 658505728
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006468365095285858,
+ "loss": 1.5116,
+ "theoretical_loss": 3.8315323565215422,
+ "tokens_seen": 658571264
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 224939,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3036456108093262,
+ "objective/train/theoretical_loss": 3.831493512565646,
+ "objective/train/tokens_used": 679096800,
+ "theoretical_loss": 3.831493512565646,
+ "tokens_seen": 658636800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006468204613841524,
+ "loss": 1.441,
+ "theoretical_loss": 3.831493512565646,
+ "tokens_seen": 658636800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006468044132397192,
+ "loss": 1.3758,
+ "theoretical_loss": 3.831454673556725,
+ "tokens_seen": 658702336
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006467883650952858,
+ "loss": 1.5545,
+ "theoretical_loss": 3.8314158394936557,
+ "tokens_seen": 658767872
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006467723169508526,
+ "loss": 1.4414,
+ "theoretical_loss": 3.831377010375318,
+ "tokens_seen": 658833408
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006467562688064192,
+ "loss": 1.2908,
+ "theoretical_loss": 3.8313381862005906,
+ "tokens_seen": 658898944
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646740220661986,
+ "loss": 1.5868,
+ "theoretical_loss": 3.8312993669683513,
+ "tokens_seen": 658964480
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006467241725175526,
+ "loss": 1.486,
+ "theoretical_loss": 3.83126055267748,
+ "tokens_seen": 659030016
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006467081243731194,
+ "loss": 1.4725,
+ "theoretical_loss": 3.8312217433268563,
+ "tokens_seen": 659095552
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646692076228686,
+ "loss": 1.3783,
+ "theoretical_loss": 3.8311829389153607,
+ "tokens_seen": 659161088
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006466760280842528,
+ "loss": 1.5176,
+ "theoretical_loss": 3.8311441394418733,
+ "tokens_seen": 659226624
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006466599799398195,
+ "loss": 1.4197,
+ "theoretical_loss": 3.831105344905274,
+ "tokens_seen": 659292160
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006466439317953862,
+ "loss": 1.5111,
+ "theoretical_loss": 3.831066555304446,
+ "tokens_seen": 659357696
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006466278836509529,
+ "loss": 1.3525,
+ "theoretical_loss": 3.8310277706382694,
+ "tokens_seen": 659423232
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006466118355065196,
+ "loss": 1.4952,
+ "theoretical_loss": 3.830988990905626,
+ "tokens_seen": 659488768
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006465957873620863,
+ "loss": 1.4209,
+ "theoretical_loss": 3.830950216105399,
+ "tokens_seen": 659554304
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646579739217653,
+ "loss": 1.3104,
+ "theoretical_loss": 3.83091144623647,
+ "tokens_seen": 659619840
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006465636910732197,
+ "loss": 1.3745,
+ "theoretical_loss": 3.830872681297723,
+ "tokens_seen": 659685376
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006465476429287864,
+ "loss": 1.5045,
+ "theoretical_loss": 3.8308339212880407,
+ "tokens_seen": 659750912
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006465315947843531,
+ "loss": 1.6476,
+ "theoretical_loss": 3.830795166206307,
+ "tokens_seen": 659816448
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006465155466399198,
+ "loss": 1.4087,
+ "theoretical_loss": 3.830756416051406,
+ "tokens_seen": 659881984
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464994984954865,
+ "loss": 1.4844,
+ "theoretical_loss": 3.8307176708222217,
+ "tokens_seen": 659947520
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464834503510532,
+ "loss": 1.426,
+ "theoretical_loss": 3.8306789305176396,
+ "tokens_seen": 660013056
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464674022066199,
+ "loss": 1.395,
+ "theoretical_loss": 3.830640195136545,
+ "tokens_seen": 660078592
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464513540621866,
+ "loss": 1.3413,
+ "theoretical_loss": 3.830601464677823,
+ "tokens_seen": 660144128
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464353059177533,
+ "loss": 1.4125,
+ "theoretical_loss": 3.8305627391403596,
+ "tokens_seen": 660209664
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064641925777332,
+ "loss": 1.6172,
+ "theoretical_loss": 3.830524018523041,
+ "tokens_seen": 660275200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006464032096288867,
+ "loss": 1.4359,
+ "theoretical_loss": 3.8304853028247536,
+ "tokens_seen": 660340736
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463871614844534,
+ "loss": 1.4082,
+ "theoretical_loss": 3.830446592044385,
+ "tokens_seen": 660406272
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463711133400201,
+ "loss": 1.3422,
+ "theoretical_loss": 3.830407886180822,
+ "tokens_seen": 660471808
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463550651955868,
+ "loss": 1.4599,
+ "theoretical_loss": 3.8303691852329522,
+ "tokens_seen": 660537344
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463390170511535,
+ "loss": 1.451,
+ "theoretical_loss": 3.830330489199665,
+ "tokens_seen": 660602880
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463229689067202,
+ "loss": 1.4021,
+ "theoretical_loss": 3.8302917980798465,
+ "tokens_seen": 660668416
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006463069207622869,
+ "loss": 1.308,
+ "theoretical_loss": 3.8302531118723877,
+ "tokens_seen": 660733952
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006462908726178536,
+ "loss": 1.3081,
+ "theoretical_loss": 3.830214430576177,
+ "tokens_seen": 660799488
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006462748244734203,
+ "loss": 1.4741,
+ "theoretical_loss": 3.8301757541901034,
+ "tokens_seen": 660865024
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646258776328987,
+ "loss": 1.5243,
+ "theoretical_loss": 3.8301370827130574,
+ "tokens_seen": 660930560
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006462427281845538,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8300984161439295,
+ "tokens_seen": 660996096
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006462266800401204,
+ "loss": 1.3741,
+ "theoretical_loss": 3.830059754481609,
+ "tokens_seen": 661061632
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006462106318956872,
+ "loss": 1.5008,
+ "theoretical_loss": 3.8300210977249884,
+ "tokens_seen": 661127168
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006461945837512538,
+ "loss": 1.4381,
+ "theoretical_loss": 3.8299824458729583,
+ "tokens_seen": 661192704
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006461785356068206,
+ "loss": 1.3359,
+ "theoretical_loss": 3.8299437989244103,
+ "tokens_seen": 661258240
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006461624874623872,
+ "loss": 1.3917,
+ "theoretical_loss": 3.8299051568782363,
+ "tokens_seen": 661323776
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646146439317954,
+ "loss": 1.3676,
+ "theoretical_loss": 3.8298665197333293,
+ "tokens_seen": 661389312
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006461303911735206,
+ "loss": 1.4281,
+ "theoretical_loss": 3.829827887488582,
+ "tokens_seen": 661454848
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006461143430290874,
+ "loss": 1.4768,
+ "theoretical_loss": 3.8297892601428867,
+ "tokens_seen": 661520384
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000646098294884654,
+ "loss": 1.4123,
+ "theoretical_loss": 3.8297506376951382,
+ "tokens_seen": 661585920
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460822467402208,
+ "loss": 1.5561,
+ "theoretical_loss": 3.829712020144229,
+ "tokens_seen": 661651456
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460661985957874,
+ "loss": 1.4405,
+ "theoretical_loss": 3.8296734074890546,
+ "tokens_seen": 661716992
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460501504513542,
+ "loss": 1.5015,
+ "theoretical_loss": 3.8296347997285083,
+ "tokens_seen": 661782528
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460341023069208,
+ "loss": 1.3775,
+ "theoretical_loss": 3.829596196861486,
+ "tokens_seen": 661848064
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 226118,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4302797317504883,
+ "objective/train/theoretical_loss": 3.8295575988868826,
+ "objective/train/tokens_used": 682373600,
+ "theoretical_loss": 3.8295575988868826,
+ "tokens_seen": 661913600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460180541624875,
+ "loss": 1.4939,
+ "theoretical_loss": 3.8295575988868826,
+ "tokens_seen": 661913600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006460020060180542,
+ "loss": 1.3983,
+ "theoretical_loss": 3.8295190058035935,
+ "tokens_seen": 661979136
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006459859578736209,
+ "loss": 1.4613,
+ "theoretical_loss": 3.8294804176105153,
+ "tokens_seen": 662044672
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006459699097291876,
+ "loss": 1.4311,
+ "theoretical_loss": 3.829441834306544,
+ "tokens_seen": 662110208
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006459538615847543,
+ "loss": 1.5059,
+ "theoretical_loss": 3.8294032558905764,
+ "tokens_seen": 662175744
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000645937813440321,
+ "loss": 1.5588,
+ "theoretical_loss": 3.829364682361509,
+ "tokens_seen": 662241280
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006459217652958877,
+ "loss": 1.427,
+ "theoretical_loss": 3.82932611371824,
+ "tokens_seen": 662306816
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006459057171514543,
+ "loss": 1.3958,
+ "theoretical_loss": 3.829287549959667,
+ "tokens_seen": 662372352
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458896690070211,
+ "loss": 1.5076,
+ "theoretical_loss": 3.8292489910846887,
+ "tokens_seen": 662437888
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458736208625877,
+ "loss": 1.5154,
+ "theoretical_loss": 3.829210437092202,
+ "tokens_seen": 662503424
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458575727181545,
+ "loss": 1.4082,
+ "theoretical_loss": 3.8291718879811074,
+ "tokens_seen": 662568960
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458415245737211,
+ "loss": 1.3067,
+ "theoretical_loss": 3.8291333437503035,
+ "tokens_seen": 662634496
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458254764292879,
+ "loss": 1.4807,
+ "theoretical_loss": 3.82909480439869,
+ "tokens_seen": 662700032
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006458094282848545,
+ "loss": 1.3389,
+ "theoretical_loss": 3.8290562699251662,
+ "tokens_seen": 662765568
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457933801404213,
+ "loss": 1.4133,
+ "theoretical_loss": 3.8290177403286334,
+ "tokens_seen": 662831104
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457773319959879,
+ "loss": 1.4489,
+ "theoretical_loss": 3.828979215607991,
+ "tokens_seen": 662896640
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457612838515547,
+ "loss": 1.3352,
+ "theoretical_loss": 3.8289406957621415,
+ "tokens_seen": 662962176
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457452357071213,
+ "loss": 1.3634,
+ "theoretical_loss": 3.828902180789985,
+ "tokens_seen": 663027712
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457291875626881,
+ "loss": 1.4564,
+ "theoretical_loss": 3.828863670690424,
+ "tokens_seen": 663093248
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006457131394182547,
+ "loss": 1.46,
+ "theoretical_loss": 3.8288251654623604,
+ "tokens_seen": 663158784
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456970912738215,
+ "loss": 1.4153,
+ "theoretical_loss": 3.8287866651046962,
+ "tokens_seen": 663224320
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456810431293881,
+ "loss": 1.4752,
+ "theoretical_loss": 3.8287481696163344,
+ "tokens_seen": 663289856
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456649949849549,
+ "loss": 1.4587,
+ "theoretical_loss": 3.828709678996179,
+ "tokens_seen": 663355392
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456489468405215,
+ "loss": 1.4478,
+ "theoretical_loss": 3.8286711932431325,
+ "tokens_seen": 663420928
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456328986960883,
+ "loss": 1.404,
+ "theoretical_loss": 3.8286327123560984,
+ "tokens_seen": 663486464
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456168505516549,
+ "loss": 1.4454,
+ "theoretical_loss": 3.8285942363339824,
+ "tokens_seen": 663552000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006456008024072217,
+ "loss": 1.4571,
+ "theoretical_loss": 3.828555765175688,
+ "tokens_seen": 663617536
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455847542627883,
+ "loss": 1.4549,
+ "theoretical_loss": 3.8285172988801195,
+ "tokens_seen": 663683072
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455687061183551,
+ "loss": 1.3885,
+ "theoretical_loss": 3.8284788374461836,
+ "tokens_seen": 663748608
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455526579739217,
+ "loss": 1.4964,
+ "theoretical_loss": 3.8284403808727854,
+ "tokens_seen": 663814144
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455366098294885,
+ "loss": 1.435,
+ "theoretical_loss": 3.8284019291588303,
+ "tokens_seen": 663879680
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455205616850551,
+ "loss": 1.3356,
+ "theoretical_loss": 3.8283634823032253,
+ "tokens_seen": 663945216
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006455045135406219,
+ "loss": 1.442,
+ "theoretical_loss": 3.828325040304877,
+ "tokens_seen": 664010752
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454884653961885,
+ "loss": 1.4236,
+ "theoretical_loss": 3.8282866031626925,
+ "tokens_seen": 664076288
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454724172517553,
+ "loss": 1.4537,
+ "theoretical_loss": 3.828248170875579,
+ "tokens_seen": 664141824
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454563691073219,
+ "loss": 1.3599,
+ "theoretical_loss": 3.828209743442444,
+ "tokens_seen": 664207360
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454403209628887,
+ "loss": 1.4581,
+ "theoretical_loss": 3.828171320862196,
+ "tokens_seen": 664272896
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454242728184553,
+ "loss": 1.4065,
+ "theoretical_loss": 3.8281329031337434,
+ "tokens_seen": 664338432
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006454082246740221,
+ "loss": 1.478,
+ "theoretical_loss": 3.828094490255995,
+ "tokens_seen": 664403968
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453921765295887,
+ "loss": 1.4048,
+ "theoretical_loss": 3.8280560822278598,
+ "tokens_seen": 664469504
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453761283851555,
+ "loss": 1.3398,
+ "theoretical_loss": 3.8280176790482474,
+ "tokens_seen": 664535040
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453600802407221,
+ "loss": 1.543,
+ "theoretical_loss": 3.827979280716068,
+ "tokens_seen": 664600576
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453440320962889,
+ "loss": 1.488,
+ "theoretical_loss": 3.8279408872302314,
+ "tokens_seen": 664666112
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453279839518556,
+ "loss": 1.3808,
+ "theoretical_loss": 3.827902498589648,
+ "tokens_seen": 664731648
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006453119358074223,
+ "loss": 1.3958,
+ "theoretical_loss": 3.827864114793229,
+ "tokens_seen": 664797184
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000645295887662989,
+ "loss": 1.4373,
+ "theoretical_loss": 3.827825735839886,
+ "tokens_seen": 664862720
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006452798395185557,
+ "loss": 1.4423,
+ "theoretical_loss": 3.8277873617285305,
+ "tokens_seen": 664928256
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006452637913741224,
+ "loss": 1.3114,
+ "theoretical_loss": 3.8277489924580745,
+ "tokens_seen": 664993792
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006452477432296891,
+ "loss": 1.3278,
+ "theoretical_loss": 3.82771062802743,
+ "tokens_seen": 665059328
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006452316950852558,
+ "loss": 1.3394,
+ "theoretical_loss": 3.827672268435509,
+ "tokens_seen": 665124864
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 227074,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4643185138702393,
+ "objective/train/theoretical_loss": 3.8276339136812263,
+ "objective/train/tokens_used": 685650400,
+ "theoretical_loss": 3.8276339136812263,
+ "tokens_seen": 665190400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006452156469408225,
+ "loss": 1.4293,
+ "theoretical_loss": 3.8276339136812263,
+ "tokens_seen": 665190400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451995987963892,
+ "loss": 1.4765,
+ "theoretical_loss": 3.8275955637634933,
+ "tokens_seen": 665255936
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451835506519559,
+ "loss": 1.3266,
+ "theoretical_loss": 3.8275572186812257,
+ "tokens_seen": 665321472
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451675025075226,
+ "loss": 1.3526,
+ "theoretical_loss": 3.8275188784333363,
+ "tokens_seen": 665387008
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451514543630893,
+ "loss": 1.4508,
+ "theoretical_loss": 3.8274805430187406,
+ "tokens_seen": 665452544
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000645135406218656,
+ "loss": 1.4785,
+ "theoretical_loss": 3.8274422124363516,
+ "tokens_seen": 665518080
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451193580742227,
+ "loss": 1.4408,
+ "theoretical_loss": 3.8274038866850866,
+ "tokens_seen": 665583616
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006451033099297894,
+ "loss": 1.3977,
+ "theoretical_loss": 3.8273655657638597,
+ "tokens_seen": 665649152
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450872617853561,
+ "loss": 1.4323,
+ "theoretical_loss": 3.827327249671587,
+ "tokens_seen": 665714688
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450712136409228,
+ "loss": 1.4965,
+ "theoretical_loss": 3.8272889384071846,
+ "tokens_seen": 665780224
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450551654964895,
+ "loss": 1.4091,
+ "theoretical_loss": 3.8272506319695694,
+ "tokens_seen": 665845760
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450391173520563,
+ "loss": 1.4464,
+ "theoretical_loss": 3.827212330357658,
+ "tokens_seen": 665911296
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450230692076229,
+ "loss": 1.3789,
+ "theoretical_loss": 3.827174033570368,
+ "tokens_seen": 665976832
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006450070210631897,
+ "loss": 1.4906,
+ "theoretical_loss": 3.8271357416066167,
+ "tokens_seen": 666042368
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449909729187563,
+ "loss": 1.4139,
+ "theoretical_loss": 3.827097454465322,
+ "tokens_seen": 666107904
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449749247743231,
+ "loss": 1.4756,
+ "theoretical_loss": 3.827059172145402,
+ "tokens_seen": 666173440
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449588766298897,
+ "loss": 1.457,
+ "theoretical_loss": 3.827020894645776,
+ "tokens_seen": 666238976
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449428284854565,
+ "loss": 1.4755,
+ "theoretical_loss": 3.826982621965363,
+ "tokens_seen": 666304512
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449267803410231,
+ "loss": 1.4857,
+ "theoretical_loss": 3.826944354103081,
+ "tokens_seen": 666370048
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006449107321965899,
+ "loss": 1.3627,
+ "theoretical_loss": 3.826906091057851,
+ "tokens_seen": 666435584
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448946840521565,
+ "loss": 1.5011,
+ "theoretical_loss": 3.8268678328285928,
+ "tokens_seen": 666501120
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448786359077233,
+ "loss": 1.3738,
+ "theoretical_loss": 3.8268295794142264,
+ "tokens_seen": 666566656
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448625877632899,
+ "loss": 1.3865,
+ "theoretical_loss": 3.826791330813673,
+ "tokens_seen": 666632192
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448465396188567,
+ "loss": 1.4636,
+ "theoretical_loss": 3.8267530870258533,
+ "tokens_seen": 666697728
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448304914744233,
+ "loss": 1.341,
+ "theoretical_loss": 3.826714848049689,
+ "tokens_seen": 666763264
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006448144433299901,
+ "loss": 1.3991,
+ "theoretical_loss": 3.8266766138841013,
+ "tokens_seen": 666828800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447983951855567,
+ "loss": 1.4756,
+ "theoretical_loss": 3.826638384528013,
+ "tokens_seen": 666894336
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447823470411235,
+ "loss": 1.3959,
+ "theoretical_loss": 3.8266001599803467,
+ "tokens_seen": 666959872
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447662988966901,
+ "loss": 1.4375,
+ "theoretical_loss": 3.8265619402400244,
+ "tokens_seen": 667025408
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447502507522569,
+ "loss": 1.3847,
+ "theoretical_loss": 3.82652372530597,
+ "tokens_seen": 667090944
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447342026078235,
+ "loss": 1.3348,
+ "theoretical_loss": 3.8264855151771067,
+ "tokens_seen": 667156480
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447181544633903,
+ "loss": 1.3579,
+ "theoretical_loss": 3.826447309852358,
+ "tokens_seen": 667222016
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006447021063189569,
+ "loss": 1.4032,
+ "theoretical_loss": 3.826409109330649,
+ "tokens_seen": 667287552
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446860581745237,
+ "loss": 1.5174,
+ "theoretical_loss": 3.826370913610903,
+ "tokens_seen": 667353088
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446700100300903,
+ "loss": 1.4905,
+ "theoretical_loss": 3.8263327226920465,
+ "tokens_seen": 667418624
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446539618856571,
+ "loss": 1.4021,
+ "theoretical_loss": 3.8262945365730032,
+ "tokens_seen": 667484160
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446379137412237,
+ "loss": 1.421,
+ "theoretical_loss": 3.8262563552526996,
+ "tokens_seen": 667549696
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446218655967904,
+ "loss": 1.3896,
+ "theoretical_loss": 3.8262181787300618,
+ "tokens_seen": 667615232
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006446058174523571,
+ "loss": 1.4655,
+ "theoretical_loss": 3.8261800070040146,
+ "tokens_seen": 667680768
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445897693079238,
+ "loss": 1.4998,
+ "theoretical_loss": 3.8261418400734866,
+ "tokens_seen": 667746304
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445737211634905,
+ "loss": 1.358,
+ "theoretical_loss": 3.826103677937403,
+ "tokens_seen": 667811840
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445576730190572,
+ "loss": 1.3546,
+ "theoretical_loss": 3.826065520594693,
+ "tokens_seen": 667877376
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445416248746239,
+ "loss": 1.4208,
+ "theoretical_loss": 3.8260273680442825,
+ "tokens_seen": 667942912
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445255767301906,
+ "loss": 1.418,
+ "theoretical_loss": 3.8259892202851002,
+ "tokens_seen": 668008448
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006445095285857573,
+ "loss": 1.5496,
+ "theoretical_loss": 3.825951077316075,
+ "tokens_seen": 668073984
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644493480441324,
+ "loss": 1.59,
+ "theoretical_loss": 3.825912939136135,
+ "tokens_seen": 668139520
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006444774322968906,
+ "loss": 1.3779,
+ "theoretical_loss": 3.825874805744209,
+ "tokens_seen": 668205056
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006444613841524574,
+ "loss": 1.378,
+ "theoretical_loss": 3.825836677139227,
+ "tokens_seen": 668270592
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644445336008024,
+ "loss": 1.4298,
+ "theoretical_loss": 3.8257985533201184,
+ "tokens_seen": 668336128
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006444292878635908,
+ "loss": 1.4356,
+ "theoretical_loss": 3.8257604342858134,
+ "tokens_seen": 668401664
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 228265,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3256380558013916,
+ "objective/train/theoretical_loss": 3.8257223200352417,
+ "objective/train/tokens_used": 688927200,
+ "theoretical_loss": 3.8257223200352417,
+ "tokens_seen": 668467200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006444132397191574,
+ "loss": 1.4047,
+ "theoretical_loss": 3.8257223200352417,
+ "tokens_seen": 668467200
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443971915747242,
+ "loss": 1.4653,
+ "theoretical_loss": 3.825684210567335,
+ "tokens_seen": 668532736
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443811434302908,
+ "loss": 1.4851,
+ "theoretical_loss": 3.8256461058810247,
+ "tokens_seen": 668598272
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443650952858576,
+ "loss": 1.4715,
+ "theoretical_loss": 3.825608005975241,
+ "tokens_seen": 668663808
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443490471414242,
+ "loss": 1.3709,
+ "theoretical_loss": 3.8255699108489165,
+ "tokens_seen": 668729344
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644332998996991,
+ "loss": 1.4355,
+ "theoretical_loss": 3.8255318205009834,
+ "tokens_seen": 668794880
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443169508525576,
+ "loss": 1.5044,
+ "theoretical_loss": 3.825493734930374,
+ "tokens_seen": 668860416
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006443009027081244,
+ "loss": 1.3877,
+ "theoretical_loss": 3.8254556541360207,
+ "tokens_seen": 668925952
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644284854563691,
+ "loss": 1.4145,
+ "theoretical_loss": 3.8254175781168573,
+ "tokens_seen": 668991488
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006442688064192578,
+ "loss": 1.3835,
+ "theoretical_loss": 3.825379506871817,
+ "tokens_seen": 669057024
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006442527582748244,
+ "loss": 1.3198,
+ "theoretical_loss": 3.8253414403998334,
+ "tokens_seen": 669122560
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006442367101303912,
+ "loss": 1.3518,
+ "theoretical_loss": 3.825303378699841,
+ "tokens_seen": 669188096
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006442206619859578,
+ "loss": 1.4474,
+ "theoretical_loss": 3.8252653217707753,
+ "tokens_seen": 669253632
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006442046138415246,
+ "loss": 1.4523,
+ "theoretical_loss": 3.8252272696115694,
+ "tokens_seen": 669319168
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006441885656970912,
+ "loss": 1.5241,
+ "theoretical_loss": 3.8251892222211596,
+ "tokens_seen": 669384704
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644172517552658,
+ "loss": 1.3396,
+ "theoretical_loss": 3.825151179598481,
+ "tokens_seen": 669450240
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006441564694082246,
+ "loss": 1.453,
+ "theoretical_loss": 3.8251131417424693,
+ "tokens_seen": 669515776
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006441404212637914,
+ "loss": 1.4106,
+ "theoretical_loss": 3.8250751086520616,
+ "tokens_seen": 669581312
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644124373119358,
+ "loss": 1.4388,
+ "theoretical_loss": 3.825037080326194,
+ "tokens_seen": 669646848
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006441083249749248,
+ "loss": 1.4614,
+ "theoretical_loss": 3.8249990567638035,
+ "tokens_seen": 669712384
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006440922768304914,
+ "loss": 1.256,
+ "theoretical_loss": 3.8249610379638277,
+ "tokens_seen": 669777920
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006440762286860582,
+ "loss": 1.4333,
+ "theoretical_loss": 3.8249230239252032,
+ "tokens_seen": 669843456
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006440601805416248,
+ "loss": 1.3193,
+ "theoretical_loss": 3.8248850146468687,
+ "tokens_seen": 669908992
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006440441323971916,
+ "loss": 1.349,
+ "theoretical_loss": 3.8248470101277627,
+ "tokens_seen": 669974528
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006440280842527582,
+ "loss": 1.3975,
+ "theoretical_loss": 3.824809010366823,
+ "tokens_seen": 670040064
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000644012036108325,
+ "loss": 1.3833,
+ "theoretical_loss": 3.8247710153629892,
+ "tokens_seen": 670105600
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006439959879638916,
+ "loss": 1.3982,
+ "theoretical_loss": 3.8247330251152007,
+ "tokens_seen": 670171136
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006439799398194584,
+ "loss": 1.4416,
+ "theoretical_loss": 3.824695039622397,
+ "tokens_seen": 670236672
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643963891675025,
+ "loss": 1.3284,
+ "theoretical_loss": 3.824657058883518,
+ "tokens_seen": 670302208
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006439478435305918,
+ "loss": 1.4482,
+ "theoretical_loss": 3.824619082897504,
+ "tokens_seen": 670367744
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006439317953861585,
+ "loss": 1.5329,
+ "theoretical_loss": 3.824581111663296,
+ "tokens_seen": 670433280
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006439157472417252,
+ "loss": 1.4133,
+ "theoretical_loss": 3.8245431451798346,
+ "tokens_seen": 670498816
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438996990972919,
+ "loss": 1.3977,
+ "theoretical_loss": 3.8245051834460613,
+ "tokens_seen": 670564352
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438836509528586,
+ "loss": 1.3864,
+ "theoretical_loss": 3.824467226460918,
+ "tokens_seen": 670629888
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438676028084253,
+ "loss": 1.4684,
+ "theoretical_loss": 3.8244292742233457,
+ "tokens_seen": 670695424
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643851554663992,
+ "loss": 1.3351,
+ "theoretical_loss": 3.8243913267322887,
+ "tokens_seen": 670760960
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438355065195588,
+ "loss": 1.4502,
+ "theoretical_loss": 3.824353383986688,
+ "tokens_seen": 670826496
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438194583751254,
+ "loss": 1.4654,
+ "theoretical_loss": 3.824315445985487,
+ "tokens_seen": 670892032
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006438034102306922,
+ "loss": 1.4255,
+ "theoretical_loss": 3.8242775127276296,
+ "tokens_seen": 670957568
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006437873620862588,
+ "loss": 1.3765,
+ "theoretical_loss": 3.8242395842120596,
+ "tokens_seen": 671023104
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006437713139418256,
+ "loss": 1.3318,
+ "theoretical_loss": 3.824201660437721,
+ "tokens_seen": 671088640
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006437552657973922,
+ "loss": 1.3525,
+ "theoretical_loss": 3.824163741403557,
+ "tokens_seen": 671154176
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643739217652959,
+ "loss": 1.4577,
+ "theoretical_loss": 3.8241258271085137,
+ "tokens_seen": 671219712
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006437231695085256,
+ "loss": 1.388,
+ "theoretical_loss": 3.824087917551536,
+ "tokens_seen": 671285248
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006437071213640924,
+ "loss": 1.3655,
+ "theoretical_loss": 3.8240500127315693,
+ "tokens_seen": 671350784
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643691073219659,
+ "loss": 1.4919,
+ "theoretical_loss": 3.8240121126475586,
+ "tokens_seen": 671416320
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006436750250752258,
+ "loss": 1.3101,
+ "theoretical_loss": 3.82397421729845,
+ "tokens_seen": 671481856
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006436589769307924,
+ "loss": 1.4365,
+ "theoretical_loss": 3.8239363266831914,
+ "tokens_seen": 671547392
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006436429287863592,
+ "loss": 1.459,
+ "theoretical_loss": 3.823898440800728,
+ "tokens_seen": 671612928
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006436268806419258,
+ "loss": 1.3721,
+ "theoretical_loss": 3.823860559650008,
+ "tokens_seen": 671678464
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 229250,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3694689273834229,
+ "objective/train/theoretical_loss": 3.823822683229978,
+ "objective/train/tokens_used": 692204000,
+ "theoretical_loss": 3.823822683229978,
+ "tokens_seen": 671744000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006436108324974926,
+ "loss": 1.3617,
+ "theoretical_loss": 3.823822683229978,
+ "tokens_seen": 671744000
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006435947843530592,
+ "loss": 1.335,
+ "theoretical_loss": 3.8237848115395865,
+ "tokens_seen": 671809536
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643578736208626,
+ "loss": 1.5724,
+ "theoretical_loss": 3.823746944577781,
+ "tokens_seen": 671875072
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006435626880641926,
+ "loss": 1.5382,
+ "theoretical_loss": 3.82370908234351,
+ "tokens_seen": 671940608
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006435466399197594,
+ "loss": 1.3706,
+ "theoretical_loss": 3.8236712248357225,
+ "tokens_seen": 672006144
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643530591775326,
+ "loss": 1.3624,
+ "theoretical_loss": 3.8236333720533677,
+ "tokens_seen": 672071680
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006435145436308928,
+ "loss": 1.3729,
+ "theoretical_loss": 3.823595523995395,
+ "tokens_seen": 672137216
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434984954864594,
+ "loss": 1.3791,
+ "theoretical_loss": 3.823557680660754,
+ "tokens_seen": 672202752
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434824473420262,
+ "loss": 1.3773,
+ "theoretical_loss": 3.823519842048395,
+ "tokens_seen": 672268288
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434663991975928,
+ "loss": 1.4648,
+ "theoretical_loss": 3.823482008157269,
+ "tokens_seen": 672333824
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434503510531596,
+ "loss": 1.2812,
+ "theoretical_loss": 3.8234441789863256,
+ "tokens_seen": 672399360
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434343029087262,
+ "loss": 1.3632,
+ "theoretical_loss": 3.8234063545345167,
+ "tokens_seen": 672464896
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643418254764293,
+ "loss": 1.4468,
+ "theoretical_loss": 3.8233685348007933,
+ "tokens_seen": 672530432
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006434022066198596,
+ "loss": 1.4096,
+ "theoretical_loss": 3.823330719784108,
+ "tokens_seen": 672595968
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006433861584754264,
+ "loss": 1.4883,
+ "theoretical_loss": 3.8232929094834125,
+ "tokens_seen": 672661504
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.000643370110330993,
+ "loss": 1.5356,
+ "theoretical_loss": 3.823255103897659,
+ "tokens_seen": 672727040
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006433540621865598,
+ "loss": 1.4702,
+ "theoretical_loss": 3.8232173030258005,
+ "tokens_seen": 672792576
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006433380140421264,
+ "loss": 1.3244,
+ "theoretical_loss": 3.823179506866791,
+ "tokens_seen": 672858112
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006433219658976932,
+ "loss": 1.3574,
+ "theoretical_loss": 3.823141715419582,
+ "tokens_seen": 672923648
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006433059177532598,
+ "loss": 1.4943,
+ "theoretical_loss": 3.823103928683129,
+ "tokens_seen": 672989184
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006432898696088266,
+ "loss": 1.4458,
+ "theoretical_loss": 3.8230661466563864,
+ "tokens_seen": 673054720
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006432738214643932,
+ "loss": 1.3608,
+ "theoretical_loss": 3.8230283693383074,
+ "tokens_seen": 673120256
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064325777331996,
+ "loss": 1.3938,
+ "theoretical_loss": 3.822990596727847,
+ "tokens_seen": 673185792
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006432417251755266,
+ "loss": 1.5127,
+ "theoretical_loss": 3.822952828823961,
+ "tokens_seen": 673251328
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006432256770310933,
+ "loss": 1.3823,
+ "theoretical_loss": 3.8229150656256046,
+ "tokens_seen": 673316864
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.00064320962888666,
+ "loss": 1.4841,
+ "theoretical_loss": 3.8228773071317335,
+ "tokens_seen": 673382400
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431935807422267,
+ "loss": 1.3449,
+ "theoretical_loss": 3.822839553341304,
+ "tokens_seen": 673447936
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431775325977934,
+ "loss": 1.4301,
+ "theoretical_loss": 3.822801804253273,
+ "tokens_seen": 673513472
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431614844533601,
+ "loss": 1.3745,
+ "theoretical_loss": 3.8227640598665964,
+ "tokens_seen": 673579008
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431454363089268,
+ "loss": 1.47,
+ "theoretical_loss": 3.822726320180232,
+ "tokens_seen": 673644544
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431293881644935,
+ "loss": 1.4192,
+ "theoretical_loss": 3.822688585193137,
+ "tokens_seen": 673710080
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006431133400200602,
+ "loss": 1.3804,
+ "theoretical_loss": 3.822650854904269,
+ "tokens_seen": 673775616
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430972918756269,
+ "loss": 1.3837,
+ "theoretical_loss": 3.822613129312587,
+ "tokens_seen": 673841152
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430812437311935,
+ "loss": 1.414,
+ "theoretical_loss": 3.8225754084170482,
+ "tokens_seen": 673906688
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430651955867603,
+ "loss": 1.4774,
+ "theoretical_loss": 3.8225376922166125,
+ "tokens_seen": 673972224
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430491474423269,
+ "loss": 1.4462,
+ "theoretical_loss": 3.822499980710239,
+ "tokens_seen": 674037760
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430330992978937,
+ "loss": 1.4094,
+ "theoretical_loss": 3.8224622738968863,
+ "tokens_seen": 674103296
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430170511534603,
+ "loss": 1.3966,
+ "theoretical_loss": 3.822424571775515,
+ "tokens_seen": 674168832
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006430010030090271,
+ "loss": 1.352,
+ "theoretical_loss": 3.822386874345085,
+ "tokens_seen": 674234368
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429849548645937,
+ "loss": 1.4683,
+ "theoretical_loss": 3.822349181604557,
+ "tokens_seen": 674299904
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429689067201605,
+ "loss": 1.399,
+ "theoretical_loss": 3.8223114935528915,
+ "tokens_seen": 674365440
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429528585757271,
+ "loss": 1.3665,
+ "theoretical_loss": 3.82227381018905,
+ "tokens_seen": 674430976
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429368104312939,
+ "loss": 1.425,
+ "theoretical_loss": 3.822236131511993,
+ "tokens_seen": 674496512
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429207622868605,
+ "loss": 1.3625,
+ "theoretical_loss": 3.8221984575206833,
+ "tokens_seen": 674562048
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006429047141424273,
+ "loss": 1.365,
+ "theoretical_loss": 3.8221607882140827,
+ "tokens_seen": 674627584
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428886659979939,
+ "loss": 1.4393,
+ "theoretical_loss": 3.8221231235911537,
+ "tokens_seen": 674693120
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428726178535607,
+ "loss": 1.4967,
+ "theoretical_loss": 3.822085463650859,
+ "tokens_seen": 674758656
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428565697091273,
+ "loss": 1.3798,
+ "theoretical_loss": 3.8220478083921616,
+ "tokens_seen": 674824192
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428405215646941,
+ "loss": 1.5147,
+ "theoretical_loss": 3.8220101578140255,
+ "tokens_seen": 674889728
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428244734202607,
+ "loss": 1.5698,
+ "theoretical_loss": 3.8219725119154138,
+ "tokens_seen": 674955264
+ },
+ {
+ "epoch": 0.2,
+ "objective/train/docs_used": 230444,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4336386919021606,
+ "objective/train/theoretical_loss": 3.821934870695291,
+ "objective/train/tokens_used": 695480800,
+ "theoretical_loss": 3.821934870695291,
+ "tokens_seen": 675020800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006428084252758275,
+ "loss": 1.3804,
+ "theoretical_loss": 3.821934870695291,
+ "tokens_seen": 675020800
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427923771313941,
+ "loss": 1.5014,
+ "theoretical_loss": 3.8218972341526216,
+ "tokens_seen": 675086336
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427763289869609,
+ "loss": 1.3548,
+ "theoretical_loss": 3.8218596022863696,
+ "tokens_seen": 675151872
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427602808425275,
+ "loss": 1.4235,
+ "theoretical_loss": 3.821821975095501,
+ "tokens_seen": 675217408
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427442326980943,
+ "loss": 1.4179,
+ "theoretical_loss": 3.821784352578981,
+ "tokens_seen": 675282944
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427281845536609,
+ "loss": 1.4752,
+ "theoretical_loss": 3.8217467347357754,
+ "tokens_seen": 675348480
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006427121364092277,
+ "loss": 1.5204,
+ "theoretical_loss": 3.82170912156485,
+ "tokens_seen": 675414016
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426960882647943,
+ "loss": 1.4027,
+ "theoretical_loss": 3.8216715130651715,
+ "tokens_seen": 675479552
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426800401203611,
+ "loss": 1.5151,
+ "theoretical_loss": 3.8216339092357066,
+ "tokens_seen": 675545088
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426639919759277,
+ "loss": 1.3896,
+ "theoretical_loss": 3.8215963100754218,
+ "tokens_seen": 675610624
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426479438314945,
+ "loss": 1.4323,
+ "theoretical_loss": 3.8215587155832855,
+ "tokens_seen": 675676160
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426318956870613,
+ "loss": 1.4006,
+ "theoretical_loss": 3.8215211257582644,
+ "tokens_seen": 675741696
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006426158475426279,
+ "loss": 1.3455,
+ "theoretical_loss": 3.8214835405993273,
+ "tokens_seen": 675807232
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425997993981947,
+ "loss": 1.5163,
+ "theoretical_loss": 3.8214459601054425,
+ "tokens_seen": 675872768
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425837512537613,
+ "loss": 1.429,
+ "theoretical_loss": 3.8214083842755784,
+ "tokens_seen": 675938304
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425677031093281,
+ "loss": 1.4165,
+ "theoretical_loss": 3.8213708131087043,
+ "tokens_seen": 676003840
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425516549648947,
+ "loss": 1.5654,
+ "theoretical_loss": 3.8213332466037895,
+ "tokens_seen": 676069376
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425356068204615,
+ "loss": 1.4415,
+ "theoretical_loss": 3.8212956847598036,
+ "tokens_seen": 676134912
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425195586760281,
+ "loss": 1.3559,
+ "theoretical_loss": 3.8212581275757165,
+ "tokens_seen": 676200448
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006425035105315949,
+ "loss": 1.5231,
+ "theoretical_loss": 3.821220575050499,
+ "tokens_seen": 676265984
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006424874623871615,
+ "loss": 1.3894,
+ "theoretical_loss": 3.821183027183122,
+ "tokens_seen": 676331520
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006424714142427283,
+ "loss": 1.4321,
+ "theoretical_loss": 3.821145483972556,
+ "tokens_seen": 676397056
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 0.0006424553660982949,
+ "loss": 1.4047,
+ "theoretical_loss": 3.821107945417771,
+ "tokens_seen": 676462592
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006424393179538617,
+ "loss": 1.4645,
+ "theoretical_loss": 3.8210704115177414,
+ "tokens_seen": 676528128
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006424232698094283,
+ "loss": 1.4276,
+ "theoretical_loss": 3.8210328822714374,
+ "tokens_seen": 676593664
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006424072216649951,
+ "loss": 1.4779,
+ "theoretical_loss": 3.820995357677832,
+ "tokens_seen": 676659200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423911735205617,
+ "loss": 1.5168,
+ "theoretical_loss": 3.8209578377358975,
+ "tokens_seen": 676724736
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423751253761285,
+ "loss": 1.5245,
+ "theoretical_loss": 3.8209203224446067,
+ "tokens_seen": 676790272
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423590772316951,
+ "loss": 1.3363,
+ "theoretical_loss": 3.820882811802933,
+ "tokens_seen": 676855808
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423430290872619,
+ "loss": 1.3885,
+ "theoretical_loss": 3.82084530580985,
+ "tokens_seen": 676921344
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423269809428285,
+ "loss": 1.4748,
+ "theoretical_loss": 3.8208078044643328,
+ "tokens_seen": 676986880
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006423109327983953,
+ "loss": 1.5085,
+ "theoretical_loss": 3.8207703077653536,
+ "tokens_seen": 677052416
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422948846539619,
+ "loss": 1.4075,
+ "theoretical_loss": 3.8207328157118887,
+ "tokens_seen": 677117952
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422788365095287,
+ "loss": 1.4019,
+ "theoretical_loss": 3.820695328302912,
+ "tokens_seen": 677183488
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422627883650953,
+ "loss": 1.413,
+ "theoretical_loss": 3.8206578455373994,
+ "tokens_seen": 677249024
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422467402206621,
+ "loss": 1.4656,
+ "theoretical_loss": 3.820620367414326,
+ "tokens_seen": 677314560
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422306920762287,
+ "loss": 1.3562,
+ "theoretical_loss": 3.8205828939326683,
+ "tokens_seen": 677380096
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006422146439317955,
+ "loss": 1.4737,
+ "theoretical_loss": 3.8205454250914013,
+ "tokens_seen": 677445632
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421985957873621,
+ "loss": 1.4588,
+ "theoretical_loss": 3.820507960889503,
+ "tokens_seen": 677511168
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421825476429289,
+ "loss": 1.3709,
+ "theoretical_loss": 3.82047050132595,
+ "tokens_seen": 677576704
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421664994984955,
+ "loss": 1.3659,
+ "theoretical_loss": 3.8204330463997183,
+ "tokens_seen": 677642240
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421504513540623,
+ "loss": 1.4767,
+ "theoretical_loss": 3.820395596109787,
+ "tokens_seen": 677707776
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421344032096289,
+ "loss": 1.3873,
+ "theoretical_loss": 3.8203581504551325,
+ "tokens_seen": 677773312
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421183550651957,
+ "loss": 1.4155,
+ "theoretical_loss": 3.820320709434734,
+ "tokens_seen": 677838848
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006421023069207623,
+ "loss": 1.4827,
+ "theoretical_loss": 3.8202832730475693,
+ "tokens_seen": 677904384
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420862587763291,
+ "loss": 1.3759,
+ "theoretical_loss": 3.820245841292618,
+ "tokens_seen": 677969920
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420702106318957,
+ "loss": 1.4155,
+ "theoretical_loss": 3.820208414168859,
+ "tokens_seen": 678035456
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420541624874625,
+ "loss": 1.3425,
+ "theoretical_loss": 3.820170991675271,
+ "tokens_seen": 678100992
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420381143430291,
+ "loss": 1.3834,
+ "theoretical_loss": 3.8201335738108346,
+ "tokens_seen": 678166528
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420220661985959,
+ "loss": 1.4239,
+ "theoretical_loss": 3.82009616057453,
+ "tokens_seen": 678232064
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 231396,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4644248485565186,
+ "objective/train/theoretical_loss": 3.8200587519653366,
+ "objective/train/tokens_used": 698757600,
+ "theoretical_loss": 3.8200587519653366,
+ "tokens_seen": 678297600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006420060180541625,
+ "loss": 1.4658,
+ "theoretical_loss": 3.8200587519653366,
+ "tokens_seen": 678297600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419899699097293,
+ "loss": 1.3733,
+ "theoretical_loss": 3.8200213479822365,
+ "tokens_seen": 678363136
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419739217652959,
+ "loss": 1.2866,
+ "theoretical_loss": 3.81998394862421,
+ "tokens_seen": 678428672
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419578736208627,
+ "loss": 1.4561,
+ "theoretical_loss": 3.8199465538902384,
+ "tokens_seen": 678494208
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419418254764293,
+ "loss": 1.5282,
+ "theoretical_loss": 3.819909163779304,
+ "tokens_seen": 678559744
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419257773319961,
+ "loss": 1.5327,
+ "theoretical_loss": 3.8198717782903886,
+ "tokens_seen": 678625280
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006419097291875627,
+ "loss": 1.4462,
+ "theoretical_loss": 3.8198343974224747,
+ "tokens_seen": 678690816
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418936810431295,
+ "loss": 1.3364,
+ "theoretical_loss": 3.8197970211745442,
+ "tokens_seen": 678756352
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418776328986961,
+ "loss": 1.3884,
+ "theoretical_loss": 3.819759649545581,
+ "tokens_seen": 678821888
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418615847542629,
+ "loss": 1.5174,
+ "theoretical_loss": 3.819722282534568,
+ "tokens_seen": 678887424
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418455366098295,
+ "loss": 1.3739,
+ "theoretical_loss": 3.8196849201404888,
+ "tokens_seen": 678952960
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418294884653962,
+ "loss": 1.3537,
+ "theoretical_loss": 3.819647562362328,
+ "tokens_seen": 679018496
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006418134403209629,
+ "loss": 1.3341,
+ "theoretical_loss": 3.819610209199069,
+ "tokens_seen": 679084032
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417973921765296,
+ "loss": 1.3727,
+ "theoretical_loss": 3.8195728606496973,
+ "tokens_seen": 679149568
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417813440320963,
+ "loss": 1.3802,
+ "theoretical_loss": 3.8195355167131977,
+ "tokens_seen": 679215104
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641765295887663,
+ "loss": 1.4435,
+ "theoretical_loss": 3.819498177388555,
+ "tokens_seen": 679280640
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417492477432297,
+ "loss": 1.421,
+ "theoretical_loss": 3.8194608426747543,
+ "tokens_seen": 679346176
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417331995987964,
+ "loss": 1.5297,
+ "theoretical_loss": 3.819423512570783,
+ "tokens_seen": 679411712
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417171514543631,
+ "loss": 1.3532,
+ "theoretical_loss": 3.8193861870756267,
+ "tokens_seen": 679477248
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006417011033099298,
+ "loss": 1.3038,
+ "theoretical_loss": 3.819348866188271,
+ "tokens_seen": 679542784
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006416850551654964,
+ "loss": 1.4278,
+ "theoretical_loss": 3.819311549907704,
+ "tokens_seen": 679608320
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006416690070210632,
+ "loss": 1.4793,
+ "theoretical_loss": 3.8192742382329126,
+ "tokens_seen": 679673856
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006416529588766298,
+ "loss": 1.4789,
+ "theoretical_loss": 3.819236931162884,
+ "tokens_seen": 679739392
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006416369107321966,
+ "loss": 1.3844,
+ "theoretical_loss": 3.8191996286966066,
+ "tokens_seen": 679804928
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006416208625877632,
+ "loss": 1.3349,
+ "theoretical_loss": 3.819162330833068,
+ "tokens_seen": 679870464
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00064160481444333,
+ "loss": 1.4222,
+ "theoretical_loss": 3.8191250375712564,
+ "tokens_seen": 679936000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006415887662988966,
+ "loss": 1.5197,
+ "theoretical_loss": 3.819087748910161,
+ "tokens_seen": 680001536
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006415727181544634,
+ "loss": 1.4457,
+ "theoretical_loss": 3.8190504648487718,
+ "tokens_seen": 680067072
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00064155667001003,
+ "loss": 1.4379,
+ "theoretical_loss": 3.8190131853860763,
+ "tokens_seen": 680132608
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006415406218655968,
+ "loss": 1.5029,
+ "theoretical_loss": 3.8189759105210657,
+ "tokens_seen": 680198144
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006415245737211634,
+ "loss": 1.4406,
+ "theoretical_loss": 3.8189386402527297,
+ "tokens_seen": 680263680
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006415085255767302,
+ "loss": 1.3714,
+ "theoretical_loss": 3.818901374580059,
+ "tokens_seen": 680329216
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006414924774322968,
+ "loss": 1.5279,
+ "theoretical_loss": 3.818864113502044,
+ "tokens_seen": 680394752
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006414764292878636,
+ "loss": 1.4553,
+ "theoretical_loss": 3.818826857017675,
+ "tokens_seen": 680460288
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006414603811434302,
+ "loss": 1.4746,
+ "theoretical_loss": 3.8187896051259442,
+ "tokens_seen": 680525824
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641444332998997,
+ "loss": 1.4045,
+ "theoretical_loss": 3.818752357825843,
+ "tokens_seen": 680591360
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006414282848545638,
+ "loss": 1.37,
+ "theoretical_loss": 3.8187151151163636,
+ "tokens_seen": 680656896
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006414122367101304,
+ "loss": 1.4326,
+ "theoretical_loss": 3.8186778769964986,
+ "tokens_seen": 680722432
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006413961885656972,
+ "loss": 1.4775,
+ "theoretical_loss": 3.81864064346524,
+ "tokens_seen": 680787968
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006413801404212638,
+ "loss": 1.4301,
+ "theoretical_loss": 3.818603414521581,
+ "tokens_seen": 680853504
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006413640922768306,
+ "loss": 1.3632,
+ "theoretical_loss": 3.818566190164515,
+ "tokens_seen": 680919040
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006413480441323972,
+ "loss": 1.4633,
+ "theoretical_loss": 3.8185289703930345,
+ "tokens_seen": 680984576
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641331995987964,
+ "loss": 1.4356,
+ "theoretical_loss": 3.818491755206135,
+ "tokens_seen": 681050112
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006413159478435306,
+ "loss": 1.4231,
+ "theoretical_loss": 3.8184545446028095,
+ "tokens_seen": 681115648
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412998996990974,
+ "loss": 1.4653,
+ "theoretical_loss": 3.8184173385820532,
+ "tokens_seen": 681181184
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641283851554664,
+ "loss": 1.3554,
+ "theoretical_loss": 3.818380137142861,
+ "tokens_seen": 681246720
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412678034102308,
+ "loss": 1.5759,
+ "theoretical_loss": 3.8183429402842277,
+ "tokens_seen": 681312256
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412517552657974,
+ "loss": 1.2995,
+ "theoretical_loss": 3.8183057480051485,
+ "tokens_seen": 681377792
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412357071213642,
+ "loss": 1.4206,
+ "theoretical_loss": 3.81826856030462,
+ "tokens_seen": 681443328
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412196589769308,
+ "loss": 1.458,
+ "theoretical_loss": 3.818231377181638,
+ "tokens_seen": 681508864
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 232668,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2436425685882568,
+ "objective/train/theoretical_loss": 3.818194198635198,
+ "objective/train/tokens_used": 702034400,
+ "theoretical_loss": 3.818194198635198,
+ "tokens_seen": 681574400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006412036108324976,
+ "loss": 1.3211,
+ "theoretical_loss": 3.818194198635198,
+ "tokens_seen": 681574400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006411875626880642,
+ "loss": 1.3687,
+ "theoretical_loss": 3.818157024664298,
+ "tokens_seen": 681639936
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641171514543631,
+ "loss": 1.4933,
+ "theoretical_loss": 3.8181198552679345,
+ "tokens_seen": 681705472
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006411554663991976,
+ "loss": 1.432,
+ "theoretical_loss": 3.818082690445105,
+ "tokens_seen": 681771008
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006411394182547644,
+ "loss": 1.4306,
+ "theoretical_loss": 3.818045530194807,
+ "tokens_seen": 681836544
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641123370110331,
+ "loss": 1.4254,
+ "theoretical_loss": 3.8180083745160394,
+ "tokens_seen": 681902080
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006411073219658978,
+ "loss": 1.455,
+ "theoretical_loss": 3.817971223407799,
+ "tokens_seen": 681967616
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006410912738214644,
+ "loss": 1.367,
+ "theoretical_loss": 3.8179340768690855,
+ "tokens_seen": 682033152
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006410752256770312,
+ "loss": 1.3193,
+ "theoretical_loss": 3.817896934898897,
+ "tokens_seen": 682098688
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006410591775325978,
+ "loss": 1.3972,
+ "theoretical_loss": 3.8178597974962343,
+ "tokens_seen": 682164224
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006410431293881646,
+ "loss": 1.4552,
+ "theoretical_loss": 3.817822664660096,
+ "tokens_seen": 682229760
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006410270812437312,
+ "loss": 1.3903,
+ "theoretical_loss": 3.817785536389481,
+ "tokens_seen": 682295296
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000641011033099298,
+ "loss": 1.41,
+ "theoretical_loss": 3.8177484126833914,
+ "tokens_seen": 682360832
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006409949849548646,
+ "loss": 1.4212,
+ "theoretical_loss": 3.8177112935408273,
+ "tokens_seen": 682426368
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006409789368104314,
+ "loss": 1.4377,
+ "theoretical_loss": 3.8176741789607886,
+ "tokens_seen": 682491904
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640962888665998,
+ "loss": 1.3976,
+ "theoretical_loss": 3.8176370689422767,
+ "tokens_seen": 682557440
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006409468405215648,
+ "loss": 1.4831,
+ "theoretical_loss": 3.8175999634842936,
+ "tokens_seen": 682622976
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006409307923771314,
+ "loss": 1.4321,
+ "theoretical_loss": 3.8175628625858407,
+ "tokens_seen": 682688512
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006409147442326982,
+ "loss": 1.4783,
+ "theoretical_loss": 3.8175257662459208,
+ "tokens_seen": 682754048
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006408986960882648,
+ "loss": 1.4594,
+ "theoretical_loss": 3.8174886744635352,
+ "tokens_seen": 682819584
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006408826479438316,
+ "loss": 1.4002,
+ "theoretical_loss": 3.8174515872376875,
+ "tokens_seen": 682885120
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006408665997993982,
+ "loss": 1.3813,
+ "theoretical_loss": 3.81741450456738,
+ "tokens_seen": 682950656
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640850551654965,
+ "loss": 1.3785,
+ "theoretical_loss": 3.817377426451617,
+ "tokens_seen": 683016192
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006408345035105316,
+ "loss": 1.5324,
+ "theoretical_loss": 3.817340352889402,
+ "tokens_seen": 683081728
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006408184553660984,
+ "loss": 1.4589,
+ "theoretical_loss": 3.8173032838797383,
+ "tokens_seen": 683147264
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640802407221665,
+ "loss": 1.4205,
+ "theoretical_loss": 3.8172662194216302,
+ "tokens_seen": 683212800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407863590772318,
+ "loss": 1.4372,
+ "theoretical_loss": 3.817229159514083,
+ "tokens_seen": 683278336
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407703109327984,
+ "loss": 1.4897,
+ "theoretical_loss": 3.817192104156101,
+ "tokens_seen": 683343872
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407542627883652,
+ "loss": 1.4622,
+ "theoretical_loss": 3.8171550533466903,
+ "tokens_seen": 683409408
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407382146439318,
+ "loss": 1.3386,
+ "theoretical_loss": 3.817118007084856,
+ "tokens_seen": 683474944
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407221664994986,
+ "loss": 1.486,
+ "theoretical_loss": 3.8170809653696036,
+ "tokens_seen": 683540480
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006407061183550652,
+ "loss": 1.3826,
+ "theoretical_loss": 3.817043928199939,
+ "tokens_seen": 683606016
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640690070210632,
+ "loss": 1.4282,
+ "theoretical_loss": 3.8170068955748695,
+ "tokens_seen": 683671552
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006406740220661986,
+ "loss": 1.4732,
+ "theoretical_loss": 3.8169698674934023,
+ "tokens_seen": 683737088
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006406579739217654,
+ "loss": 1.4747,
+ "theoretical_loss": 3.816932843954543,
+ "tokens_seen": 683802624
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640641925777332,
+ "loss": 1.4705,
+ "theoretical_loss": 3.8168958249573004,
+ "tokens_seen": 683868160
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006406258776328988,
+ "loss": 1.451,
+ "theoretical_loss": 3.8168588105006815,
+ "tokens_seen": 683933696
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006406098294884654,
+ "loss": 1.4812,
+ "theoretical_loss": 3.816821800583695,
+ "tokens_seen": 683999232
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006405937813440322,
+ "loss": 1.5242,
+ "theoretical_loss": 3.816784795205348,
+ "tokens_seen": 684064768
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006405777331995988,
+ "loss": 1.3584,
+ "theoretical_loss": 3.816747794364651,
+ "tokens_seen": 684130304
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006405616850551656,
+ "loss": 1.4066,
+ "theoretical_loss": 3.816710798060611,
+ "tokens_seen": 684195840
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006405456369107322,
+ "loss": 1.4265,
+ "theoretical_loss": 3.816673806292239,
+ "tokens_seen": 684261376
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640529588766299,
+ "loss": 1.4504,
+ "theoretical_loss": 3.8166368190585436,
+ "tokens_seen": 684326912
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006405135406218656,
+ "loss": 1.5065,
+ "theoretical_loss": 3.816599836358535,
+ "tokens_seen": 684392448
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404974924774324,
+ "loss": 1.2701,
+ "theoretical_loss": 3.816562858191223,
+ "tokens_seen": 684457984
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640481444332999,
+ "loss": 1.433,
+ "theoretical_loss": 3.8165258845556194,
+ "tokens_seen": 684523520
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404653961885658,
+ "loss": 1.4168,
+ "theoretical_loss": 3.8164889154507335,
+ "tokens_seen": 684589056
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404493480441324,
+ "loss": 1.3753,
+ "theoretical_loss": 3.816451950875578,
+ "tokens_seen": 684654592
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404332998996991,
+ "loss": 1.4882,
+ "theoretical_loss": 3.8164149908291627,
+ "tokens_seen": 684720128
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404172517552658,
+ "loss": 1.4322,
+ "theoretical_loss": 3.8163780353105006,
+ "tokens_seen": 684785664
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 233601,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3923734426498413,
+ "objective/train/theoretical_loss": 3.816341084318604,
+ "objective/train/tokens_used": 705311200,
+ "theoretical_loss": 3.816341084318604,
+ "tokens_seen": 684851200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006404012036108325,
+ "loss": 1.4253,
+ "theoretical_loss": 3.816341084318604,
+ "tokens_seen": 684851200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006403851554663992,
+ "loss": 1.5158,
+ "theoretical_loss": 3.816304137852484,
+ "tokens_seen": 684916736
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006403691073219659,
+ "loss": 1.2934,
+ "theoretical_loss": 3.8162671959111543,
+ "tokens_seen": 684982272
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006403530591775326,
+ "loss": 1.4347,
+ "theoretical_loss": 3.8162302584936274,
+ "tokens_seen": 685047808
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006403370110330993,
+ "loss": 1.4166,
+ "theoretical_loss": 3.8161933255989173,
+ "tokens_seen": 685113344
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640320962888666,
+ "loss": 1.4776,
+ "theoretical_loss": 3.8161563972260373,
+ "tokens_seen": 685178880
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006403049147442327,
+ "loss": 1.437,
+ "theoretical_loss": 3.816119473374001,
+ "tokens_seen": 685244416
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402888665997993,
+ "loss": 1.407,
+ "theoretical_loss": 3.816082554041823,
+ "tokens_seen": 685309952
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402728184553661,
+ "loss": 1.546,
+ "theoretical_loss": 3.816045639228518,
+ "tokens_seen": 685375488
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402567703109327,
+ "loss": 1.4558,
+ "theoretical_loss": 3.8160087289331006,
+ "tokens_seen": 685441024
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402407221664995,
+ "loss": 1.574,
+ "theoretical_loss": 3.815971823154586,
+ "tokens_seen": 685506560
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402246740220662,
+ "loss": 1.415,
+ "theoretical_loss": 3.81593492189199,
+ "tokens_seen": 685572096
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006402086258776329,
+ "loss": 1.4116,
+ "theoretical_loss": 3.8158980251443277,
+ "tokens_seen": 685637632
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006401925777331996,
+ "loss": 1.4347,
+ "theoretical_loss": 3.8158611329106162,
+ "tokens_seen": 685703168
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006401765295887663,
+ "loss": 1.3947,
+ "theoretical_loss": 3.815824245189871,
+ "tokens_seen": 685768704
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000640160481444333,
+ "loss": 1.5681,
+ "theoretical_loss": 3.81578736198111,
+ "tokens_seen": 685834240
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006401444332998997,
+ "loss": 1.4623,
+ "theoretical_loss": 3.815750483283349,
+ "tokens_seen": 685899776
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006401283851554664,
+ "loss": 1.4556,
+ "theoretical_loss": 3.8157136090956056,
+ "tokens_seen": 685965312
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006401123370110331,
+ "loss": 1.5815,
+ "theoretical_loss": 3.8156767394168982,
+ "tokens_seen": 686030848
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400962888665998,
+ "loss": 1.4877,
+ "theoretical_loss": 3.8156398742462443,
+ "tokens_seen": 686096384
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400802407221665,
+ "loss": 1.3404,
+ "theoretical_loss": 3.8156030135826615,
+ "tokens_seen": 686161920
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400641925777332,
+ "loss": 1.3622,
+ "theoretical_loss": 3.81556615742517,
+ "tokens_seen": 686227456
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400481444332999,
+ "loss": 1.55,
+ "theoretical_loss": 3.815529305772787,
+ "tokens_seen": 686292992
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400320962888667,
+ "loss": 1.4737,
+ "theoretical_loss": 3.815492458624532,
+ "tokens_seen": 686358528
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006400160481444333,
+ "loss": 1.4545,
+ "theoretical_loss": 3.8154556159794257,
+ "tokens_seen": 686424064
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00064,
+ "loss": 1.4074,
+ "theoretical_loss": 3.815418777836487,
+ "tokens_seen": 686489600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399839518555667,
+ "loss": 1.5461,
+ "theoretical_loss": 3.8153819441947356,
+ "tokens_seen": 686555136
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399679037111335,
+ "loss": 1.3217,
+ "theoretical_loss": 3.815345115053193,
+ "tokens_seen": 686620672
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399518555667001,
+ "loss": 1.4626,
+ "theoretical_loss": 3.815308290410879,
+ "tokens_seen": 686686208
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399358074222669,
+ "loss": 1.3287,
+ "theoretical_loss": 3.815271470266815,
+ "tokens_seen": 686751744
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399197592778335,
+ "loss": 1.522,
+ "theoretical_loss": 3.8152346546200224,
+ "tokens_seen": 686817280
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006399037111334003,
+ "loss": 1.5696,
+ "theoretical_loss": 3.815197843469523,
+ "tokens_seen": 686882816
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398876629889669,
+ "loss": 1.4541,
+ "theoretical_loss": 3.815161036814338,
+ "tokens_seen": 686948352
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398716148445337,
+ "loss": 1.4031,
+ "theoretical_loss": 3.8151242346534904,
+ "tokens_seen": 687013888
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398555667001003,
+ "loss": 1.4871,
+ "theoretical_loss": 3.8150874369860026,
+ "tokens_seen": 687079424
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398395185556671,
+ "loss": 1.5176,
+ "theoretical_loss": 3.8150506438108973,
+ "tokens_seen": 687144960
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398234704112337,
+ "loss": 1.4159,
+ "theoretical_loss": 3.815013855127198,
+ "tokens_seen": 687210496
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006398074222668005,
+ "loss": 1.3235,
+ "theoretical_loss": 3.814977070933928,
+ "tokens_seen": 687276032
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397913741223671,
+ "loss": 1.4435,
+ "theoretical_loss": 3.814940291230111,
+ "tokens_seen": 687341568
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397753259779339,
+ "loss": 1.4527,
+ "theoretical_loss": 3.8149035160147706,
+ "tokens_seen": 687407104
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397592778335005,
+ "loss": 1.4967,
+ "theoretical_loss": 3.8148667452869325,
+ "tokens_seen": 687472640
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397432296890673,
+ "loss": 1.4179,
+ "theoretical_loss": 3.81482997904562,
+ "tokens_seen": 687538176
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397271815446339,
+ "loss": 1.3892,
+ "theoretical_loss": 3.8147932172898598,
+ "tokens_seen": 687603712
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006397111334002007,
+ "loss": 1.5138,
+ "theoretical_loss": 3.814756460018675,
+ "tokens_seen": 687669248
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396950852557673,
+ "loss": 1.4843,
+ "theoretical_loss": 3.814719707231093,
+ "tokens_seen": 687734784
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396790371113341,
+ "loss": 1.3816,
+ "theoretical_loss": 3.814682958926139,
+ "tokens_seen": 687800320
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396629889669007,
+ "loss": 1.456,
+ "theoretical_loss": 3.8146462151028393,
+ "tokens_seen": 687865856
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396469408224675,
+ "loss": 1.4038,
+ "theoretical_loss": 3.8146094757602205,
+ "tokens_seen": 687931392
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396308926780341,
+ "loss": 1.3949,
+ "theoretical_loss": 3.814572740897309,
+ "tokens_seen": 687996928
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006396148445336009,
+ "loss": 1.2949,
+ "theoretical_loss": 3.814536010513133,
+ "tokens_seen": 688062464
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 234843,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4647266864776611,
+ "objective/train/theoretical_loss": 3.8144992846067187,
+ "objective/train/tokens_used": 708588000,
+ "theoretical_loss": 3.8144992846067187,
+ "tokens_seen": 688128000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395987963891675,
+ "loss": 1.4728,
+ "theoretical_loss": 3.8144992846067187,
+ "tokens_seen": 688128000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395827482447343,
+ "loss": 1.4255,
+ "theoretical_loss": 3.8144625631770945,
+ "tokens_seen": 688193536
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395667001003009,
+ "loss": 1.496,
+ "theoretical_loss": 3.8144258462232887,
+ "tokens_seen": 688259072
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395506519558677,
+ "loss": 1.5404,
+ "theoretical_loss": 3.8143891337443288,
+ "tokens_seen": 688324608
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395346038114343,
+ "loss": 1.5205,
+ "theoretical_loss": 3.8143524257392443,
+ "tokens_seen": 688390144
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395185556670011,
+ "loss": 1.4784,
+ "theoretical_loss": 3.814315722207064,
+ "tokens_seen": 688455680
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006395025075225677,
+ "loss": 1.4064,
+ "theoretical_loss": 3.814279023146817,
+ "tokens_seen": 688521216
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394864593781345,
+ "loss": 1.3965,
+ "theoretical_loss": 3.8142423285575324,
+ "tokens_seen": 688586752
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394704112337011,
+ "loss": 1.4664,
+ "theoretical_loss": 3.814205638438241,
+ "tokens_seen": 688652288
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394543630892679,
+ "loss": 1.3477,
+ "theoretical_loss": 3.814168952787972,
+ "tokens_seen": 688717824
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394383149448345,
+ "loss": 1.4602,
+ "theoretical_loss": 3.8141322716057564,
+ "tokens_seen": 688783360
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394222668004013,
+ "loss": 1.3104,
+ "theoretical_loss": 3.814095594890625,
+ "tokens_seen": 688848896
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006394062186559679,
+ "loss": 1.4389,
+ "theoretical_loss": 3.8140589226416095,
+ "tokens_seen": 688914432
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393901705115347,
+ "loss": 1.4372,
+ "theoretical_loss": 3.81402225485774,
+ "tokens_seen": 688979968
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393741223671013,
+ "loss": 1.4144,
+ "theoretical_loss": 3.8139855915380485,
+ "tokens_seen": 689045504
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393580742226681,
+ "loss": 1.3738,
+ "theoretical_loss": 3.813948932681568,
+ "tokens_seen": 689111040
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393420260782347,
+ "loss": 1.4324,
+ "theoretical_loss": 3.8139122782873294,
+ "tokens_seen": 689176576
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393259779338015,
+ "loss": 1.2945,
+ "theoretical_loss": 3.813875628354366,
+ "tokens_seen": 689242112
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006393099297893681,
+ "loss": 1.5289,
+ "theoretical_loss": 3.8138389828817116,
+ "tokens_seen": 689307648
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392938816449349,
+ "loss": 1.4435,
+ "theoretical_loss": 3.8138023418683975,
+ "tokens_seen": 689373184
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392778335005015,
+ "loss": 1.5137,
+ "theoretical_loss": 3.8137657053134584,
+ "tokens_seen": 689438720
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392617853560683,
+ "loss": 1.3912,
+ "theoretical_loss": 3.813729073215928,
+ "tokens_seen": 689504256
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392457372116349,
+ "loss": 1.447,
+ "theoretical_loss": 3.8136924455748398,
+ "tokens_seen": 689569792
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392296890672017,
+ "loss": 1.3175,
+ "theoretical_loss": 3.813655822389229,
+ "tokens_seen": 689635328
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006392136409227683,
+ "loss": 1.4715,
+ "theoretical_loss": 3.8136192036581296,
+ "tokens_seen": 689700864
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391975927783351,
+ "loss": 1.4261,
+ "theoretical_loss": 3.813582589380577,
+ "tokens_seen": 689766400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391815446339017,
+ "loss": 1.4151,
+ "theoretical_loss": 3.8135459795556064,
+ "tokens_seen": 689831936
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391654964894685,
+ "loss": 1.5456,
+ "theoretical_loss": 3.8135093741822534,
+ "tokens_seen": 689897472
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391494483450351,
+ "loss": 1.5161,
+ "theoretical_loss": 3.813472773259554,
+ "tokens_seen": 689963008
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391334002006019,
+ "loss": 1.4062,
+ "theoretical_loss": 3.8134361767865443,
+ "tokens_seen": 690028544
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391173520561685,
+ "loss": 1.38,
+ "theoretical_loss": 3.8133995847622613,
+ "tokens_seen": 690094080
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006391013039117353,
+ "loss": 1.5294,
+ "theoretical_loss": 3.8133629971857403,
+ "tokens_seen": 690159616
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390852557673019,
+ "loss": 1.4732,
+ "theoretical_loss": 3.8133264140560197,
+ "tokens_seen": 690225152
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390692076228687,
+ "loss": 1.4324,
+ "theoretical_loss": 3.813289835372137,
+ "tokens_seen": 690290688
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390531594784353,
+ "loss": 1.4654,
+ "theoretical_loss": 3.81325326113313,
+ "tokens_seen": 690356224
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390371113340021,
+ "loss": 1.4102,
+ "theoretical_loss": 3.8132166913380354,
+ "tokens_seen": 690421760
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390210631895687,
+ "loss": 1.4409,
+ "theoretical_loss": 3.8131801259858924,
+ "tokens_seen": 690487296
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006390050150451354,
+ "loss": 1.4745,
+ "theoretical_loss": 3.8131435650757397,
+ "tokens_seen": 690552832
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389889669007021,
+ "loss": 1.373,
+ "theoretical_loss": 3.8131070086066154,
+ "tokens_seen": 690618368
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389729187562688,
+ "loss": 1.3771,
+ "theoretical_loss": 3.8130704565775604,
+ "tokens_seen": 690683904
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389568706118355,
+ "loss": 1.4569,
+ "theoretical_loss": 3.8130339089876126,
+ "tokens_seen": 690749440
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389408224674022,
+ "loss": 1.3497,
+ "theoretical_loss": 3.8129973658358125,
+ "tokens_seen": 690814976
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389247743229689,
+ "loss": 1.4776,
+ "theoretical_loss": 3.8129608271211994,
+ "tokens_seen": 690880512
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006389087261785356,
+ "loss": 1.4584,
+ "theoretical_loss": 3.8129242928428146,
+ "tokens_seen": 690946048
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006388926780341023,
+ "loss": 1.5172,
+ "theoretical_loss": 3.812887762999699,
+ "tokens_seen": 691011584
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638876629889669,
+ "loss": 1.4696,
+ "theoretical_loss": 3.8128512375908925,
+ "tokens_seen": 691077120
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006388605817452357,
+ "loss": 1.4223,
+ "theoretical_loss": 3.8128147166154376,
+ "tokens_seen": 691142656
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006388445336008024,
+ "loss": 1.4317,
+ "theoretical_loss": 3.8127782000723744,
+ "tokens_seen": 691208192
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006388284854563691,
+ "loss": 1.3205,
+ "theoretical_loss": 3.812741687960746,
+ "tokens_seen": 691273728
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006388124373119358,
+ "loss": 1.4841,
+ "theoretical_loss": 3.8127051802795946,
+ "tokens_seen": 691339264
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 235714,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3893272876739502,
+ "objective/train/theoretical_loss": 3.812668677027962,
+ "objective/train/tokens_used": 711864800,
+ "theoretical_loss": 3.812668677027962,
+ "tokens_seen": 691404800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006387963891675025,
+ "loss": 1.4677,
+ "theoretical_loss": 3.812668677027962,
+ "tokens_seen": 691404800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006387803410230692,
+ "loss": 1.4838,
+ "theoretical_loss": 3.8126321782048915,
+ "tokens_seen": 691470336
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638764292878636,
+ "loss": 1.4568,
+ "theoretical_loss": 3.8125956838094255,
+ "tokens_seen": 691535872
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006387482447342026,
+ "loss": 1.5229,
+ "theoretical_loss": 3.812559193840608,
+ "tokens_seen": 691601408
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006387321965897693,
+ "loss": 1.4393,
+ "theoretical_loss": 3.8125227082974824,
+ "tokens_seen": 691666944
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638716148445336,
+ "loss": 1.4725,
+ "theoretical_loss": 3.812486227179093,
+ "tokens_seen": 691732480
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006387001003009027,
+ "loss": 1.5032,
+ "theoretical_loss": 3.812449750484484,
+ "tokens_seen": 691798016
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006386840521564694,
+ "loss": 1.3823,
+ "theoretical_loss": 3.8124132782126994,
+ "tokens_seen": 691863552
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006386680040120362,
+ "loss": 1.3504,
+ "theoretical_loss": 3.812376810362785,
+ "tokens_seen": 691929088
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006386519558676028,
+ "loss": 1.4185,
+ "theoretical_loss": 3.8123403469337847,
+ "tokens_seen": 691994624
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006386359077231696,
+ "loss": 1.5098,
+ "theoretical_loss": 3.812303887924745,
+ "tokens_seen": 692060160
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006386198595787362,
+ "loss": 1.5385,
+ "theoretical_loss": 3.8122674333347115,
+ "tokens_seen": 692125696
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638603811434303,
+ "loss": 1.3335,
+ "theoretical_loss": 3.8122309831627295,
+ "tokens_seen": 692191232
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006385877632898696,
+ "loss": 1.4043,
+ "theoretical_loss": 3.8121945374078465,
+ "tokens_seen": 692256768
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006385717151454364,
+ "loss": 1.481,
+ "theoretical_loss": 3.8121580960691084,
+ "tokens_seen": 692322304
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638555667001003,
+ "loss": 1.5184,
+ "theoretical_loss": 3.812121659145562,
+ "tokens_seen": 692387840
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006385396188565698,
+ "loss": 1.5102,
+ "theoretical_loss": 3.8120852266362553,
+ "tokens_seen": 692453376
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006385235707121364,
+ "loss": 1.4312,
+ "theoretical_loss": 3.8120487985402347,
+ "tokens_seen": 692518912
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006385075225677032,
+ "loss": 1.4521,
+ "theoretical_loss": 3.8120123748565486,
+ "tokens_seen": 692584448
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006384914744232698,
+ "loss": 1.3721,
+ "theoretical_loss": 3.811975955584246,
+ "tokens_seen": 692649984
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006384754262788366,
+ "loss": 1.4386,
+ "theoretical_loss": 3.8119395407223737,
+ "tokens_seen": 692715520
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006384593781344032,
+ "loss": 1.3849,
+ "theoretical_loss": 3.8119031302699815,
+ "tokens_seen": 692781056
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00063844332998997,
+ "loss": 1.3678,
+ "theoretical_loss": 3.8118667242261175,
+ "tokens_seen": 692846592
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006384272818455366,
+ "loss": 1.3124,
+ "theoretical_loss": 3.8118303225898327,
+ "tokens_seen": 692912128
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006384112337011034,
+ "loss": 1.4687,
+ "theoretical_loss": 3.8117939253601745,
+ "tokens_seen": 692977664
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00063839518555667,
+ "loss": 1.4331,
+ "theoretical_loss": 3.811757532536194,
+ "tokens_seen": 693043200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006383791374122368,
+ "loss": 1.4734,
+ "theoretical_loss": 3.811721144116942,
+ "tokens_seen": 693108736
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006383630892678034,
+ "loss": 1.4495,
+ "theoretical_loss": 3.8116847601014676,
+ "tokens_seen": 693174272
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006383470411233702,
+ "loss": 1.3543,
+ "theoretical_loss": 3.811648380488822,
+ "tokens_seen": 693239808
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006383309929789368,
+ "loss": 1.483,
+ "theoretical_loss": 3.811612005278057,
+ "tokens_seen": 693305344
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006383149448345036,
+ "loss": 1.3364,
+ "theoretical_loss": 3.8115756344682232,
+ "tokens_seen": 693370880
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006382988966900702,
+ "loss": 1.3732,
+ "theoretical_loss": 3.811539268058372,
+ "tokens_seen": 693436416
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638282848545637,
+ "loss": 1.3328,
+ "theoretical_loss": 3.811502906047556,
+ "tokens_seen": 693501952
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006382668004012036,
+ "loss": 1.2924,
+ "theoretical_loss": 3.8114665484348276,
+ "tokens_seen": 693567488
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006382507522567704,
+ "loss": 1.4352,
+ "theoretical_loss": 3.8114301952192386,
+ "tokens_seen": 693633024
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638234704112337,
+ "loss": 1.3933,
+ "theoretical_loss": 3.8113938463998425,
+ "tokens_seen": 693698560
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006382186559679038,
+ "loss": 1.4227,
+ "theoretical_loss": 3.811357501975692,
+ "tokens_seen": 693764096
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006382026078234704,
+ "loss": 1.4534,
+ "theoretical_loss": 3.8113211619458403,
+ "tokens_seen": 693829632
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006381865596790372,
+ "loss": 1.4038,
+ "theoretical_loss": 3.811284826309342,
+ "tokens_seen": 693895168
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006381705115346038,
+ "loss": 1.4275,
+ "theoretical_loss": 3.81124849506525,
+ "tokens_seen": 693960704
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006381544633901706,
+ "loss": 1.4279,
+ "theoretical_loss": 3.811212168212619,
+ "tokens_seen": 694026240
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006381384152457372,
+ "loss": 1.5186,
+ "theoretical_loss": 3.8111758457505047,
+ "tokens_seen": 694091776
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638122367101304,
+ "loss": 1.3418,
+ "theoretical_loss": 3.8111395276779607,
+ "tokens_seen": 694157312
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006381063189568706,
+ "loss": 1.4026,
+ "theoretical_loss": 3.8111032139940417,
+ "tokens_seen": 694222848
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006380902708124374,
+ "loss": 1.4274,
+ "theoretical_loss": 3.811066904697805,
+ "tokens_seen": 694288384
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000638074222668004,
+ "loss": 1.4758,
+ "theoretical_loss": 3.811030599788304,
+ "tokens_seen": 694353920
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006380581745235708,
+ "loss": 1.3915,
+ "theoretical_loss": 3.810994299264597,
+ "tokens_seen": 694419456
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006380421263791374,
+ "loss": 1.4937,
+ "theoretical_loss": 3.810958003125739,
+ "tokens_seen": 694484992
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006380260782347042,
+ "loss": 1.3575,
+ "theoretical_loss": 3.8109217113707876,
+ "tokens_seen": 694550528
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006380100300902708,
+ "loss": 1.4403,
+ "theoretical_loss": 3.8108854239987986,
+ "tokens_seen": 694616064
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 236782,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2472243309020996,
+ "objective/train/theoretical_loss": 3.81084914100883,
+ "objective/train/tokens_used": 715141600,
+ "theoretical_loss": 3.81084914100883,
+ "tokens_seen": 694681600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006379939819458376,
+ "loss": 1.3429,
+ "theoretical_loss": 3.81084914100883,
+ "tokens_seen": 694681600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006379779338014042,
+ "loss": 1.4143,
+ "theoretical_loss": 3.810812862399939,
+ "tokens_seen": 694747136
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637961885656971,
+ "loss": 1.4951,
+ "theoretical_loss": 3.8107765881711835,
+ "tokens_seen": 694812672
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006379458375125376,
+ "loss": 1.3626,
+ "theoretical_loss": 3.8107403183216215,
+ "tokens_seen": 694878208
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006379297893681044,
+ "loss": 1.453,
+ "theoretical_loss": 3.810704052850311,
+ "tokens_seen": 694943744
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637913741223671,
+ "loss": 1.41,
+ "theoretical_loss": 3.8106677917563117,
+ "tokens_seen": 695009280
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378976930792378,
+ "loss": 1.4152,
+ "theoretical_loss": 3.810631535038682,
+ "tokens_seen": 695074816
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378816449348044,
+ "loss": 1.4707,
+ "theoretical_loss": 3.810595282696481,
+ "tokens_seen": 695140352
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378655967903712,
+ "loss": 1.3896,
+ "theoretical_loss": 3.810559034728769,
+ "tokens_seen": 695205888
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378495486459378,
+ "loss": 1.3994,
+ "theoretical_loss": 3.8105227911346042,
+ "tokens_seen": 695271424
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378335005015046,
+ "loss": 1.4259,
+ "theoretical_loss": 3.810486551913048,
+ "tokens_seen": 695336960
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006378174523570713,
+ "loss": 1.3895,
+ "theoretical_loss": 3.810450317063161,
+ "tokens_seen": 695402496
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637801404212638,
+ "loss": 1.4288,
+ "theoretical_loss": 3.8104140865840033,
+ "tokens_seen": 695468032
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006377853560682047,
+ "loss": 1.4986,
+ "theoretical_loss": 3.810377860474636,
+ "tokens_seen": 695533568
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006377693079237714,
+ "loss": 1.5363,
+ "theoretical_loss": 3.8103416387341205,
+ "tokens_seen": 695599104
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637753259779338,
+ "loss": 1.5343,
+ "theoretical_loss": 3.8103054213615186,
+ "tokens_seen": 695664640
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006377372116349048,
+ "loss": 1.3992,
+ "theoretical_loss": 3.8102692083558916,
+ "tokens_seen": 695730176
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006377211634904714,
+ "loss": 1.4556,
+ "theoretical_loss": 3.810232999716302,
+ "tokens_seen": 695795712
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006377051153460382,
+ "loss": 1.5742,
+ "theoretical_loss": 3.8101967954418123,
+ "tokens_seen": 695861248
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006376890672016048,
+ "loss": 1.4954,
+ "theoretical_loss": 3.8101605955314857,
+ "tokens_seen": 695926784
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006376730190571716,
+ "loss": 1.4589,
+ "theoretical_loss": 3.8101243999843843,
+ "tokens_seen": 695992320
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006376569709127382,
+ "loss": 1.4251,
+ "theoretical_loss": 3.8100882087995718,
+ "tokens_seen": 696057856
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637640922768305,
+ "loss": 1.369,
+ "theoretical_loss": 3.810052021976112,
+ "tokens_seen": 696123392
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006376248746238716,
+ "loss": 1.5036,
+ "theoretical_loss": 3.8100158395130683,
+ "tokens_seen": 696188928
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006376088264794383,
+ "loss": 1.5799,
+ "theoretical_loss": 3.809979661409505,
+ "tokens_seen": 696254464
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637592778335005,
+ "loss": 1.4678,
+ "theoretical_loss": 3.809943487664488,
+ "tokens_seen": 696320000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006375767301905717,
+ "loss": 1.3974,
+ "theoretical_loss": 3.8099073182770797,
+ "tokens_seen": 696385536
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006375606820461384,
+ "loss": 1.3969,
+ "theoretical_loss": 3.809871153246347,
+ "tokens_seen": 696451072
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006375446339017051,
+ "loss": 1.3703,
+ "theoretical_loss": 3.8098349925713544,
+ "tokens_seen": 696516608
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006375285857572718,
+ "loss": 1.389,
+ "theoretical_loss": 3.809798836251167,
+ "tokens_seen": 696582144
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006375125376128385,
+ "loss": 1.4636,
+ "theoretical_loss": 3.809762684284852,
+ "tokens_seen": 696647680
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374964894684052,
+ "loss": 1.5068,
+ "theoretical_loss": 3.809726536671475,
+ "tokens_seen": 696713216
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374804413239719,
+ "loss": 1.4978,
+ "theoretical_loss": 3.8096903934101025,
+ "tokens_seen": 696778752
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374643931795386,
+ "loss": 1.4403,
+ "theoretical_loss": 3.8096542544998018,
+ "tokens_seen": 696844288
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374483450351053,
+ "loss": 1.408,
+ "theoretical_loss": 3.8096181199396386,
+ "tokens_seen": 696909824
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637432296890672,
+ "loss": 1.5257,
+ "theoretical_loss": 3.8095819897286813,
+ "tokens_seen": 696975360
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374162487462387,
+ "loss": 1.2994,
+ "theoretical_loss": 3.8095458638659974,
+ "tokens_seen": 697040896
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006374002006018054,
+ "loss": 1.431,
+ "theoretical_loss": 3.809509742350655,
+ "tokens_seen": 697106432
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373841524573721,
+ "loss": 1.4339,
+ "theoretical_loss": 3.809473625181722,
+ "tokens_seen": 697171968
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373681043129388,
+ "loss": 1.3853,
+ "theoretical_loss": 3.809437512358267,
+ "tokens_seen": 697237504
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373520561685055,
+ "loss": 1.4096,
+ "theoretical_loss": 3.809401403879359,
+ "tokens_seen": 697303040
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373360080240722,
+ "loss": 1.4918,
+ "theoretical_loss": 3.8093652997440666,
+ "tokens_seen": 697368576
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373199598796389,
+ "loss": 1.4135,
+ "theoretical_loss": 3.80932919995146,
+ "tokens_seen": 697434112
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006373039117352057,
+ "loss": 1.4797,
+ "theoretical_loss": 3.8092931045006075,
+ "tokens_seen": 697499648
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006372878635907723,
+ "loss": 1.3962,
+ "theoretical_loss": 3.8092570133905808,
+ "tokens_seen": 697565184
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000637271815446339,
+ "loss": 1.5149,
+ "theoretical_loss": 3.809220926620448,
+ "tokens_seen": 697630720
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006372557673019057,
+ "loss": 1.5153,
+ "theoretical_loss": 3.809184844189282,
+ "tokens_seen": 697696256
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006372397191574725,
+ "loss": 1.4797,
+ "theoretical_loss": 3.8091487660961514,
+ "tokens_seen": 697761792
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006372236710130391,
+ "loss": 1.4721,
+ "theoretical_loss": 3.809112692340129,
+ "tokens_seen": 697827328
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006372076228686059,
+ "loss": 1.3884,
+ "theoretical_loss": 3.8090766229202853,
+ "tokens_seen": 697892864
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 238184,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4217039346694946,
+ "objective/train/theoretical_loss": 3.8090405578356927,
+ "objective/train/tokens_used": 718418400,
+ "theoretical_loss": 3.8090405578356927,
+ "tokens_seen": 697958400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371915747241725,
+ "loss": 1.3576,
+ "theoretical_loss": 3.8090405578356927,
+ "tokens_seen": 697958400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371755265797393,
+ "loss": 1.4004,
+ "theoretical_loss": 3.8090044970854215,
+ "tokens_seen": 698023936
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371594784353059,
+ "loss": 1.4644,
+ "theoretical_loss": 3.8089684406685453,
+ "tokens_seen": 698089472
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371434302908727,
+ "loss": 1.3212,
+ "theoretical_loss": 3.808932388584137,
+ "tokens_seen": 698155008
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371273821464393,
+ "loss": 1.3765,
+ "theoretical_loss": 3.808896340831268,
+ "tokens_seen": 698220544
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006371113340020061,
+ "loss": 1.4736,
+ "theoretical_loss": 3.808860297409012,
+ "tokens_seen": 698286080
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370952858575727,
+ "loss": 1.3824,
+ "theoretical_loss": 3.8088242583164433,
+ "tokens_seen": 698351616
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370792377131395,
+ "loss": 1.4751,
+ "theoretical_loss": 3.808788223552634,
+ "tokens_seen": 698417152
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370631895687061,
+ "loss": 1.5259,
+ "theoretical_loss": 3.8087521931166592,
+ "tokens_seen": 698482688
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370471414242729,
+ "loss": 1.4854,
+ "theoretical_loss": 3.8087161670075926,
+ "tokens_seen": 698548224
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370310932798395,
+ "loss": 1.5663,
+ "theoretical_loss": 3.8086801452245087,
+ "tokens_seen": 698613760
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006370150451354063,
+ "loss": 1.4904,
+ "theoretical_loss": 3.808644127766483,
+ "tokens_seen": 698679296
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369989969909729,
+ "loss": 1.4431,
+ "theoretical_loss": 3.808608114632589,
+ "tokens_seen": 698744832
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369829488465397,
+ "loss": 1.392,
+ "theoretical_loss": 3.808572105821904,
+ "tokens_seen": 698810368
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369669007021063,
+ "loss": 1.4019,
+ "theoretical_loss": 3.8085361013335026,
+ "tokens_seen": 698875904
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369508525576731,
+ "loss": 1.3516,
+ "theoretical_loss": 3.8085001011664605,
+ "tokens_seen": 698941440
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369348044132397,
+ "loss": 1.4292,
+ "theoretical_loss": 3.8084641053198545,
+ "tokens_seen": 699006976
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369187562688065,
+ "loss": 1.4497,
+ "theoretical_loss": 3.808428113792761,
+ "tokens_seen": 699072512
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006369027081243731,
+ "loss": 1.4372,
+ "theoretical_loss": 3.8083921265842564,
+ "tokens_seen": 699138048
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368866599799399,
+ "loss": 1.4794,
+ "theoretical_loss": 3.8083561436934183,
+ "tokens_seen": 699203584
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368706118355065,
+ "loss": 1.3279,
+ "theoretical_loss": 3.8083201651193237,
+ "tokens_seen": 699269120
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368545636910733,
+ "loss": 1.3738,
+ "theoretical_loss": 3.80828419086105,
+ "tokens_seen": 699334656
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368385155466399,
+ "loss": 1.4592,
+ "theoretical_loss": 3.808248220917676,
+ "tokens_seen": 699400192
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368224674022067,
+ "loss": 1.5415,
+ "theoretical_loss": 3.8082122552882796,
+ "tokens_seen": 699465728
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006368064192577733,
+ "loss": 1.4874,
+ "theoretical_loss": 3.808176293971939,
+ "tokens_seen": 699531264
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367903711133401,
+ "loss": 1.4419,
+ "theoretical_loss": 3.8081403369677322,
+ "tokens_seen": 699596800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367743229689067,
+ "loss": 1.4602,
+ "theoretical_loss": 3.8081043842747397,
+ "tokens_seen": 699662336
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367582748244735,
+ "loss": 1.2952,
+ "theoretical_loss": 3.8080684358920402,
+ "tokens_seen": 699727872
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367422266800401,
+ "loss": 1.4549,
+ "theoretical_loss": 3.8080324918187136,
+ "tokens_seen": 699793408
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367261785356069,
+ "loss": 1.4145,
+ "theoretical_loss": 3.8079965520538392,
+ "tokens_seen": 699858944
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006367101303911735,
+ "loss": 1.4283,
+ "theoretical_loss": 3.8079606165964974,
+ "tokens_seen": 699924480
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366940822467403,
+ "loss": 1.4507,
+ "theoretical_loss": 3.8079246854457693,
+ "tokens_seen": 699990016
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366780341023069,
+ "loss": 1.4945,
+ "theoretical_loss": 3.807888758600735,
+ "tokens_seen": 700055552
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366619859578737,
+ "loss": 1.4224,
+ "theoretical_loss": 3.8078528360604755,
+ "tokens_seen": 700121088
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366459378134403,
+ "loss": 1.3546,
+ "theoretical_loss": 3.8078169178240726,
+ "tokens_seen": 700186624
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366298896690071,
+ "loss": 1.4382,
+ "theoretical_loss": 3.8077810038906073,
+ "tokens_seen": 700252160
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006366138415245738,
+ "loss": 1.3947,
+ "theoretical_loss": 3.807745094259162,
+ "tokens_seen": 700317696
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365977933801405,
+ "loss": 1.3775,
+ "theoretical_loss": 3.807709188928819,
+ "tokens_seen": 700383232
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365817452357072,
+ "loss": 1.4585,
+ "theoretical_loss": 3.80767328789866,
+ "tokens_seen": 700448768
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365656970912739,
+ "loss": 1.4953,
+ "theoretical_loss": 3.807637391167768,
+ "tokens_seen": 700514304
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365496489468406,
+ "loss": 1.5442,
+ "theoretical_loss": 3.8076014987352265,
+ "tokens_seen": 700579840
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365336008024073,
+ "loss": 1.4671,
+ "theoretical_loss": 3.8075656106001183,
+ "tokens_seen": 700645376
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000636517552657974,
+ "loss": 1.4537,
+ "theoretical_loss": 3.807529726761527,
+ "tokens_seen": 700710912
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006365015045135407,
+ "loss": 1.4597,
+ "theoretical_loss": 3.807493847218537,
+ "tokens_seen": 700776448
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364854563691074,
+ "loss": 1.4342,
+ "theoretical_loss": 3.8074579719702317,
+ "tokens_seen": 700841984
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364694082246741,
+ "loss": 1.4122,
+ "theoretical_loss": 3.8074221010156957,
+ "tokens_seen": 700907520
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364533600802408,
+ "loss": 1.5221,
+ "theoretical_loss": 3.8073862343540137,
+ "tokens_seen": 700973056
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364373119358075,
+ "loss": 1.4289,
+ "theoretical_loss": 3.807350371984271,
+ "tokens_seen": 701038592
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364212637913742,
+ "loss": 1.491,
+ "theoretical_loss": 3.8073145139055526,
+ "tokens_seen": 701104128
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006364052156469409,
+ "loss": 1.4687,
+ "theoretical_loss": 3.8072786601169435,
+ "tokens_seen": 701169664
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 239083,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2331576347351074,
+ "objective/train/theoretical_loss": 3.807242810617531,
+ "objective/train/tokens_used": 721695200,
+ "theoretical_loss": 3.807242810617531,
+ "tokens_seen": 701235200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363891675025076,
+ "loss": 1.3397,
+ "theoretical_loss": 3.807242810617531,
+ "tokens_seen": 701235200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363731193580743,
+ "loss": 1.5871,
+ "theoretical_loss": 3.8072069654063996,
+ "tokens_seen": 701300736
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363570712136409,
+ "loss": 1.4155,
+ "theoretical_loss": 3.8071711244826365,
+ "tokens_seen": 701366272
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363410230692077,
+ "loss": 1.4349,
+ "theoretical_loss": 3.807135287845328,
+ "tokens_seen": 701431808
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363249749247743,
+ "loss": 1.4464,
+ "theoretical_loss": 3.807099455493561,
+ "tokens_seen": 701497344
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006363089267803411,
+ "loss": 1.5763,
+ "theoretical_loss": 3.807063627426424,
+ "tokens_seen": 701562880
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362928786359077,
+ "loss": 1.5114,
+ "theoretical_loss": 3.8070278036430025,
+ "tokens_seen": 701628416
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362768304914745,
+ "loss": 1.3981,
+ "theoretical_loss": 3.8069919841423854,
+ "tokens_seen": 701693952
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362607823470411,
+ "loss": 1.504,
+ "theoretical_loss": 3.8069561689236604,
+ "tokens_seen": 701759488
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362447342026079,
+ "loss": 1.4841,
+ "theoretical_loss": 3.8069203579859163,
+ "tokens_seen": 701825024
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362286860581745,
+ "loss": 1.4901,
+ "theoretical_loss": 3.806884551328242,
+ "tokens_seen": 701890560
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006362126379137412,
+ "loss": 1.3995,
+ "theoretical_loss": 3.806848748949725,
+ "tokens_seen": 701956096
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361965897693079,
+ "loss": 1.4441,
+ "theoretical_loss": 3.806812950849455,
+ "tokens_seen": 702021632
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361805416248746,
+ "loss": 1.5151,
+ "theoretical_loss": 3.806777157026522,
+ "tokens_seen": 702087168
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361644934804413,
+ "loss": 1.5374,
+ "theoretical_loss": 3.806741367480016,
+ "tokens_seen": 702152704
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000636148445336008,
+ "loss": 1.364,
+ "theoretical_loss": 3.806705582209026,
+ "tokens_seen": 702218240
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361323971915747,
+ "loss": 1.3221,
+ "theoretical_loss": 3.8066698012126423,
+ "tokens_seen": 702283776
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361163490471414,
+ "loss": 1.4006,
+ "theoretical_loss": 3.8066340244899566,
+ "tokens_seen": 702349312
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006361003009027081,
+ "loss": 1.426,
+ "theoretical_loss": 3.8065982520400583,
+ "tokens_seen": 702414848
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006360842527582748,
+ "loss": 1.3471,
+ "theoretical_loss": 3.80656248386204,
+ "tokens_seen": 702480384
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006360682046138415,
+ "loss": 1.4601,
+ "theoretical_loss": 3.8065267199549924,
+ "tokens_seen": 702545920
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006360521564694082,
+ "loss": 1.4746,
+ "theoretical_loss": 3.8064909603180066,
+ "tokens_seen": 702611456
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000636036108324975,
+ "loss": 1.4666,
+ "theoretical_loss": 3.8064552049501756,
+ "tokens_seen": 702676992
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006360200601805416,
+ "loss": 1.5028,
+ "theoretical_loss": 3.806419453850591,
+ "tokens_seen": 702742528
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006360040120361083,
+ "loss": 1.322,
+ "theoretical_loss": 3.806383707018345,
+ "tokens_seen": 702808064
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635987963891675,
+ "loss": 1.4048,
+ "theoretical_loss": 3.806347964452531,
+ "tokens_seen": 702873600
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006359719157472417,
+ "loss": 1.3752,
+ "theoretical_loss": 3.806312226152242,
+ "tokens_seen": 702939136
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006359558676028084,
+ "loss": 1.4836,
+ "theoretical_loss": 3.806276492116571,
+ "tokens_seen": 703004672
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006359398194583752,
+ "loss": 1.4695,
+ "theoretical_loss": 3.806240762344612,
+ "tokens_seen": 703070208
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006359237713139418,
+ "loss": 1.4428,
+ "theoretical_loss": 3.806205036835459,
+ "tokens_seen": 703135744
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006359077231695086,
+ "loss": 1.384,
+ "theoretical_loss": 3.806169315588206,
+ "tokens_seen": 703201280
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006358916750250752,
+ "loss": 1.408,
+ "theoretical_loss": 3.8061335986019467,
+ "tokens_seen": 703266816
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635875626880642,
+ "loss": 1.4797,
+ "theoretical_loss": 3.806097885875777,
+ "tokens_seen": 703332352
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006358595787362086,
+ "loss": 1.439,
+ "theoretical_loss": 3.8060621774087915,
+ "tokens_seen": 703397888
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006358435305917754,
+ "loss": 1.4779,
+ "theoretical_loss": 3.8060264732000846,
+ "tokens_seen": 703463424
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635827482447342,
+ "loss": 1.5238,
+ "theoretical_loss": 3.8059907732487535,
+ "tokens_seen": 703528960
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006358114343029088,
+ "loss": 1.3687,
+ "theoretical_loss": 3.8059550775538926,
+ "tokens_seen": 703594496
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006357953861584754,
+ "loss": 1.5413,
+ "theoretical_loss": 3.805919386114599,
+ "tokens_seen": 703660032
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006357793380140422,
+ "loss": 1.3239,
+ "theoretical_loss": 3.8058836989299683,
+ "tokens_seen": 703725568
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006357632898696088,
+ "loss": 1.5187,
+ "theoretical_loss": 3.8058480159990977,
+ "tokens_seen": 703791104
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006357472417251756,
+ "loss": 1.438,
+ "theoretical_loss": 3.805812337321084,
+ "tokens_seen": 703856640
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006357311935807422,
+ "loss": 1.4494,
+ "theoretical_loss": 3.805776662895024,
+ "tokens_seen": 703922176
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635715145436309,
+ "loss": 1.4415,
+ "theoretical_loss": 3.8057409927200156,
+ "tokens_seen": 703987712
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356990972918756,
+ "loss": 1.2744,
+ "theoretical_loss": 3.805705326795157,
+ "tokens_seen": 704053248
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356830491474424,
+ "loss": 1.4325,
+ "theoretical_loss": 3.805669665119545,
+ "tokens_seen": 704118784
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635667001003009,
+ "loss": 1.3515,
+ "theoretical_loss": 3.8056340076922797,
+ "tokens_seen": 704184320
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356509528585758,
+ "loss": 1.5127,
+ "theoretical_loss": 3.8055983545124583,
+ "tokens_seen": 704249856
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356349047141424,
+ "loss": 1.4126,
+ "theoretical_loss": 3.805562705579179,
+ "tokens_seen": 704315392
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356188565697092,
+ "loss": 1.4153,
+ "theoretical_loss": 3.805527060891543,
+ "tokens_seen": 704380928
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006356028084252758,
+ "loss": 1.3536,
+ "theoretical_loss": 3.8054914204486487,
+ "tokens_seen": 704446464
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 240373,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5571424961090088,
+ "objective/train/theoretical_loss": 3.8054557842495957,
+ "objective/train/tokens_used": 724972000,
+ "theoretical_loss": 3.8054557842495957,
+ "tokens_seen": 704512000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006355867602808426,
+ "loss": 1.4782,
+ "theoretical_loss": 3.8054557842495957,
+ "tokens_seen": 704512000
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006355707121364092,
+ "loss": 1.4989,
+ "theoretical_loss": 3.8054201522934843,
+ "tokens_seen": 704577536
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635554663991976,
+ "loss": 1.5441,
+ "theoretical_loss": 3.805384524579414,
+ "tokens_seen": 704643072
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006355386158475426,
+ "loss": 1.5061,
+ "theoretical_loss": 3.805348901106486,
+ "tokens_seen": 704708608
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006355225677031094,
+ "loss": 1.4131,
+ "theoretical_loss": 3.8053132818738002,
+ "tokens_seen": 704774144
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635506519558676,
+ "loss": 1.4338,
+ "theoretical_loss": 3.805277666880459,
+ "tokens_seen": 704839680
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354904714142428,
+ "loss": 1.4124,
+ "theoretical_loss": 3.8052420561255627,
+ "tokens_seen": 704905216
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354744232698094,
+ "loss": 1.5405,
+ "theoretical_loss": 3.805206449608214,
+ "tokens_seen": 704970752
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354583751253762,
+ "loss": 1.5212,
+ "theoretical_loss": 3.8051708473275134,
+ "tokens_seen": 705036288
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354423269809428,
+ "loss": 1.4513,
+ "theoretical_loss": 3.805135249282564,
+ "tokens_seen": 705101824
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354262788365096,
+ "loss": 1.4789,
+ "theoretical_loss": 3.8050996554724676,
+ "tokens_seen": 705167360
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006354102306920763,
+ "loss": 1.513,
+ "theoretical_loss": 3.8050640658963277,
+ "tokens_seen": 705232896
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635394182547643,
+ "loss": 1.4029,
+ "theoretical_loss": 3.8050284805532466,
+ "tokens_seen": 705298432
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006353781344032097,
+ "loss": 1.5177,
+ "theoretical_loss": 3.804992899442328,
+ "tokens_seen": 705363968
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006353620862587764,
+ "loss": 1.4642,
+ "theoretical_loss": 3.804957322562675,
+ "tokens_seen": 705429504
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006353460381143431,
+ "loss": 1.3885,
+ "theoretical_loss": 3.804921749913392,
+ "tokens_seen": 705495040
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006353299899699098,
+ "loss": 1.4526,
+ "theoretical_loss": 3.804886181493582,
+ "tokens_seen": 705560576
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006353139418254765,
+ "loss": 1.3952,
+ "theoretical_loss": 3.8048506173023506,
+ "tokens_seen": 705626112
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352978936810432,
+ "loss": 1.3474,
+ "theoretical_loss": 3.8048150573388018,
+ "tokens_seen": 705691648
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352818455366099,
+ "loss": 1.2652,
+ "theoretical_loss": 3.8047795016020407,
+ "tokens_seen": 705757184
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352657973921766,
+ "loss": 1.4717,
+ "theoretical_loss": 3.804743950091172,
+ "tokens_seen": 705822720
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352497492477433,
+ "loss": 1.4024,
+ "theoretical_loss": 3.8047084028053013,
+ "tokens_seen": 705888256
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.00063523370110331,
+ "loss": 1.3059,
+ "theoretical_loss": 3.8046728597435346,
+ "tokens_seen": 705953792
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352176529588767,
+ "loss": 1.4265,
+ "theoretical_loss": 3.8046373209049777,
+ "tokens_seen": 706019328
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006352016048144434,
+ "loss": 1.4273,
+ "theoretical_loss": 3.804601786288737,
+ "tokens_seen": 706084864
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351855566700101,
+ "loss": 1.4164,
+ "theoretical_loss": 3.804566255893919,
+ "tokens_seen": 706150400
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351695085255768,
+ "loss": 1.409,
+ "theoretical_loss": 3.8045307297196302,
+ "tokens_seen": 706215936
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351534603811435,
+ "loss": 1.5206,
+ "theoretical_loss": 3.804495207764978,
+ "tokens_seen": 706281472
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351374122367102,
+ "loss": 1.4473,
+ "theoretical_loss": 3.804459690029069,
+ "tokens_seen": 706347008
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351213640922769,
+ "loss": 1.5032,
+ "theoretical_loss": 3.804424176511012,
+ "tokens_seen": 706412544
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006351053159478436,
+ "loss": 1.3958,
+ "theoretical_loss": 3.8043886672099148,
+ "tokens_seen": 706478080
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006350892678034103,
+ "loss": 1.516,
+ "theoretical_loss": 3.804353162124884,
+ "tokens_seen": 706543616
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000635073219658977,
+ "loss": 1.2106,
+ "theoretical_loss": 3.8043176612550296,
+ "tokens_seen": 706609152
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006350571715145437,
+ "loss": 1.3856,
+ "theoretical_loss": 3.80428216459946,
+ "tokens_seen": 706674688
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006350411233701104,
+ "loss": 1.3575,
+ "theoretical_loss": 3.8042466721572836,
+ "tokens_seen": 706740224
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006350250752256771,
+ "loss": 1.3736,
+ "theoretical_loss": 3.80421118392761,
+ "tokens_seen": 706805760
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006350090270812438,
+ "loss": 1.3582,
+ "theoretical_loss": 3.804175699909549,
+ "tokens_seen": 706871296
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006349929789368105,
+ "loss": 1.4533,
+ "theoretical_loss": 3.80414022010221,
+ "tokens_seen": 706936832
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006349769307923772,
+ "loss": 1.4468,
+ "theoretical_loss": 3.804104744504703,
+ "tokens_seen": 707002368
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006349608826479438,
+ "loss": 1.4524,
+ "theoretical_loss": 3.8040692731161383,
+ "tokens_seen": 707067904
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006349448345035106,
+ "loss": 1.4523,
+ "theoretical_loss": 3.8040338059356267,
+ "tokens_seen": 707133440
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006349287863590772,
+ "loss": 1.4276,
+ "theoretical_loss": 3.803998342962279,
+ "tokens_seen": 707198976
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000634912738214644,
+ "loss": 1.3684,
+ "theoretical_loss": 3.8039628841952067,
+ "tokens_seen": 707264512
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348966900702106,
+ "loss": 1.4081,
+ "theoretical_loss": 3.8039274296335206,
+ "tokens_seen": 707330048
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348806419257774,
+ "loss": 1.4177,
+ "theoretical_loss": 3.803891979276332,
+ "tokens_seen": 707395584
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000634864593781344,
+ "loss": 1.4522,
+ "theoretical_loss": 3.8038565331227545,
+ "tokens_seen": 707461120
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348485456369108,
+ "loss": 1.3709,
+ "theoretical_loss": 3.8038210911718986,
+ "tokens_seen": 707526656
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348324974924774,
+ "loss": 1.4075,
+ "theoretical_loss": 3.803785653422878,
+ "tokens_seen": 707592192
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348164493480441,
+ "loss": 1.4133,
+ "theoretical_loss": 3.8037502198748046,
+ "tokens_seen": 707657728
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006348004012036108,
+ "loss": 1.3512,
+ "theoretical_loss": 3.8037147905267914,
+ "tokens_seen": 707723264
+ },
+ {
+ "epoch": 0.21,
+ "objective/train/docs_used": 241367,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5710557699203491,
+ "objective/train/theoretical_loss": 3.8036793653779526,
+ "objective/train/tokens_used": 728248800,
+ "theoretical_loss": 3.8036793653779526,
+ "tokens_seen": 707788800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006347843530591775,
+ "loss": 1.4849,
+ "theoretical_loss": 3.8036793653779526,
+ "tokens_seen": 707788800
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006347683049147442,
+ "loss": 1.4448,
+ "theoretical_loss": 3.8036439444274004,
+ "tokens_seen": 707854336
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006347522567703109,
+ "loss": 1.5149,
+ "theoretical_loss": 3.80360852767425,
+ "tokens_seen": 707919872
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006347362086258776,
+ "loss": 1.4236,
+ "theoretical_loss": 3.803573115117615,
+ "tokens_seen": 707985408
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006347201604814443,
+ "loss": 1.4195,
+ "theoretical_loss": 3.803537706756609,
+ "tokens_seen": 708050944
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000634704112337011,
+ "loss": 1.5822,
+ "theoretical_loss": 3.8035023025903483,
+ "tokens_seen": 708116480
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346880641925777,
+ "loss": 1.4559,
+ "theoretical_loss": 3.803466902617946,
+ "tokens_seen": 708182016
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346720160481444,
+ "loss": 1.4631,
+ "theoretical_loss": 3.8034315068385185,
+ "tokens_seen": 708247552
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346559679037111,
+ "loss": 1.388,
+ "theoretical_loss": 3.8033961152511804,
+ "tokens_seen": 708313088
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346399197592778,
+ "loss": 1.4286,
+ "theoretical_loss": 3.8033607278550483,
+ "tokens_seen": 708378624
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346238716148445,
+ "loss": 1.4811,
+ "theoretical_loss": 3.803325344649237,
+ "tokens_seen": 708444160
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006346078234704112,
+ "loss": 1.4126,
+ "theoretical_loss": 3.8032899656328647,
+ "tokens_seen": 708509696
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006345917753259779,
+ "loss": 1.3167,
+ "theoretical_loss": 3.8032545908050457,
+ "tokens_seen": 708575232
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006345757271815446,
+ "loss": 1.5087,
+ "theoretical_loss": 3.803219220164898,
+ "tokens_seen": 708640768
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006345596790371113,
+ "loss": 1.4969,
+ "theoretical_loss": 3.8031838537115386,
+ "tokens_seen": 708706304
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.000634543630892678,
+ "loss": 1.4194,
+ "theoretical_loss": 3.8031484914440847,
+ "tokens_seen": 708771840
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006345275827482447,
+ "loss": 1.4666,
+ "theoretical_loss": 3.803113133361654,
+ "tokens_seen": 708837376
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006345115346038115,
+ "loss": 1.3104,
+ "theoretical_loss": 3.8030777794633632,
+ "tokens_seen": 708902912
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344954864593781,
+ "loss": 1.3697,
+ "theoretical_loss": 3.8030424297483325,
+ "tokens_seen": 708968448
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344794383149449,
+ "loss": 1.4544,
+ "theoretical_loss": 3.803007084215678,
+ "tokens_seen": 709033984
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344633901705115,
+ "loss": 1.4216,
+ "theoretical_loss": 3.8029717428645204,
+ "tokens_seen": 709099520
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344473420260783,
+ "loss": 1.3437,
+ "theoretical_loss": 3.802936405693978,
+ "tokens_seen": 709165056
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344312938816449,
+ "loss": 1.5726,
+ "theoretical_loss": 3.802901072703169,
+ "tokens_seen": 709230592
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006344152457372117,
+ "loss": 1.5667,
+ "theoretical_loss": 3.802865743891214,
+ "tokens_seen": 709296128
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006343991975927783,
+ "loss": 1.5112,
+ "theoretical_loss": 3.8028304192572326,
+ "tokens_seen": 709361664
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006343831494483451,
+ "loss": 1.4424,
+ "theoretical_loss": 3.802795098800344,
+ "tokens_seen": 709427200
+ },
+ {
+ "epoch": 0.21,
+ "learning_rate": 0.0006343671013039117,
+ "loss": 1.5096,
+ "theoretical_loss": 3.8027597825196695,
+ "tokens_seen": 709492736
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006343510531594785,
+ "loss": 1.4287,
+ "theoretical_loss": 3.802724470414329,
+ "tokens_seen": 709558272
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006343350050150451,
+ "loss": 1.3461,
+ "theoretical_loss": 3.8026891624834427,
+ "tokens_seen": 709623808
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006343189568706119,
+ "loss": 1.4013,
+ "theoretical_loss": 3.802653858726133,
+ "tokens_seen": 709689344
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006343029087261785,
+ "loss": 1.3793,
+ "theoretical_loss": 3.8026185591415205,
+ "tokens_seen": 709754880
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342868605817453,
+ "loss": 1.4748,
+ "theoretical_loss": 3.802583263728727,
+ "tokens_seen": 709820416
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342708124373119,
+ "loss": 1.4307,
+ "theoretical_loss": 3.802547972486874,
+ "tokens_seen": 709885952
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342547642928787,
+ "loss": 1.4739,
+ "theoretical_loss": 3.802512685415084,
+ "tokens_seen": 709951488
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342387161484453,
+ "loss": 1.4624,
+ "theoretical_loss": 3.802477402512479,
+ "tokens_seen": 710017024
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342226680040121,
+ "loss": 1.4674,
+ "theoretical_loss": 3.8024421237781825,
+ "tokens_seen": 710082560
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006342066198595788,
+ "loss": 1.5095,
+ "theoretical_loss": 3.8024068492113163,
+ "tokens_seen": 710148096
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006341905717151455,
+ "loss": 1.5026,
+ "theoretical_loss": 3.802371578811004,
+ "tokens_seen": 710213632
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006341745235707122,
+ "loss": 1.3745,
+ "theoretical_loss": 3.8023363125763696,
+ "tokens_seen": 710279168
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006341584754262789,
+ "loss": 1.3888,
+ "theoretical_loss": 3.8023010505065358,
+ "tokens_seen": 710344704
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006341424272818456,
+ "loss": 1.4819,
+ "theoretical_loss": 3.802265792600627,
+ "tokens_seen": 710410240
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006341263791374123,
+ "loss": 1.4422,
+ "theoretical_loss": 3.8022305388577684,
+ "tokens_seen": 710475776
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000634110330992979,
+ "loss": 1.4109,
+ "theoretical_loss": 3.8021952892770834,
+ "tokens_seen": 710541312
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340942828485457,
+ "loss": 1.4645,
+ "theoretical_loss": 3.802160043857697,
+ "tokens_seen": 710606848
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340782347041124,
+ "loss": 1.4037,
+ "theoretical_loss": 3.802124802598734,
+ "tokens_seen": 710672384
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340621865596791,
+ "loss": 1.3728,
+ "theoretical_loss": 3.8020895654993203,
+ "tokens_seen": 710737920
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340461384152458,
+ "loss": 1.4698,
+ "theoretical_loss": 3.8020543325585807,
+ "tokens_seen": 710803456
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340300902708125,
+ "loss": 1.4225,
+ "theoretical_loss": 3.802019103775642,
+ "tokens_seen": 710868992
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006340140421263792,
+ "loss": 1.3455,
+ "theoretical_loss": 3.8019838791496303,
+ "tokens_seen": 710934528
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339979939819459,
+ "loss": 1.3481,
+ "theoretical_loss": 3.8019486586796707,
+ "tokens_seen": 711000064
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 242639,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.499024510383606,
+ "objective/train/theoretical_loss": 3.8019134423648904,
+ "objective/train/tokens_used": 731525600,
+ "theoretical_loss": 3.8019134423648904,
+ "tokens_seen": 711065600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339819458375126,
+ "loss": 1.4308,
+ "theoretical_loss": 3.8019134423648904,
+ "tokens_seen": 711065600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339658976930793,
+ "loss": 1.3665,
+ "theoretical_loss": 3.8018782302044167,
+ "tokens_seen": 711131136
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633949849548646,
+ "loss": 1.3976,
+ "theoretical_loss": 3.801843022197377,
+ "tokens_seen": 711196672
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339338014042127,
+ "loss": 1.4437,
+ "theoretical_loss": 3.801807818342898,
+ "tokens_seen": 711262208
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339177532597794,
+ "loss": 1.4583,
+ "theoretical_loss": 3.8017726186401077,
+ "tokens_seen": 711327744
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006339017051153461,
+ "loss": 1.3954,
+ "theoretical_loss": 3.8017374230881336,
+ "tokens_seen": 711393280
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338856569709128,
+ "loss": 1.4625,
+ "theoretical_loss": 3.8017022316861047,
+ "tokens_seen": 711458816
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338696088264795,
+ "loss": 1.4932,
+ "theoretical_loss": 3.801667044433149,
+ "tokens_seen": 711524352
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338535606820462,
+ "loss": 1.3748,
+ "theoretical_loss": 3.801631861328395,
+ "tokens_seen": 711589888
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338375125376129,
+ "loss": 1.5972,
+ "theoretical_loss": 3.801596682370972,
+ "tokens_seen": 711655424
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338214643931796,
+ "loss": 1.3926,
+ "theoretical_loss": 3.8015615075600095,
+ "tokens_seen": 711720960
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006338054162487463,
+ "loss": 1.4615,
+ "theoretical_loss": 3.8015263368946366,
+ "tokens_seen": 711786496
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633789368104313,
+ "loss": 1.6014,
+ "theoretical_loss": 3.8014911703739833,
+ "tokens_seen": 711852032
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006337733199598797,
+ "loss": 1.4169,
+ "theoretical_loss": 3.8014560079971798,
+ "tokens_seen": 711917568
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006337572718154464,
+ "loss": 1.3755,
+ "theoretical_loss": 3.801420849763356,
+ "tokens_seen": 711983104
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006337412236710131,
+ "loss": 1.4058,
+ "theoretical_loss": 3.8013856956716428,
+ "tokens_seen": 712048640
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006337251755265798,
+ "loss": 1.4429,
+ "theoretical_loss": 3.8013505457211707,
+ "tokens_seen": 712114176
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006337091273821465,
+ "loss": 1.4631,
+ "theoretical_loss": 3.801315399911071,
+ "tokens_seen": 712179712
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006336930792377132,
+ "loss": 1.3433,
+ "theoretical_loss": 3.8012802582404754,
+ "tokens_seen": 712245248
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006336770310932799,
+ "loss": 1.5574,
+ "theoretical_loss": 3.801245120708515,
+ "tokens_seen": 712310784
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006336609829488466,
+ "loss": 1.4838,
+ "theoretical_loss": 3.801209987314322,
+ "tokens_seen": 712376320
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006336449348044133,
+ "loss": 1.4883,
+ "theoretical_loss": 3.8011748580570286,
+ "tokens_seen": 712441856
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00063362888665998,
+ "loss": 1.4876,
+ "theoretical_loss": 3.8011397329357672,
+ "tokens_seen": 712507392
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006336128385155467,
+ "loss": 1.3607,
+ "theoretical_loss": 3.80110461194967,
+ "tokens_seen": 712572928
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335967903711134,
+ "loss": 1.4238,
+ "theoretical_loss": 3.8010694950978703,
+ "tokens_seen": 712638464
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335807422266801,
+ "loss": 1.3884,
+ "theoretical_loss": 3.801034382379501,
+ "tokens_seen": 712704000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335646940822468,
+ "loss": 1.4922,
+ "theoretical_loss": 3.800999273793696,
+ "tokens_seen": 712769536
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335486459378135,
+ "loss": 1.4384,
+ "theoretical_loss": 3.800964169339589,
+ "tokens_seen": 712835072
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335325977933801,
+ "loss": 1.3729,
+ "theoretical_loss": 3.8009290690163136,
+ "tokens_seen": 712900608
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335165496489469,
+ "loss": 1.4204,
+ "theoretical_loss": 3.8008939728230047,
+ "tokens_seen": 712966144
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006335005015045135,
+ "loss": 1.4367,
+ "theoretical_loss": 3.8008588807587955,
+ "tokens_seen": 713031680
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006334844533600803,
+ "loss": 1.4261,
+ "theoretical_loss": 3.8008237928228223,
+ "tokens_seen": 713097216
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006334684052156469,
+ "loss": 1.4637,
+ "theoretical_loss": 3.8007887090142196,
+ "tokens_seen": 713162752
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006334523570712137,
+ "loss": 1.3785,
+ "theoretical_loss": 3.800753629332122,
+ "tokens_seen": 713228288
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006334363089267803,
+ "loss": 1.3275,
+ "theoretical_loss": 3.800718553775666,
+ "tokens_seen": 713293824
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633420260782347,
+ "loss": 1.4474,
+ "theoretical_loss": 3.8006834823439863,
+ "tokens_seen": 713359360
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006334042126379137,
+ "loss": 1.456,
+ "theoretical_loss": 3.80064841503622,
+ "tokens_seen": 713424896
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006333881644934804,
+ "loss": 1.4341,
+ "theoretical_loss": 3.800613351851503,
+ "tokens_seen": 713490432
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006333721163490471,
+ "loss": 1.397,
+ "theoretical_loss": 3.800578292788972,
+ "tokens_seen": 713555968
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006333560682046138,
+ "loss": 1.4599,
+ "theoretical_loss": 3.800543237847764,
+ "tokens_seen": 713621504
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006333400200601805,
+ "loss": 1.4813,
+ "theoretical_loss": 3.8005081870270154,
+ "tokens_seen": 713687040
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006333239719157472,
+ "loss": 1.4561,
+ "theoretical_loss": 3.8004731403258645,
+ "tokens_seen": 713752576
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633307923771314,
+ "loss": 1.3359,
+ "theoretical_loss": 3.800438097743448,
+ "tokens_seen": 713818112
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006332918756268806,
+ "loss": 1.4228,
+ "theoretical_loss": 3.8004030592789046,
+ "tokens_seen": 713883648
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006332758274824473,
+ "loss": 1.5044,
+ "theoretical_loss": 3.8003680249313723,
+ "tokens_seen": 713949184
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633259779338014,
+ "loss": 1.3843,
+ "theoretical_loss": 3.800332994699989,
+ "tokens_seen": 714014720
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006332437311935807,
+ "loss": 1.3726,
+ "theoretical_loss": 3.8002979685838936,
+ "tokens_seen": 714080256
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006332276830491474,
+ "loss": 1.4088,
+ "theoretical_loss": 3.800262946582225,
+ "tokens_seen": 714145792
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006332116349047141,
+ "loss": 1.4281,
+ "theoretical_loss": 3.800227928694123,
+ "tokens_seen": 714211328
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006331955867602808,
+ "loss": 1.5304,
+ "theoretical_loss": 3.800192914918726,
+ "tokens_seen": 714276864
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 243571,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.517408847808838,
+ "objective/train/theoretical_loss": 3.800157905255175,
+ "objective/train/tokens_used": 734802400,
+ "theoretical_loss": 3.800157905255175,
+ "tokens_seen": 714342400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006331795386158476,
+ "loss": 1.5384,
+ "theoretical_loss": 3.800157905255175,
+ "tokens_seen": 714342400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006331634904714142,
+ "loss": 1.4538,
+ "theoretical_loss": 3.8001228997026084,
+ "tokens_seen": 714407936
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633147442326981,
+ "loss": 1.4125,
+ "theoretical_loss": 3.800087898260167,
+ "tokens_seen": 714473472
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006331313941825476,
+ "loss": 1.5228,
+ "theoretical_loss": 3.800052900926992,
+ "tokens_seen": 714539008
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006331153460381144,
+ "loss": 1.4806,
+ "theoretical_loss": 3.8000179077022236,
+ "tokens_seen": 714604544
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000633099297893681,
+ "loss": 1.5372,
+ "theoretical_loss": 3.7999829185850027,
+ "tokens_seen": 714670080
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330832497492478,
+ "loss": 1.3878,
+ "theoretical_loss": 3.7999479335744706,
+ "tokens_seen": 714735616
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330672016048144,
+ "loss": 1.4084,
+ "theoretical_loss": 3.799912952669769,
+ "tokens_seen": 714801152
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330511534603812,
+ "loss": 1.3532,
+ "theoretical_loss": 3.7998779758700394,
+ "tokens_seen": 714866688
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330351053159478,
+ "loss": 1.4402,
+ "theoretical_loss": 3.7998430031744235,
+ "tokens_seen": 714932224
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330190571715146,
+ "loss": 1.4258,
+ "theoretical_loss": 3.7998080345820644,
+ "tokens_seen": 714997760
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006330030090270813,
+ "loss": 1.3957,
+ "theoretical_loss": 3.7997730700921046,
+ "tokens_seen": 715063296
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632986960882648,
+ "loss": 1.3841,
+ "theoretical_loss": 3.7997381097036858,
+ "tokens_seen": 715128832
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006329709127382147,
+ "loss": 1.4965,
+ "theoretical_loss": 3.7997031534159524,
+ "tokens_seen": 715194368
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006329548645937814,
+ "loss": 1.412,
+ "theoretical_loss": 3.799668201228047,
+ "tokens_seen": 715259904
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006329388164493481,
+ "loss": 1.403,
+ "theoretical_loss": 3.7996332531391133,
+ "tokens_seen": 715325440
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006329227683049148,
+ "loss": 1.4338,
+ "theoretical_loss": 3.7995983091482954,
+ "tokens_seen": 715390976
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006329067201604815,
+ "loss": 1.384,
+ "theoretical_loss": 3.799563369254737,
+ "tokens_seen": 715456512
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006328906720160482,
+ "loss": 1.454,
+ "theoretical_loss": 3.7995284334575823,
+ "tokens_seen": 715522048
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006328746238716149,
+ "loss": 1.3987,
+ "theoretical_loss": 3.7994935017559763,
+ "tokens_seen": 715587584
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006328585757271816,
+ "loss": 1.4896,
+ "theoretical_loss": 3.799458574149064,
+ "tokens_seen": 715653120
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006328425275827483,
+ "loss": 1.4257,
+ "theoretical_loss": 3.7994236506359895,
+ "tokens_seen": 715718656
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632826479438315,
+ "loss": 1.4415,
+ "theoretical_loss": 3.7993887312158994,
+ "tokens_seen": 715784192
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006328104312938817,
+ "loss": 1.3706,
+ "theoretical_loss": 3.799353815887939,
+ "tokens_seen": 715849728
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327943831494484,
+ "loss": 1.4093,
+ "theoretical_loss": 3.799318904651254,
+ "tokens_seen": 715915264
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327783350050151,
+ "loss": 1.4663,
+ "theoretical_loss": 3.7992839975049906,
+ "tokens_seen": 715980800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327622868605818,
+ "loss": 1.581,
+ "theoretical_loss": 3.799249094448295,
+ "tokens_seen": 716046336
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327462387161485,
+ "loss": 1.4882,
+ "theoretical_loss": 3.799214195480314,
+ "tokens_seen": 716111872
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327301905717152,
+ "loss": 1.4267,
+ "theoretical_loss": 3.799179300600194,
+ "tokens_seen": 716177408
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006327141424272819,
+ "loss": 1.37,
+ "theoretical_loss": 3.7991444098070835,
+ "tokens_seen": 716242944
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326980942828486,
+ "loss": 1.5027,
+ "theoretical_loss": 3.7991095231001286,
+ "tokens_seen": 716308480
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326820461384153,
+ "loss": 1.5437,
+ "theoretical_loss": 3.7990746404784774,
+ "tokens_seen": 716374016
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632665997993982,
+ "loss": 1.39,
+ "theoretical_loss": 3.7990397619412786,
+ "tokens_seen": 716439552
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326499498495487,
+ "loss": 1.4869,
+ "theoretical_loss": 3.799004887487679,
+ "tokens_seen": 716505088
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326339017051154,
+ "loss": 1.4229,
+ "theoretical_loss": 3.798970017116828,
+ "tokens_seen": 716570624
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326178535606821,
+ "loss": 1.323,
+ "theoretical_loss": 3.7989351508278744,
+ "tokens_seen": 716636160
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006326018054162488,
+ "loss": 1.2717,
+ "theoretical_loss": 3.7989002886199663,
+ "tokens_seen": 716701696
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006325857572718155,
+ "loss": 1.3605,
+ "theoretical_loss": 3.7988654304922536,
+ "tokens_seen": 716767232
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006325697091273822,
+ "loss": 1.4774,
+ "theoretical_loss": 3.798830576443886,
+ "tokens_seen": 716832768
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006325536609829489,
+ "loss": 1.4702,
+ "theoretical_loss": 3.7987957264740118,
+ "tokens_seen": 716898304
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006325376128385156,
+ "loss": 1.4558,
+ "theoretical_loss": 3.7987608805817827,
+ "tokens_seen": 716963840
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006325215646940823,
+ "loss": 1.4032,
+ "theoretical_loss": 3.7987260387663486,
+ "tokens_seen": 717029376
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632505516549649,
+ "loss": 1.3756,
+ "theoretical_loss": 3.7986912010268585,
+ "tokens_seen": 717094912
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324894684052157,
+ "loss": 1.4841,
+ "theoretical_loss": 3.798656367362465,
+ "tokens_seen": 717160448
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324734202607824,
+ "loss": 1.3519,
+ "theoretical_loss": 3.798621537772318,
+ "tokens_seen": 717225984
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324573721163491,
+ "loss": 1.5479,
+ "theoretical_loss": 3.7985867122555694,
+ "tokens_seen": 717291520
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324413239719158,
+ "loss": 1.4504,
+ "theoretical_loss": 3.79855189081137,
+ "tokens_seen": 717357056
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324252758274825,
+ "loss": 1.5485,
+ "theoretical_loss": 3.7985170734388722,
+ "tokens_seen": 717422592
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006324092276830492,
+ "loss": 1.3741,
+ "theoretical_loss": 3.7984822601372272,
+ "tokens_seen": 717488128
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006323931795386159,
+ "loss": 1.3699,
+ "theoretical_loss": 3.7984474509055888,
+ "tokens_seen": 717553664
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 244811,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4988033771514893,
+ "objective/train/theoretical_loss": 3.7984126457431078,
+ "objective/train/tokens_used": 738079200,
+ "theoretical_loss": 3.7984126457431078,
+ "tokens_seen": 717619200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006323771313941826,
+ "loss": 1.4228,
+ "theoretical_loss": 3.7984126457431078,
+ "tokens_seen": 717619200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006323610832497493,
+ "loss": 1.5434,
+ "theoretical_loss": 3.798377844648938,
+ "tokens_seen": 717684736
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632345035105316,
+ "loss": 1.3938,
+ "theoretical_loss": 3.7983430476222315,
+ "tokens_seen": 717750272
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006323289869608827,
+ "loss": 1.3932,
+ "theoretical_loss": 3.7983082546621425,
+ "tokens_seen": 717815808
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006323129388164494,
+ "loss": 1.4452,
+ "theoretical_loss": 3.7982734657678243,
+ "tokens_seen": 717881344
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322968906720161,
+ "loss": 1.4019,
+ "theoretical_loss": 3.798238680938431,
+ "tokens_seen": 717946880
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322808425275828,
+ "loss": 1.4289,
+ "theoretical_loss": 3.798203900173116,
+ "tokens_seen": 718012416
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322647943831495,
+ "loss": 1.3972,
+ "theoretical_loss": 3.798169123471034,
+ "tokens_seen": 718077952
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322487462387162,
+ "loss": 1.3648,
+ "theoretical_loss": 3.798134350831339,
+ "tokens_seen": 718143488
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322326980942829,
+ "loss": 1.4046,
+ "theoretical_loss": 3.7980995822531862,
+ "tokens_seen": 718209024
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322166499498496,
+ "loss": 1.3455,
+ "theoretical_loss": 3.7980648177357312,
+ "tokens_seen": 718274560
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006322006018054163,
+ "loss": 1.3513,
+ "theoretical_loss": 3.7980300572781283,
+ "tokens_seen": 718340096
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632184553660983,
+ "loss": 1.4353,
+ "theoretical_loss": 3.797995300879534,
+ "tokens_seen": 718405632
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006321685055165497,
+ "loss": 1.3864,
+ "theoretical_loss": 3.797960548539103,
+ "tokens_seen": 718471168
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006321524573721164,
+ "loss": 1.4364,
+ "theoretical_loss": 3.7979258002559924,
+ "tokens_seen": 718536704
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000632136409227683,
+ "loss": 1.3891,
+ "theoretical_loss": 3.797891056029358,
+ "tokens_seen": 718602240
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006321203610832498,
+ "loss": 1.3073,
+ "theoretical_loss": 3.7978563158583567,
+ "tokens_seen": 718667776
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006321043129388164,
+ "loss": 1.5076,
+ "theoretical_loss": 3.797821579742145,
+ "tokens_seen": 718733312
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006320882647943832,
+ "loss": 1.3615,
+ "theoretical_loss": 3.7977868476798795,
+ "tokens_seen": 718798848
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006320722166499498,
+ "loss": 1.5054,
+ "theoretical_loss": 3.7977521196707187,
+ "tokens_seen": 718864384
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006320561685055166,
+ "loss": 1.435,
+ "theoretical_loss": 3.7977173957138195,
+ "tokens_seen": 718929920
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006320401203610832,
+ "loss": 1.3536,
+ "theoretical_loss": 3.79768267580834,
+ "tokens_seen": 718995456
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00063202407221665,
+ "loss": 1.4225,
+ "theoretical_loss": 3.797647959953438,
+ "tokens_seen": 719060992
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006320080240722166,
+ "loss": 1.3485,
+ "theoretical_loss": 3.7976132481482714,
+ "tokens_seen": 719126528
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006319919759277833,
+ "loss": 1.4536,
+ "theoretical_loss": 3.7975785403919997,
+ "tokens_seen": 719192064
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00063197592778335,
+ "loss": 1.3513,
+ "theoretical_loss": 3.797543836683781,
+ "tokens_seen": 719257600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006319598796389167,
+ "loss": 1.399,
+ "theoretical_loss": 3.797509137022775,
+ "tokens_seen": 719323136
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006319438314944834,
+ "loss": 1.3285,
+ "theoretical_loss": 3.797474441408141,
+ "tokens_seen": 719388672
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006319277833500501,
+ "loss": 1.4618,
+ "theoretical_loss": 3.797439749839038,
+ "tokens_seen": 719454208
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006319117352056168,
+ "loss": 1.3587,
+ "theoretical_loss": 3.7974050623146267,
+ "tokens_seen": 719519744
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006318956870611835,
+ "loss": 1.409,
+ "theoretical_loss": 3.7973703788340663,
+ "tokens_seen": 719585280
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006318796389167502,
+ "loss": 1.3907,
+ "theoretical_loss": 3.797335699396517,
+ "tokens_seen": 719650816
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006318635907723169,
+ "loss": 1.4135,
+ "theoretical_loss": 3.7973010240011407,
+ "tokens_seen": 719716352
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006318475426278836,
+ "loss": 1.3498,
+ "theoretical_loss": 3.7972663526470973,
+ "tokens_seen": 719781888
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006318314944834504,
+ "loss": 1.3264,
+ "theoretical_loss": 3.797231685333548,
+ "tokens_seen": 719847424
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631815446339017,
+ "loss": 1.3904,
+ "theoretical_loss": 3.7971970220596543,
+ "tokens_seen": 719912960
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317993981945838,
+ "loss": 1.5169,
+ "theoretical_loss": 3.7971623628245776,
+ "tokens_seen": 719978496
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317833500501505,
+ "loss": 1.3087,
+ "theoretical_loss": 3.79712770762748,
+ "tokens_seen": 720044032
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317673019057172,
+ "loss": 1.348,
+ "theoretical_loss": 3.797093056467523,
+ "tokens_seen": 720109568
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317512537612839,
+ "loss": 1.5558,
+ "theoretical_loss": 3.7970584093438697,
+ "tokens_seen": 720175104
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317352056168506,
+ "loss": 1.5001,
+ "theoretical_loss": 3.7970237662556823,
+ "tokens_seen": 720240640
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006317191574724173,
+ "loss": 1.5201,
+ "theoretical_loss": 3.7969891272021234,
+ "tokens_seen": 720306176
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631703109327984,
+ "loss": 1.4684,
+ "theoretical_loss": 3.796954492182357,
+ "tokens_seen": 720371712
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316870611835507,
+ "loss": 1.2957,
+ "theoretical_loss": 3.796919861195546,
+ "tokens_seen": 720437248
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316710130391174,
+ "loss": 1.4522,
+ "theoretical_loss": 3.796885234240853,
+ "tokens_seen": 720502784
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316549648946841,
+ "loss": 1.3751,
+ "theoretical_loss": 3.796850611317443,
+ "tokens_seen": 720568320
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316389167502508,
+ "loss": 1.3219,
+ "theoretical_loss": 3.79681599242448,
+ "tokens_seen": 720633856
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316228686058175,
+ "loss": 1.3869,
+ "theoretical_loss": 3.7967813775611283,
+ "tokens_seen": 720699392
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006316068204613842,
+ "loss": 1.5114,
+ "theoretical_loss": 3.796746766726552,
+ "tokens_seen": 720764928
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006315907723169509,
+ "loss": 1.4575,
+ "theoretical_loss": 3.7967121599199167,
+ "tokens_seen": 720830464
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 246110,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3479599952697754,
+ "objective/train/theoretical_loss": 3.7966775571403866,
+ "objective/train/tokens_used": 741356000,
+ "theoretical_loss": 3.7966775571403866,
+ "tokens_seen": 720896000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006315747241725176,
+ "loss": 1.4673,
+ "theoretical_loss": 3.7966775571403866,
+ "tokens_seen": 720896000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006315586760280843,
+ "loss": 1.4039,
+ "theoretical_loss": 3.7966429583871277,
+ "tokens_seen": 720961536
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631542627883651,
+ "loss": 1.4162,
+ "theoretical_loss": 3.7966083636593053,
+ "tokens_seen": 721027072
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006315265797392177,
+ "loss": 1.4734,
+ "theoretical_loss": 3.7965737729560853,
+ "tokens_seen": 721092608
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006315105315947844,
+ "loss": 1.4231,
+ "theoretical_loss": 3.7965391862766342,
+ "tokens_seen": 721158144
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314944834503511,
+ "loss": 1.4973,
+ "theoretical_loss": 3.7965046036201175,
+ "tokens_seen": 721223680
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314784353059178,
+ "loss": 1.4844,
+ "theoretical_loss": 3.7964700249857026,
+ "tokens_seen": 721289216
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314623871614845,
+ "loss": 1.5109,
+ "theoretical_loss": 3.796435450372556,
+ "tokens_seen": 721354752
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314463390170512,
+ "loss": 1.5156,
+ "theoretical_loss": 3.7964008797798443,
+ "tokens_seen": 721420288
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314302908726179,
+ "loss": 1.3812,
+ "theoretical_loss": 3.796366313206735,
+ "tokens_seen": 721485824
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006314142427281846,
+ "loss": 1.4729,
+ "theoretical_loss": 3.7963317506523966,
+ "tokens_seen": 721551360
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313981945837513,
+ "loss": 1.508,
+ "theoretical_loss": 3.796297192115996,
+ "tokens_seen": 721616896
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631382146439318,
+ "loss": 1.4971,
+ "theoretical_loss": 3.796262637596702,
+ "tokens_seen": 721682432
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313660982948847,
+ "loss": 1.404,
+ "theoretical_loss": 3.7962280870936818,
+ "tokens_seen": 721747968
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313500501504514,
+ "loss": 1.4409,
+ "theoretical_loss": 3.7961935406061054,
+ "tokens_seen": 721813504
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313340020060181,
+ "loss": 1.5395,
+ "theoretical_loss": 3.79615899813314,
+ "tokens_seen": 721879040
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313179538615848,
+ "loss": 1.3501,
+ "theoretical_loss": 3.7961244596739556,
+ "tokens_seen": 721944576
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006313019057171515,
+ "loss": 1.3767,
+ "theoretical_loss": 3.7960899252277214,
+ "tokens_seen": 722010112
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006312858575727182,
+ "loss": 1.4148,
+ "theoretical_loss": 3.7960553947936075,
+ "tokens_seen": 722075648
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006312698094282849,
+ "loss": 1.5147,
+ "theoretical_loss": 3.796020868370783,
+ "tokens_seen": 722141184
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006312537612838516,
+ "loss": 1.6155,
+ "theoretical_loss": 3.795986345958418,
+ "tokens_seen": 722206720
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006312377131394183,
+ "loss": 1.4997,
+ "theoretical_loss": 3.7959518275556827,
+ "tokens_seen": 722272256
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631221664994985,
+ "loss": 1.4589,
+ "theoretical_loss": 3.7959173131617483,
+ "tokens_seen": 722337792
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006312056168505517,
+ "loss": 1.546,
+ "theoretical_loss": 3.795882802775785,
+ "tokens_seen": 722403328
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311895687061184,
+ "loss": 1.3899,
+ "theoretical_loss": 3.7958482963969638,
+ "tokens_seen": 722468864
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311735205616851,
+ "loss": 1.4844,
+ "theoretical_loss": 3.7958137940244567,
+ "tokens_seen": 722534400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311574724172518,
+ "loss": 1.4567,
+ "theoretical_loss": 3.795779295657434,
+ "tokens_seen": 722599936
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311414242728185,
+ "loss": 1.4734,
+ "theoretical_loss": 3.7957448012950685,
+ "tokens_seen": 722665472
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311253761283852,
+ "loss": 1.4436,
+ "theoretical_loss": 3.795710310936532,
+ "tokens_seen": 722731008
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006311093279839519,
+ "loss": 1.4782,
+ "theoretical_loss": 3.7956758245809965,
+ "tokens_seen": 722796544
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006310932798395186,
+ "loss": 1.4097,
+ "theoretical_loss": 3.795641342227635,
+ "tokens_seen": 722862080
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006310772316950853,
+ "loss": 1.3761,
+ "theoretical_loss": 3.7956068638756197,
+ "tokens_seen": 722927616
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000631061183550652,
+ "loss": 1.3399,
+ "theoretical_loss": 3.795572389524124,
+ "tokens_seen": 722993152
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006310451354062187,
+ "loss": 1.3759,
+ "theoretical_loss": 3.795537919172321,
+ "tokens_seen": 723058688
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006310290872617854,
+ "loss": 1.4874,
+ "theoretical_loss": 3.795503452819384,
+ "tokens_seen": 723124224
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006310130391173521,
+ "loss": 1.41,
+ "theoretical_loss": 3.795468990464487,
+ "tokens_seen": 723189760
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309969909729188,
+ "loss": 1.3574,
+ "theoretical_loss": 3.795434532106804,
+ "tokens_seen": 723255296
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309809428284855,
+ "loss": 1.3829,
+ "theoretical_loss": 3.79540007774551,
+ "tokens_seen": 723320832
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309648946840522,
+ "loss": 1.5397,
+ "theoretical_loss": 3.795365627379778,
+ "tokens_seen": 723386368
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309488465396189,
+ "loss": 1.4052,
+ "theoretical_loss": 3.795331181008783,
+ "tokens_seen": 723451904
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309327983951856,
+ "loss": 1.4492,
+ "theoretical_loss": 3.795296738631701,
+ "tokens_seen": 723517440
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006309167502507523,
+ "loss": 1.565,
+ "theoretical_loss": 3.795262300247707,
+ "tokens_seen": 723582976
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000630900702106319,
+ "loss": 1.408,
+ "theoretical_loss": 3.795227865855975,
+ "tokens_seen": 723648512
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308846539618857,
+ "loss": 1.341,
+ "theoretical_loss": 3.7951934354556824,
+ "tokens_seen": 723714048
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308686058174524,
+ "loss": 1.5522,
+ "theoretical_loss": 3.7951590090460043,
+ "tokens_seen": 723779584
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308525576730191,
+ "loss": 1.508,
+ "theoretical_loss": 3.795124586626117,
+ "tokens_seen": 723845120
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308365095285858,
+ "loss": 1.4448,
+ "theoretical_loss": 3.7950901681951974,
+ "tokens_seen": 723910656
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308204613841525,
+ "loss": 1.4212,
+ "theoretical_loss": 3.795055753752422,
+ "tokens_seen": 723976192
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006308044132397192,
+ "loss": 1.4526,
+ "theoretical_loss": 3.795021343296967,
+ "tokens_seen": 724041728
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307883650952859,
+ "loss": 1.3965,
+ "theoretical_loss": 3.7949869368280105,
+ "tokens_seen": 724107264
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 247050,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4974820613861084,
+ "objective/train/theoretical_loss": 3.7949525343447297,
+ "objective/train/tokens_used": 744632800,
+ "theoretical_loss": 3.7949525343447297,
+ "tokens_seen": 724172800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307723169508526,
+ "loss": 1.5663,
+ "theoretical_loss": 3.7949525343447297,
+ "tokens_seen": 724172800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307562688064193,
+ "loss": 1.4248,
+ "theoretical_loss": 3.794918135846302,
+ "tokens_seen": 724238336
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307402206619859,
+ "loss": 1.4183,
+ "theoretical_loss": 3.7948837413319056,
+ "tokens_seen": 724303872
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307241725175527,
+ "loss": 1.4792,
+ "theoretical_loss": 3.7948493508007184,
+ "tokens_seen": 724369408
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006307081243731193,
+ "loss": 1.4819,
+ "theoretical_loss": 3.794814964251919,
+ "tokens_seen": 724434944
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306920762286861,
+ "loss": 1.4506,
+ "theoretical_loss": 3.7947805816846856,
+ "tokens_seen": 724500480
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306760280842527,
+ "loss": 1.3404,
+ "theoretical_loss": 3.7947462030981978,
+ "tokens_seen": 724566016
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306599799398195,
+ "loss": 1.3182,
+ "theoretical_loss": 3.794711828491634,
+ "tokens_seen": 724631552
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306439317953861,
+ "loss": 1.4068,
+ "theoretical_loss": 3.794677457864174,
+ "tokens_seen": 724697088
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306278836509529,
+ "loss": 1.297,
+ "theoretical_loss": 3.7946430912149975,
+ "tokens_seen": 724762624
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006306118355065195,
+ "loss": 1.4566,
+ "theoretical_loss": 3.794608728543284,
+ "tokens_seen": 724828160
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305957873620863,
+ "loss": 1.5626,
+ "theoretical_loss": 3.794574369848214,
+ "tokens_seen": 724893696
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305797392176529,
+ "loss": 1.3466,
+ "theoretical_loss": 3.794540015128968,
+ "tokens_seen": 724959232
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305636910732197,
+ "loss": 1.3054,
+ "theoretical_loss": 3.7945056643847255,
+ "tokens_seen": 725024768
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305476429287863,
+ "loss": 1.5714,
+ "theoretical_loss": 3.7944713176146685,
+ "tokens_seen": 725090304
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305315947843531,
+ "loss": 1.3652,
+ "theoretical_loss": 3.794436974817977,
+ "tokens_seen": 725155840
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006305155466399197,
+ "loss": 1.4185,
+ "theoretical_loss": 3.7944026359938334,
+ "tokens_seen": 725221376
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304994984954865,
+ "loss": 1.4229,
+ "theoretical_loss": 3.794368301141419,
+ "tokens_seen": 725286912
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304834503510531,
+ "loss": 1.356,
+ "theoretical_loss": 3.794333970259915,
+ "tokens_seen": 725352448
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304674022066199,
+ "loss": 1.3642,
+ "theoretical_loss": 3.794299643348504,
+ "tokens_seen": 725417984
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304513540621866,
+ "loss": 1.4366,
+ "theoretical_loss": 3.7942653204063683,
+ "tokens_seen": 725483520
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304353059177533,
+ "loss": 1.5196,
+ "theoretical_loss": 3.79423100143269,
+ "tokens_seen": 725549056
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00063041925777332,
+ "loss": 1.4281,
+ "theoretical_loss": 3.794196686426652,
+ "tokens_seen": 725614592
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006304032096288867,
+ "loss": 1.4114,
+ "theoretical_loss": 3.7941623753874376,
+ "tokens_seen": 725680128
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303871614844534,
+ "loss": 1.3399,
+ "theoretical_loss": 3.79412806831423,
+ "tokens_seen": 725745664
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303711133400201,
+ "loss": 1.4427,
+ "theoretical_loss": 3.7940937652062123,
+ "tokens_seen": 725811200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303550651955868,
+ "loss": 1.5487,
+ "theoretical_loss": 3.794059466062569,
+ "tokens_seen": 725876736
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303390170511535,
+ "loss": 1.4869,
+ "theoretical_loss": 3.7940251708824833,
+ "tokens_seen": 725942272
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303229689067202,
+ "loss": 1.3933,
+ "theoretical_loss": 3.79399087966514,
+ "tokens_seen": 726007808
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006303069207622869,
+ "loss": 1.474,
+ "theoretical_loss": 3.7939565924097227,
+ "tokens_seen": 726073344
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006302908726178536,
+ "loss": 1.4447,
+ "theoretical_loss": 3.793922309115417,
+ "tokens_seen": 726138880
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006302748244734203,
+ "loss": 1.3871,
+ "theoretical_loss": 3.7938880297814075,
+ "tokens_seen": 726204416
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000630258776328987,
+ "loss": 1.3355,
+ "theoretical_loss": 3.793853754406879,
+ "tokens_seen": 726269952
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006302427281845537,
+ "loss": 1.378,
+ "theoretical_loss": 3.793819482991018,
+ "tokens_seen": 726335488
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006302266800401204,
+ "loss": 1.4091,
+ "theoretical_loss": 3.793785215533009,
+ "tokens_seen": 726401024
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006302106318956871,
+ "loss": 1.3496,
+ "theoretical_loss": 3.7937509520320387,
+ "tokens_seen": 726466560
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301945837512538,
+ "loss": 1.3943,
+ "theoretical_loss": 3.793716692487293,
+ "tokens_seen": 726532096
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301785356068205,
+ "loss": 1.3716,
+ "theoretical_loss": 3.7936824368979583,
+ "tokens_seen": 726597632
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301624874623872,
+ "loss": 1.3933,
+ "theoretical_loss": 3.793648185263221,
+ "tokens_seen": 726663168
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301464393179539,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7936139375822684,
+ "tokens_seen": 726728704
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301303911735206,
+ "loss": 1.437,
+ "theoretical_loss": 3.793579693854287,
+ "tokens_seen": 726794240
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006301143430290873,
+ "loss": 1.3149,
+ "theoretical_loss": 3.7935454540784646,
+ "tokens_seen": 726859776
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000630098294884654,
+ "loss": 1.3953,
+ "theoretical_loss": 3.7935112182539887,
+ "tokens_seen": 726925312
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300822467402207,
+ "loss": 1.4703,
+ "theoretical_loss": 3.7934769863800475,
+ "tokens_seen": 726990848
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300661985957874,
+ "loss": 1.3592,
+ "theoretical_loss": 3.7934427584558286,
+ "tokens_seen": 727056384
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300501504513541,
+ "loss": 1.4148,
+ "theoretical_loss": 3.7934085344805206,
+ "tokens_seen": 727121920
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300341023069208,
+ "loss": 1.4887,
+ "theoretical_loss": 3.7933743144533114,
+ "tokens_seen": 727187456
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300180541624875,
+ "loss": 1.3985,
+ "theoretical_loss": 3.7933400983733905,
+ "tokens_seen": 727252992
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006300020060180542,
+ "loss": 1.4509,
+ "theoretical_loss": 3.7933058862399465,
+ "tokens_seen": 727318528
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006299859578736209,
+ "loss": 1.3862,
+ "theoretical_loss": 3.7932716780521694,
+ "tokens_seen": 727384064
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 247886,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3108396530151367,
+ "objective/train/theoretical_loss": 3.793237473809248,
+ "objective/train/tokens_used": 747909600,
+ "theoretical_loss": 3.793237473809248,
+ "tokens_seen": 727449600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006299699097291876,
+ "loss": 1.3084,
+ "theoretical_loss": 3.793237473809248,
+ "tokens_seen": 727449600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006299538615847543,
+ "loss": 1.3268,
+ "theoretical_loss": 3.7932032735103722,
+ "tokens_seen": 727515136
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629937813440321,
+ "loss": 1.3663,
+ "theoretical_loss": 3.793169077154732,
+ "tokens_seen": 727580672
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006299217652958877,
+ "loss": 1.4104,
+ "theoretical_loss": 3.7931348847415176,
+ "tokens_seen": 727646208
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006299057171514544,
+ "loss": 1.3822,
+ "theoretical_loss": 3.79310069626992,
+ "tokens_seen": 727711744
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298896690070211,
+ "loss": 1.3625,
+ "theoretical_loss": 3.793066511739129,
+ "tokens_seen": 727777280
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298736208625878,
+ "loss": 1.5019,
+ "theoretical_loss": 3.793032331148336,
+ "tokens_seen": 727842816
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298575727181545,
+ "loss": 1.5795,
+ "theoretical_loss": 3.7929981544967326,
+ "tokens_seen": 727908352
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298415245737212,
+ "loss": 1.3908,
+ "theoretical_loss": 3.7929639817835095,
+ "tokens_seen": 727973888
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298254764292879,
+ "loss": 1.3007,
+ "theoretical_loss": 3.7929298130078584,
+ "tokens_seen": 728039424
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006298094282848546,
+ "loss": 1.4696,
+ "theoretical_loss": 3.792895648168972,
+ "tokens_seen": 728104960
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006297933801404213,
+ "loss": 1.3193,
+ "theoretical_loss": 3.792861487266042,
+ "tokens_seen": 728170496
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629777331995988,
+ "loss": 1.3398,
+ "theoretical_loss": 3.7928273302982602,
+ "tokens_seen": 728236032
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006297612838515547,
+ "loss": 1.4541,
+ "theoretical_loss": 3.7927931772648202,
+ "tokens_seen": 728301568
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006297452357071214,
+ "loss": 1.4588,
+ "theoretical_loss": 3.792759028164914,
+ "tokens_seen": 728367104
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006297291875626881,
+ "loss": 1.3585,
+ "theoretical_loss": 3.792724882997735,
+ "tokens_seen": 728432640
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006297131394182548,
+ "loss": 1.3819,
+ "theoretical_loss": 3.792690741762477,
+ "tokens_seen": 728498176
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296970912738215,
+ "loss": 1.2567,
+ "theoretical_loss": 3.792656604458333,
+ "tokens_seen": 728563712
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296810431293882,
+ "loss": 1.4528,
+ "theoretical_loss": 3.7926224710844973,
+ "tokens_seen": 728629248
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296649949849549,
+ "loss": 1.3446,
+ "theoretical_loss": 3.7925883416401636,
+ "tokens_seen": 728694784
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296489468405216,
+ "loss": 1.3966,
+ "theoretical_loss": 3.792554216124526,
+ "tokens_seen": 728760320
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296328986960883,
+ "loss": 1.4292,
+ "theoretical_loss": 3.792520094536779,
+ "tokens_seen": 728825856
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629616850551655,
+ "loss": 1.4539,
+ "theoretical_loss": 3.792485976876118,
+ "tokens_seen": 728891392
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006296008024072217,
+ "loss": 1.4471,
+ "theoretical_loss": 3.7924518631417374,
+ "tokens_seen": 728956928
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295847542627884,
+ "loss": 1.5046,
+ "theoretical_loss": 3.792417753332833,
+ "tokens_seen": 729022464
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295687061183551,
+ "loss": 1.4383,
+ "theoretical_loss": 3.7923836474485997,
+ "tokens_seen": 729088000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295526579739218,
+ "loss": 1.3187,
+ "theoretical_loss": 3.792349545488233,
+ "tokens_seen": 729153536
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295366098294885,
+ "loss": 1.3993,
+ "theoretical_loss": 3.79231544745093,
+ "tokens_seen": 729219072
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295205616850552,
+ "loss": 1.3409,
+ "theoretical_loss": 3.7922813533358855,
+ "tokens_seen": 729284608
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006295045135406219,
+ "loss": 1.3396,
+ "theoretical_loss": 3.792247263142297,
+ "tokens_seen": 729350144
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006294884653961886,
+ "loss": 1.4328,
+ "theoretical_loss": 3.7922131768693603,
+ "tokens_seen": 729415680
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006294724172517553,
+ "loss": 1.3611,
+ "theoretical_loss": 3.792179094516273,
+ "tokens_seen": 729481216
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629456369107322,
+ "loss": 1.4281,
+ "theoretical_loss": 3.792145016082232,
+ "tokens_seen": 729546752
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006294403209628887,
+ "loss": 1.4321,
+ "theoretical_loss": 3.7921109415664347,
+ "tokens_seen": 729612288
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006294242728184554,
+ "loss": 1.3427,
+ "theoretical_loss": 3.792076870968078,
+ "tokens_seen": 729677824
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006294082246740221,
+ "loss": 1.3667,
+ "theoretical_loss": 3.792042804286361,
+ "tokens_seen": 729743360
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293921765295888,
+ "loss": 1.3824,
+ "theoretical_loss": 3.792008741520481,
+ "tokens_seen": 729808896
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293761283851555,
+ "loss": 1.3687,
+ "theoretical_loss": 3.791974682669636,
+ "tokens_seen": 729874432
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293600802407222,
+ "loss": 1.306,
+ "theoretical_loss": 3.791940627733026,
+ "tokens_seen": 729939968
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293440320962888,
+ "loss": 1.4359,
+ "theoretical_loss": 3.791906576709848,
+ "tokens_seen": 730005504
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293279839518556,
+ "loss": 1.412,
+ "theoretical_loss": 3.791872529599302,
+ "tokens_seen": 730071040
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006293119358074222,
+ "loss": 1.3936,
+ "theoretical_loss": 3.791838486400587,
+ "tokens_seen": 730136576
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629295887662989,
+ "loss": 1.3971,
+ "theoretical_loss": 3.7918044471129027,
+ "tokens_seen": 730202112
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006292798395185556,
+ "loss": 1.4084,
+ "theoretical_loss": 3.7917704117354485,
+ "tokens_seen": 730267648
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006292637913741224,
+ "loss": 1.4487,
+ "theoretical_loss": 3.7917363802674244,
+ "tokens_seen": 730333184
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629247743229689,
+ "loss": 1.4476,
+ "theoretical_loss": 3.791702352708031,
+ "tokens_seen": 730398720
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006292316950852558,
+ "loss": 1.4352,
+ "theoretical_loss": 3.7916683290564683,
+ "tokens_seen": 730464256
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006292156469408224,
+ "loss": 1.4082,
+ "theoretical_loss": 3.7916343093119376,
+ "tokens_seen": 730529792
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291995987963892,
+ "loss": 1.3462,
+ "theoretical_loss": 3.791600293473639,
+ "tokens_seen": 730595328
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291835506519558,
+ "loss": 1.4365,
+ "theoretical_loss": 3.791566281540774,
+ "tokens_seen": 730660864
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 249085,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4543607234954834,
+ "objective/train/theoretical_loss": 3.791532273512544,
+ "objective/train/tokens_used": 751186400,
+ "theoretical_loss": 3.791532273512544,
+ "tokens_seen": 730726400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291675025075226,
+ "loss": 1.4237,
+ "theoretical_loss": 3.791532273512544,
+ "tokens_seen": 730726400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291514543630892,
+ "loss": 1.4221,
+ "theoretical_loss": 3.7914982693881503,
+ "tokens_seen": 730791936
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629135406218656,
+ "loss": 1.4758,
+ "theoretical_loss": 3.791464269166795,
+ "tokens_seen": 730857472
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291193580742226,
+ "loss": 1.4386,
+ "theoretical_loss": 3.7914302728476805,
+ "tokens_seen": 730923008
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006291033099297894,
+ "loss": 1.4968,
+ "theoretical_loss": 3.7913962804300088,
+ "tokens_seen": 730988544
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000629087261785356,
+ "loss": 1.4565,
+ "theoretical_loss": 3.791362291912982,
+ "tokens_seen": 731054080
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006290712136409228,
+ "loss": 1.2575,
+ "theoretical_loss": 3.791328307295804,
+ "tokens_seen": 731119616
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006290551654964895,
+ "loss": 1.3963,
+ "theoretical_loss": 3.791294326577677,
+ "tokens_seen": 731185152
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006290391173520562,
+ "loss": 1.4078,
+ "theoretical_loss": 3.7912603497578043,
+ "tokens_seen": 731250688
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006290230692076229,
+ "loss": 1.4262,
+ "theoretical_loss": 3.7912263768353895,
+ "tokens_seen": 731316224
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006290070210631896,
+ "loss": 1.4075,
+ "theoretical_loss": 3.7911924078096364,
+ "tokens_seen": 731381760
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006289909729187563,
+ "loss": 1.4069,
+ "theoretical_loss": 3.791158442679748,
+ "tokens_seen": 731447296
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628974924774323,
+ "loss": 1.3423,
+ "theoretical_loss": 3.7911244814449305,
+ "tokens_seen": 731512832
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006289588766298897,
+ "loss": 1.5558,
+ "theoretical_loss": 3.7910905241043866,
+ "tokens_seen": 731578368
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006289428284854564,
+ "loss": 1.3795,
+ "theoretical_loss": 3.7910565706573216,
+ "tokens_seen": 731643904
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006289267803410231,
+ "loss": 1.4941,
+ "theoretical_loss": 3.7910226211029405,
+ "tokens_seen": 731709440
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006289107321965898,
+ "loss": 1.4243,
+ "theoretical_loss": 3.790988675440448,
+ "tokens_seen": 731774976
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006288946840521565,
+ "loss": 1.4918,
+ "theoretical_loss": 3.79095473366905,
+ "tokens_seen": 731840512
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006288786359077232,
+ "loss": 1.4058,
+ "theoretical_loss": 3.790920795787952,
+ "tokens_seen": 731906048
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006288625877632899,
+ "loss": 1.3867,
+ "theoretical_loss": 3.7908868617963587,
+ "tokens_seen": 731971584
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006288465396188566,
+ "loss": 1.4297,
+ "theoretical_loss": 3.7908529316934776,
+ "tokens_seen": 732037120
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006288304914744233,
+ "loss": 1.3749,
+ "theoretical_loss": 3.7908190054785145,
+ "tokens_seen": 732102656
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00062881444332999,
+ "loss": 1.2819,
+ "theoretical_loss": 3.7907850831506753,
+ "tokens_seen": 732168192
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287983951855567,
+ "loss": 1.485,
+ "theoretical_loss": 3.7907511647091683,
+ "tokens_seen": 732233728
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287823470411234,
+ "loss": 1.5243,
+ "theoretical_loss": 3.7907172501531985,
+ "tokens_seen": 732299264
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287662988966901,
+ "loss": 1.4483,
+ "theoretical_loss": 3.7906833394819746,
+ "tokens_seen": 732364800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287502507522568,
+ "loss": 1.3798,
+ "theoretical_loss": 3.790649432694704,
+ "tokens_seen": 732430336
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287342026078235,
+ "loss": 1.3989,
+ "theoretical_loss": 3.7906155297905935,
+ "tokens_seen": 732495872
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287181544633902,
+ "loss": 1.3249,
+ "theoretical_loss": 3.7905816307688514,
+ "tokens_seen": 732561408
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006287021063189569,
+ "loss": 1.4464,
+ "theoretical_loss": 3.790547735628686,
+ "tokens_seen": 732626944
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006286860581745236,
+ "loss": 1.4306,
+ "theoretical_loss": 3.790513844369306,
+ "tokens_seen": 732692480
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006286700100300903,
+ "loss": 1.328,
+ "theoretical_loss": 3.7904799569899192,
+ "tokens_seen": 732758016
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628653961885657,
+ "loss": 1.5002,
+ "theoretical_loss": 3.790446073489735,
+ "tokens_seen": 732823552
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006286379137412237,
+ "loss": 1.5499,
+ "theoretical_loss": 3.7904121938679625,
+ "tokens_seen": 732889088
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006286218655967904,
+ "loss": 1.4239,
+ "theoretical_loss": 3.7903783181238104,
+ "tokens_seen": 732954624
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006286058174523571,
+ "loss": 1.3344,
+ "theoretical_loss": 3.7903444462564893,
+ "tokens_seen": 733020160
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285897693079238,
+ "loss": 1.4331,
+ "theoretical_loss": 3.790310578265208,
+ "tokens_seen": 733085696
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285737211634905,
+ "loss": 1.4736,
+ "theoretical_loss": 3.7902767141491776,
+ "tokens_seen": 733151232
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285576730190572,
+ "loss": 1.472,
+ "theoretical_loss": 3.790242853907607,
+ "tokens_seen": 733216768
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285416248746239,
+ "loss": 1.4226,
+ "theoretical_loss": 3.790208997539707,
+ "tokens_seen": 733282304
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285255767301906,
+ "loss": 1.446,
+ "theoretical_loss": 3.7901751450446897,
+ "tokens_seen": 733347840
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006285095285857573,
+ "loss": 1.5121,
+ "theoretical_loss": 3.790141296421764,
+ "tokens_seen": 733413376
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628493480441324,
+ "loss": 1.4117,
+ "theoretical_loss": 3.790107451670142,
+ "tokens_seen": 733478912
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006284774322968907,
+ "loss": 1.4495,
+ "theoretical_loss": 3.790073610789036,
+ "tokens_seen": 733544448
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006284613841524574,
+ "loss": 1.4667,
+ "theoretical_loss": 3.7900397737776554,
+ "tokens_seen": 733609984
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006284453360080241,
+ "loss": 1.4249,
+ "theoretical_loss": 3.790005940635214,
+ "tokens_seen": 733675520
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006284292878635908,
+ "loss": 1.4262,
+ "theoretical_loss": 3.7899721113609237,
+ "tokens_seen": 733741056
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006284132397191575,
+ "loss": 1.3886,
+ "theoretical_loss": 3.789938285953996,
+ "tokens_seen": 733806592
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283971915747242,
+ "loss": 1.3485,
+ "theoretical_loss": 3.789904464413644,
+ "tokens_seen": 733872128
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283811434302909,
+ "loss": 1.3421,
+ "theoretical_loss": 3.7898706467390806,
+ "tokens_seen": 733937664
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 250229,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4468345642089844,
+ "objective/train/theoretical_loss": 3.7898368329295176,
+ "objective/train/tokens_used": 754463200,
+ "theoretical_loss": 3.7898368329295176,
+ "tokens_seen": 734003200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283650952858576,
+ "loss": 1.395,
+ "theoretical_loss": 3.7898368329295176,
+ "tokens_seen": 734003200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283490471414243,
+ "loss": 1.3423,
+ "theoretical_loss": 3.7898030229841697,
+ "tokens_seen": 734068736
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628332998996991,
+ "loss": 1.4906,
+ "theoretical_loss": 3.78976921690225,
+ "tokens_seen": 734134272
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283169508525577,
+ "loss": 1.3971,
+ "theoretical_loss": 3.789735414682972,
+ "tokens_seen": 734199808
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006283009027081244,
+ "loss": 1.4308,
+ "theoretical_loss": 3.7897016163255497,
+ "tokens_seen": 734265344
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006282848545636911,
+ "loss": 1.3764,
+ "theoretical_loss": 3.789667821829197,
+ "tokens_seen": 734330880
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006282688064192578,
+ "loss": 1.4429,
+ "theoretical_loss": 3.7896340311931285,
+ "tokens_seen": 734396416
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006282527582748245,
+ "loss": 1.4776,
+ "theoretical_loss": 3.789600244416559,
+ "tokens_seen": 734461952
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006282367101303912,
+ "loss": 1.4843,
+ "theoretical_loss": 3.789566461498703,
+ "tokens_seen": 734527488
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628220661985958,
+ "loss": 1.4393,
+ "theoretical_loss": 3.789532682438776,
+ "tokens_seen": 734593024
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006282046138415246,
+ "loss": 1.46,
+ "theoretical_loss": 3.789498907235993,
+ "tokens_seen": 734658560
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006281885656970914,
+ "loss": 1.4159,
+ "theoretical_loss": 3.7894651358895697,
+ "tokens_seen": 734724096
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628172517552658,
+ "loss": 1.5819,
+ "theoretical_loss": 3.789431368398722,
+ "tokens_seen": 734789632
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006281564694082248,
+ "loss": 1.4192,
+ "theoretical_loss": 3.789397604762666,
+ "tokens_seen": 734855168
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006281404212637914,
+ "loss": 1.4651,
+ "theoretical_loss": 3.789363844980617,
+ "tokens_seen": 734920704
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006281243731193582,
+ "loss": 1.2948,
+ "theoretical_loss": 3.7893300890517922,
+ "tokens_seen": 734986240
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006281083249749248,
+ "loss": 1.3556,
+ "theoretical_loss": 3.789296336975408,
+ "tokens_seen": 735051776
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006280922768304916,
+ "loss": 1.3993,
+ "theoretical_loss": 3.789262588750682,
+ "tokens_seen": 735117312
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006280762286860582,
+ "loss": 1.383,
+ "theoretical_loss": 3.7892288443768303,
+ "tokens_seen": 735182848
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006280601805416249,
+ "loss": 1.3438,
+ "theoretical_loss": 3.7891951038530713,
+ "tokens_seen": 735248384
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006280441323971916,
+ "loss": 1.3622,
+ "theoretical_loss": 3.7891613671786217,
+ "tokens_seen": 735313920
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006280280842527583,
+ "loss": 1.4872,
+ "theoretical_loss": 3.7891276343527,
+ "tokens_seen": 735379456
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000628012036108325,
+ "loss": 1.3421,
+ "theoretical_loss": 3.789093905374524,
+ "tokens_seen": 735444992
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279959879638917,
+ "loss": 1.4801,
+ "theoretical_loss": 3.789060180243312,
+ "tokens_seen": 735510528
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279799398194584,
+ "loss": 1.3814,
+ "theoretical_loss": 3.7890264589582827,
+ "tokens_seen": 735576064
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279638916750251,
+ "loss": 1.3895,
+ "theoretical_loss": 3.7889927415186544,
+ "tokens_seen": 735641600
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279478435305917,
+ "loss": 1.3125,
+ "theoretical_loss": 3.7889590279236467,
+ "tokens_seen": 735707136
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279317953861585,
+ "loss": 1.3544,
+ "theoretical_loss": 3.788925318172478,
+ "tokens_seen": 735772672
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006279157472417251,
+ "loss": 1.4446,
+ "theoretical_loss": 3.7888916122643685,
+ "tokens_seen": 735838208
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278996990972919,
+ "loss": 1.3253,
+ "theoretical_loss": 3.788857910198537,
+ "tokens_seen": 735903744
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278836509528585,
+ "loss": 1.4505,
+ "theoretical_loss": 3.7888242119742044,
+ "tokens_seen": 735969280
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278676028084253,
+ "loss": 1.4354,
+ "theoretical_loss": 3.7887905175905905,
+ "tokens_seen": 736034816
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278515546639919,
+ "loss": 1.4241,
+ "theoretical_loss": 3.7887568270469147,
+ "tokens_seen": 736100352
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278355065195587,
+ "loss": 1.3215,
+ "theoretical_loss": 3.788723140342399,
+ "tokens_seen": 736165888
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278194583751253,
+ "loss": 1.4426,
+ "theoretical_loss": 3.7886894574762637,
+ "tokens_seen": 736231424
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006278034102306921,
+ "loss": 1.4753,
+ "theoretical_loss": 3.7886557784477293,
+ "tokens_seen": 736296960
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277873620862587,
+ "loss": 1.3266,
+ "theoretical_loss": 3.7886221032560172,
+ "tokens_seen": 736362496
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277713139418255,
+ "loss": 1.3998,
+ "theoretical_loss": 3.7885884319003496,
+ "tokens_seen": 736428032
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277552657973921,
+ "loss": 1.3853,
+ "theoretical_loss": 3.7885547643799473,
+ "tokens_seen": 736493568
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277392176529589,
+ "loss": 1.4296,
+ "theoretical_loss": 3.788521100694033,
+ "tokens_seen": 736559104
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277231695085255,
+ "loss": 1.3047,
+ "theoretical_loss": 3.7884874408418288,
+ "tokens_seen": 736624640
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006277071213640923,
+ "loss": 1.3722,
+ "theoretical_loss": 3.788453784822556,
+ "tokens_seen": 736690176
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000627691073219659,
+ "loss": 1.456,
+ "theoretical_loss": 3.788420132635438,
+ "tokens_seen": 736755712
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006276750250752257,
+ "loss": 1.408,
+ "theoretical_loss": 3.788386484279698,
+ "tokens_seen": 736821248
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006276589769307924,
+ "loss": 1.294,
+ "theoretical_loss": 3.788352839754559,
+ "tokens_seen": 736886784
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006276429287863591,
+ "loss": 1.4556,
+ "theoretical_loss": 3.7883191990592433,
+ "tokens_seen": 736952320
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006276268806419258,
+ "loss": 1.3902,
+ "theoretical_loss": 3.788285562192976,
+ "tokens_seen": 737017856
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006276108324974925,
+ "loss": 1.4227,
+ "theoretical_loss": 3.7882519291549794,
+ "tokens_seen": 737083392
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006275947843530592,
+ "loss": 1.4074,
+ "theoretical_loss": 3.788218299944478,
+ "tokens_seen": 737148928
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006275787362086259,
+ "loss": 1.4197,
+ "theoretical_loss": 3.7881846745606964,
+ "tokens_seen": 737214464
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 251121,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3285048007965088,
+ "objective/train/theoretical_loss": 3.788151053002858,
+ "objective/train/tokens_used": 757740000,
+ "theoretical_loss": 3.788151053002858,
+ "tokens_seen": 737280000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006275626880641926,
+ "loss": 1.3462,
+ "theoretical_loss": 3.788151053002858,
+ "tokens_seen": 737280000
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006275466399197593,
+ "loss": 1.3299,
+ "theoretical_loss": 3.788117435270189,
+ "tokens_seen": 737345536
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000627530591775326,
+ "loss": 1.4813,
+ "theoretical_loss": 3.788083821361913,
+ "tokens_seen": 737411072
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006275145436308927,
+ "loss": 1.4779,
+ "theoretical_loss": 3.7880502112772554,
+ "tokens_seen": 737476608
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274984954864594,
+ "loss": 1.4615,
+ "theoretical_loss": 3.788016605015442,
+ "tokens_seen": 737542144
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274824473420261,
+ "loss": 1.3011,
+ "theoretical_loss": 3.7879830025756975,
+ "tokens_seen": 737607680
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274663991975928,
+ "loss": 1.4051,
+ "theoretical_loss": 3.7879494039572483,
+ "tokens_seen": 737673216
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274503510531595,
+ "loss": 1.5,
+ "theoretical_loss": 3.787915809159321,
+ "tokens_seen": 737738752
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274343029087262,
+ "loss": 1.3121,
+ "theoretical_loss": 3.7878822181811405,
+ "tokens_seen": 737804288
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274182547642929,
+ "loss": 1.3735,
+ "theoretical_loss": 3.7878486310219337,
+ "tokens_seen": 737869824
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006274022066198596,
+ "loss": 1.472,
+ "theoretical_loss": 3.787815047680928,
+ "tokens_seen": 737935360
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006273861584754263,
+ "loss": 1.3119,
+ "theoretical_loss": 3.787781468157349,
+ "tokens_seen": 738000896
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000627370110330993,
+ "loss": 1.3764,
+ "theoretical_loss": 3.787747892450425,
+ "tokens_seen": 738066432
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006273540621865597,
+ "loss": 1.4331,
+ "theoretical_loss": 3.7877143205593837,
+ "tokens_seen": 738131968
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006273380140421264,
+ "loss": 1.4099,
+ "theoretical_loss": 3.787680752483451,
+ "tokens_seen": 738197504
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006273219658976931,
+ "loss": 1.4504,
+ "theoretical_loss": 3.787647188221856,
+ "tokens_seen": 738263040
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006273059177532598,
+ "loss": 1.4757,
+ "theoretical_loss": 3.787613627773826,
+ "tokens_seen": 738328576
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006272898696088265,
+ "loss": 1.3534,
+ "theoretical_loss": 3.7875800711385903,
+ "tokens_seen": 738394112
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006272738214643932,
+ "loss": 1.2828,
+ "theoretical_loss": 3.7875465183153767,
+ "tokens_seen": 738459648
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006272577733199599,
+ "loss": 1.5056,
+ "theoretical_loss": 3.787512969303414,
+ "tokens_seen": 738525184
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006272417251755266,
+ "loss": 1.4318,
+ "theoretical_loss": 3.7874794241019307,
+ "tokens_seen": 738590720
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006272256770310933,
+ "loss": 1.4262,
+ "theoretical_loss": 3.7874458827101565,
+ "tokens_seen": 738656256
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.00062720962888666,
+ "loss": 1.3815,
+ "theoretical_loss": 3.7874123451273203,
+ "tokens_seen": 738721792
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271935807422267,
+ "loss": 1.4049,
+ "theoretical_loss": 3.787378811352652,
+ "tokens_seen": 738787328
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271775325977934,
+ "loss": 1.3613,
+ "theoretical_loss": 3.787345281385382,
+ "tokens_seen": 738852864
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271614844533601,
+ "loss": 1.3919,
+ "theoretical_loss": 3.7873117552247395,
+ "tokens_seen": 738918400
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271454363089268,
+ "loss": 1.4125,
+ "theoretical_loss": 3.7872782328699555,
+ "tokens_seen": 738983936
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271293881644935,
+ "loss": 1.3893,
+ "theoretical_loss": 3.7872447143202597,
+ "tokens_seen": 739049472
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006271133400200602,
+ "loss": 1.5538,
+ "theoretical_loss": 3.787211199574883,
+ "tokens_seen": 739115008
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270972918756269,
+ "loss": 1.3634,
+ "theoretical_loss": 3.787177688633057,
+ "tokens_seen": 739180544
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270812437311936,
+ "loss": 1.4892,
+ "theoretical_loss": 3.7871441814940123,
+ "tokens_seen": 739246080
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270651955867603,
+ "loss": 1.3793,
+ "theoretical_loss": 3.7871106781569805,
+ "tokens_seen": 739311616
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000627049147442327,
+ "loss": 1.5246,
+ "theoretical_loss": 3.7870771786211925,
+ "tokens_seen": 739377152
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270330992978937,
+ "loss": 1.4735,
+ "theoretical_loss": 3.787043682885882,
+ "tokens_seen": 739442688
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270170511534605,
+ "loss": 1.4128,
+ "theoretical_loss": 3.7870101909502787,
+ "tokens_seen": 739508224
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006270010030090271,
+ "loss": 1.3249,
+ "theoretical_loss": 3.7869767028136163,
+ "tokens_seen": 739573760
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269849548645939,
+ "loss": 1.4879,
+ "theoretical_loss": 3.7869432184751277,
+ "tokens_seen": 739639296
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269689067201605,
+ "loss": 1.3336,
+ "theoretical_loss": 3.7869097379340446,
+ "tokens_seen": 739704832
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269528585757273,
+ "loss": 1.4394,
+ "theoretical_loss": 3.7868762611896,
+ "tokens_seen": 739770368
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269368104312939,
+ "loss": 1.3731,
+ "theoretical_loss": 3.786842788241028,
+ "tokens_seen": 739835904
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269207622868607,
+ "loss": 1.475,
+ "theoretical_loss": 3.7868093190875616,
+ "tokens_seen": 739901440
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006269047141424273,
+ "loss": 1.3288,
+ "theoretical_loss": 3.7867758537284337,
+ "tokens_seen": 739966976
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268886659979941,
+ "loss": 1.5498,
+ "theoretical_loss": 3.786742392162879,
+ "tokens_seen": 740032512
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268726178535607,
+ "loss": 1.3945,
+ "theoretical_loss": 3.7867089343901315,
+ "tokens_seen": 740098048
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268565697091275,
+ "loss": 1.4395,
+ "theoretical_loss": 3.786675480409425,
+ "tokens_seen": 740163584
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268405215646941,
+ "loss": 1.5172,
+ "theoretical_loss": 3.786642030219994,
+ "tokens_seen": 740229120
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268244734202609,
+ "loss": 1.426,
+ "theoretical_loss": 3.7866085838210735,
+ "tokens_seen": 740294656
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006268084252758275,
+ "loss": 1.3196,
+ "theoretical_loss": 3.786575141211899,
+ "tokens_seen": 740360192
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267923771313943,
+ "loss": 1.356,
+ "theoretical_loss": 3.7865417023917054,
+ "tokens_seen": 740425728
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267763289869609,
+ "loss": 1.4566,
+ "theoretical_loss": 3.7865082673597277,
+ "tokens_seen": 740491264
+ },
+ {
+ "epoch": 0.22,
+ "objective/train/docs_used": 252416,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3830698728561401,
+ "objective/train/theoretical_loss": 3.786474836115201,
+ "objective/train/tokens_used": 761016800,
+ "theoretical_loss": 3.786474836115201,
+ "tokens_seen": 740556800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267602808425277,
+ "loss": 1.3715,
+ "theoretical_loss": 3.786474836115201,
+ "tokens_seen": 740556800
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267442326980943,
+ "loss": 1.3389,
+ "theoretical_loss": 3.786441408657363,
+ "tokens_seen": 740622336
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267281845536611,
+ "loss": 1.4551,
+ "theoretical_loss": 3.7864079849854475,
+ "tokens_seen": 740687872
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006267121364092277,
+ "loss": 1.3781,
+ "theoretical_loss": 3.7863745650986926,
+ "tokens_seen": 740753408
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006266960882647945,
+ "loss": 1.4396,
+ "theoretical_loss": 3.7863411489963346,
+ "tokens_seen": 740818944
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006266800401203611,
+ "loss": 1.3852,
+ "theoretical_loss": 3.786307736677609,
+ "tokens_seen": 740884480
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006266639919759278,
+ "loss": 1.3442,
+ "theoretical_loss": 3.7862743281417535,
+ "tokens_seen": 740950016
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006266479438314945,
+ "loss": 1.4706,
+ "theoretical_loss": 3.786240923388006,
+ "tokens_seen": 741015552
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006266318956870612,
+ "loss": 1.3943,
+ "theoretical_loss": 3.7862075224156024,
+ "tokens_seen": 741081088
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000626615847542628,
+ "loss": 1.465,
+ "theoretical_loss": 3.7861741252237815,
+ "tokens_seen": 741146624
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006265997993981946,
+ "loss": 1.4682,
+ "theoretical_loss": 3.7861407318117815,
+ "tokens_seen": 741212160
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006265837512537613,
+ "loss": 1.3761,
+ "theoretical_loss": 3.7861073421788385,
+ "tokens_seen": 741277696
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000626567703109328,
+ "loss": 1.4734,
+ "theoretical_loss": 3.786073956324193,
+ "tokens_seen": 741343232
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006265516549648948,
+ "loss": 1.396,
+ "theoretical_loss": 3.786040574247082,
+ "tokens_seen": 741408768
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006265356068204614,
+ "loss": 1.3962,
+ "theoretical_loss": 3.7860071959467447,
+ "tokens_seen": 741474304
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000626519558676028,
+ "loss": 1.3869,
+ "theoretical_loss": 3.785973821422421,
+ "tokens_seen": 741539840
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006265035105315948,
+ "loss": 1.4752,
+ "theoretical_loss": 3.7859404506733485,
+ "tokens_seen": 741605376
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006264874623871614,
+ "loss": 1.4597,
+ "theoretical_loss": 3.7859070836987674,
+ "tokens_seen": 741670912
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006264714142427282,
+ "loss": 1.3884,
+ "theoretical_loss": 3.785873720497917,
+ "tokens_seen": 741736448
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006264553660982948,
+ "loss": 1.3725,
+ "theoretical_loss": 3.7858403610700373,
+ "tokens_seen": 741801984
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006264393179538616,
+ "loss": 1.4943,
+ "theoretical_loss": 3.7858070054143687,
+ "tokens_seen": 741867520
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006264232698094282,
+ "loss": 1.4139,
+ "theoretical_loss": 3.7857736535301507,
+ "tokens_seen": 741933056
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000626407221664995,
+ "loss": 1.3827,
+ "theoretical_loss": 3.785740305416624,
+ "tokens_seen": 741998592
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006263911735205616,
+ "loss": 1.433,
+ "theoretical_loss": 3.7857069610730303,
+ "tokens_seen": 742064128
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006263751253761284,
+ "loss": 1.5033,
+ "theoretical_loss": 3.785673620498609,
+ "tokens_seen": 742129664
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.000626359077231695,
+ "loss": 1.5053,
+ "theoretical_loss": 3.7856402836926026,
+ "tokens_seen": 742195200
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006263430290872618,
+ "loss": 1.5469,
+ "theoretical_loss": 3.785606950654252,
+ "tokens_seen": 742260736
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006263269809428285,
+ "loss": 1.4765,
+ "theoretical_loss": 3.785573621382798,
+ "tokens_seen": 742326272
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006263109327983952,
+ "loss": 1.3257,
+ "theoretical_loss": 3.7855402958774835,
+ "tokens_seen": 742391808
+ },
+ {
+ "epoch": 0.22,
+ "learning_rate": 0.0006262948846539619,
+ "loss": 1.3639,
+ "theoretical_loss": 3.78550697413755,
+ "tokens_seen": 742457344
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006262788365095286,
+ "loss": 1.4911,
+ "theoretical_loss": 3.7854736561622397,
+ "tokens_seen": 742522880
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006262627883650953,
+ "loss": 1.4253,
+ "theoretical_loss": 3.785440341950795,
+ "tokens_seen": 742588416
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000626246740220662,
+ "loss": 1.4194,
+ "theoretical_loss": 3.7854070315024595,
+ "tokens_seen": 742653952
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006262306920762287,
+ "loss": 1.4468,
+ "theoretical_loss": 3.7853737248164743,
+ "tokens_seen": 742719488
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006262146439317954,
+ "loss": 1.4094,
+ "theoretical_loss": 3.7853404218920845,
+ "tokens_seen": 742785024
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261985957873621,
+ "loss": 1.367,
+ "theoretical_loss": 3.7853071227285326,
+ "tokens_seen": 742850560
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261825476429288,
+ "loss": 1.3086,
+ "theoretical_loss": 3.785273827325062,
+ "tokens_seen": 742916096
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261664994984955,
+ "loss": 1.5286,
+ "theoretical_loss": 3.7852405356809165,
+ "tokens_seen": 742981632
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261504513540622,
+ "loss": 1.3893,
+ "theoretical_loss": 3.78520724779534,
+ "tokens_seen": 743047168
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261344032096289,
+ "loss": 1.367,
+ "theoretical_loss": 3.785173963667577,
+ "tokens_seen": 743112704
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261183550651956,
+ "loss": 1.3122,
+ "theoretical_loss": 3.785140683296872,
+ "tokens_seen": 743178240
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006261023069207623,
+ "loss": 1.4675,
+ "theoretical_loss": 3.7851074066824695,
+ "tokens_seen": 743243776
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000626086258776329,
+ "loss": 1.4181,
+ "theoretical_loss": 3.7850741338236142,
+ "tokens_seen": 743309312
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006260702106318957,
+ "loss": 1.4501,
+ "theoretical_loss": 3.7850408647195515,
+ "tokens_seen": 743374848
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006260541624874624,
+ "loss": 1.4349,
+ "theoretical_loss": 3.7850075993695267,
+ "tokens_seen": 743440384
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006260381143430291,
+ "loss": 1.3888,
+ "theoretical_loss": 3.7849743377727854,
+ "tokens_seen": 743505920
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006260220661985958,
+ "loss": 1.5065,
+ "theoretical_loss": 3.784941079928573,
+ "tokens_seen": 743571456
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006260060180541625,
+ "loss": 1.3948,
+ "theoretical_loss": 3.7849078258361355,
+ "tokens_seen": 743636992
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006259899699097292,
+ "loss": 1.3591,
+ "theoretical_loss": 3.784874575494719,
+ "tokens_seen": 743702528
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006259739217652959,
+ "loss": 1.3867,
+ "theoretical_loss": 3.7848413289035703,
+ "tokens_seen": 743768064
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 253356,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4743261337280273,
+ "objective/train/theoretical_loss": 3.784808086061936,
+ "objective/train/tokens_used": 764293600,
+ "theoretical_loss": 3.784808086061936,
+ "tokens_seen": 743833600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006259578736208626,
+ "loss": 1.4374,
+ "theoretical_loss": 3.784808086061936,
+ "tokens_seen": 743833600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006259418254764293,
+ "loss": 1.4514,
+ "theoretical_loss": 3.7847748469690625,
+ "tokens_seen": 743899136
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625925777331996,
+ "loss": 1.4886,
+ "theoretical_loss": 3.784741611624198,
+ "tokens_seen": 743964672
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006259097291875627,
+ "loss": 1.3899,
+ "theoretical_loss": 3.784708380026588,
+ "tokens_seen": 744030208
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006258936810431294,
+ "loss": 1.4282,
+ "theoretical_loss": 3.784675152175481,
+ "tokens_seen": 744095744
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006258776328986961,
+ "loss": 1.4404,
+ "theoretical_loss": 3.7846419280701245,
+ "tokens_seen": 744161280
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006258615847542628,
+ "loss": 1.4096,
+ "theoretical_loss": 3.7846087077097668,
+ "tokens_seen": 744226816
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006258455366098295,
+ "loss": 1.328,
+ "theoretical_loss": 3.7845754910936558,
+ "tokens_seen": 744292352
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006258294884653962,
+ "loss": 1.4444,
+ "theoretical_loss": 3.784542278221039,
+ "tokens_seen": 744357888
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625813440320963,
+ "loss": 1.4253,
+ "theoretical_loss": 3.784509069091167,
+ "tokens_seen": 744423424
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257973921765296,
+ "loss": 1.3054,
+ "theoretical_loss": 3.7844758637032867,
+ "tokens_seen": 744488960
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257813440320964,
+ "loss": 1.619,
+ "theoretical_loss": 3.784442662056648,
+ "tokens_seen": 744554496
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625765295887663,
+ "loss": 1.4287,
+ "theoretical_loss": 3.7844094641504995,
+ "tokens_seen": 744620032
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257492477432298,
+ "loss": 1.4623,
+ "theoretical_loss": 3.784376269984092,
+ "tokens_seen": 744685568
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257331995987964,
+ "loss": 1.4089,
+ "theoretical_loss": 3.784343079556673,
+ "tokens_seen": 744751104
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257171514543632,
+ "loss": 1.3625,
+ "theoretical_loss": 3.784309892867494,
+ "tokens_seen": 744816640
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006257011033099298,
+ "loss": 1.411,
+ "theoretical_loss": 3.784276709915805,
+ "tokens_seen": 744882176
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006256850551654966,
+ "loss": 1.3846,
+ "theoretical_loss": 3.7842435307008557,
+ "tokens_seen": 744947712
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006256690070210632,
+ "loss": 1.4369,
+ "theoretical_loss": 3.784210355221897,
+ "tokens_seen": 745013248
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.00062565295887663,
+ "loss": 1.4207,
+ "theoretical_loss": 3.78417718347818,
+ "tokens_seen": 745078784
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006256369107321966,
+ "loss": 1.3438,
+ "theoretical_loss": 3.7841440154689545,
+ "tokens_seen": 745144320
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006256208625877634,
+ "loss": 1.3528,
+ "theoretical_loss": 3.7841108511934727,
+ "tokens_seen": 745209856
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.00062560481444333,
+ "loss": 1.4787,
+ "theoretical_loss": 3.7840776906509856,
+ "tokens_seen": 745275392
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255887662988968,
+ "loss": 1.4328,
+ "theoretical_loss": 3.784044533840745,
+ "tokens_seen": 745340928
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255727181544634,
+ "loss": 1.5011,
+ "theoretical_loss": 3.784011380762003,
+ "tokens_seen": 745406464
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255566700100302,
+ "loss": 1.4873,
+ "theoretical_loss": 3.7839782314140105,
+ "tokens_seen": 745472000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255406218655968,
+ "loss": 1.4528,
+ "theoretical_loss": 3.78394508579602,
+ "tokens_seen": 745537536
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255245737211636,
+ "loss": 1.4958,
+ "theoretical_loss": 3.7839119439072855,
+ "tokens_seen": 745603072
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006255085255767302,
+ "loss": 1.4287,
+ "theoretical_loss": 3.7838788057470585,
+ "tokens_seen": 745668608
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625492477432297,
+ "loss": 1.387,
+ "theoretical_loss": 3.783845671314592,
+ "tokens_seen": 745734144
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006254764292878636,
+ "loss": 1.5027,
+ "theoretical_loss": 3.783812540609139,
+ "tokens_seen": 745799680
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006254603811434304,
+ "loss": 1.3966,
+ "theoretical_loss": 3.783779413629953,
+ "tokens_seen": 745865216
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625444332998997,
+ "loss": 1.5242,
+ "theoretical_loss": 3.7837462903762873,
+ "tokens_seen": 745930752
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006254282848545638,
+ "loss": 1.3962,
+ "theoretical_loss": 3.7837131708473963,
+ "tokens_seen": 745996288
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006254122367101304,
+ "loss": 1.5299,
+ "theoretical_loss": 3.7836800550425336,
+ "tokens_seen": 746061824
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006253961885656972,
+ "loss": 1.4618,
+ "theoretical_loss": 3.783646942960953,
+ "tokens_seen": 746127360
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006253801404212638,
+ "loss": 1.4436,
+ "theoretical_loss": 3.7836138346019093,
+ "tokens_seen": 746192896
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006253640922768306,
+ "loss": 1.4595,
+ "theoretical_loss": 3.783580729964657,
+ "tokens_seen": 746258432
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006253480441323972,
+ "loss": 1.3886,
+ "theoretical_loss": 3.783547629048451,
+ "tokens_seen": 746323968
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625331995987964,
+ "loss": 1.3628,
+ "theoretical_loss": 3.7835145318525463,
+ "tokens_seen": 746389504
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006253159478435306,
+ "loss": 1.378,
+ "theoretical_loss": 3.7834814383761985,
+ "tokens_seen": 746455040
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252998996990974,
+ "loss": 1.4002,
+ "theoretical_loss": 3.7834483486186627,
+ "tokens_seen": 746520576
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000625283851554664,
+ "loss": 1.3338,
+ "theoretical_loss": 3.7834152625791946,
+ "tokens_seen": 746586112
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252678034102307,
+ "loss": 1.4126,
+ "theoretical_loss": 3.7833821802570506,
+ "tokens_seen": 746651648
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252517552657974,
+ "loss": 1.4488,
+ "theoretical_loss": 3.7833491016514857,
+ "tokens_seen": 746717184
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252357071213641,
+ "loss": 1.3234,
+ "theoretical_loss": 3.7833160267617574,
+ "tokens_seen": 746782720
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252196589769308,
+ "loss": 1.3528,
+ "theoretical_loss": 3.783282955587122,
+ "tokens_seen": 746848256
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006252036108324975,
+ "loss": 1.3483,
+ "theoretical_loss": 3.7832498881268357,
+ "tokens_seen": 746913792
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251875626880643,
+ "loss": 1.4388,
+ "theoretical_loss": 3.7832168243801556,
+ "tokens_seen": 746979328
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251715145436309,
+ "loss": 1.3172,
+ "theoretical_loss": 3.7831837643463393,
+ "tokens_seen": 747044864
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 254556,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5359313488006592,
+ "objective/train/theoretical_loss": 3.7831507080246443,
+ "objective/train/tokens_used": 767570400,
+ "theoretical_loss": 3.7831507080246443,
+ "tokens_seen": 747110400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251554663991977,
+ "loss": 1.4609,
+ "theoretical_loss": 3.7831507080246443,
+ "tokens_seen": 747110400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251394182547643,
+ "loss": 1.5006,
+ "theoretical_loss": 3.783117655414328,
+ "tokens_seen": 747175936
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251233701103309,
+ "loss": 1.5097,
+ "theoretical_loss": 3.783084606514648,
+ "tokens_seen": 747241472
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006251073219658977,
+ "loss": 1.4837,
+ "theoretical_loss": 3.7830515613248625,
+ "tokens_seen": 747307008
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250912738214643,
+ "loss": 1.5801,
+ "theoretical_loss": 3.7830185198442297,
+ "tokens_seen": 747372544
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250752256770311,
+ "loss": 1.3344,
+ "theoretical_loss": 3.782985482072008,
+ "tokens_seen": 747438080
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250591775325977,
+ "loss": 1.3575,
+ "theoretical_loss": 3.782952448007457,
+ "tokens_seen": 747503616
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250431293881645,
+ "loss": 1.476,
+ "theoretical_loss": 3.7829194176498335,
+ "tokens_seen": 747569152
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250270812437311,
+ "loss": 1.4671,
+ "theoretical_loss": 3.7828863909983994,
+ "tokens_seen": 747634688
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006250110330992979,
+ "loss": 1.3915,
+ "theoretical_loss": 3.7828533680524115,
+ "tokens_seen": 747700224
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006249949849548645,
+ "loss": 1.4057,
+ "theoretical_loss": 3.782820348811131,
+ "tokens_seen": 747765760
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006249789368104313,
+ "loss": 1.5221,
+ "theoretical_loss": 3.7827873332738164,
+ "tokens_seen": 747831296
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000624962888665998,
+ "loss": 1.4371,
+ "theoretical_loss": 3.782754321439729,
+ "tokens_seen": 747896832
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006249468405215647,
+ "loss": 1.3435,
+ "theoretical_loss": 3.7827213133081283,
+ "tokens_seen": 747962368
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006249307923771314,
+ "loss": 1.3588,
+ "theoretical_loss": 3.7826883088782743,
+ "tokens_seen": 748027904
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006249147442326981,
+ "loss": 1.3512,
+ "theoretical_loss": 3.782655308149428,
+ "tokens_seen": 748093440
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248986960882648,
+ "loss": 1.3859,
+ "theoretical_loss": 3.78262231112085,
+ "tokens_seen": 748158976
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248826479438315,
+ "loss": 1.331,
+ "theoretical_loss": 3.7825893177918015,
+ "tokens_seen": 748224512
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248665997993982,
+ "loss": 1.5332,
+ "theoretical_loss": 3.7825563281615446,
+ "tokens_seen": 748290048
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248505516549649,
+ "loss": 1.3855,
+ "theoretical_loss": 3.7825233422293394,
+ "tokens_seen": 748355584
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248345035105316,
+ "loss": 1.3496,
+ "theoretical_loss": 3.7824903599944477,
+ "tokens_seen": 748421120
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006248184553660983,
+ "loss": 1.4544,
+ "theoretical_loss": 3.7824573814561324,
+ "tokens_seen": 748486656
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000624802407221665,
+ "loss": 1.4627,
+ "theoretical_loss": 3.782424406613654,
+ "tokens_seen": 748552192
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247863590772317,
+ "loss": 1.3918,
+ "theoretical_loss": 3.782391435466277,
+ "tokens_seen": 748617728
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247703109327984,
+ "loss": 1.486,
+ "theoretical_loss": 3.7823584680132614,
+ "tokens_seen": 748683264
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247542627883651,
+ "loss": 1.4443,
+ "theoretical_loss": 3.7823255042538717,
+ "tokens_seen": 748748800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247382146439318,
+ "loss": 1.4635,
+ "theoretical_loss": 3.78229254418737,
+ "tokens_seen": 748814336
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247221664994985,
+ "loss": 1.4962,
+ "theoretical_loss": 3.7822595878130203,
+ "tokens_seen": 748879872
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006247061183550652,
+ "loss": 1.3376,
+ "theoretical_loss": 3.782226635130085,
+ "tokens_seen": 748945408
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006246900702106319,
+ "loss": 1.3647,
+ "theoretical_loss": 3.782193686137828,
+ "tokens_seen": 749010944
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006246740220661986,
+ "loss": 1.5502,
+ "theoretical_loss": 3.782160740835513,
+ "tokens_seen": 749076480
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006246579739217653,
+ "loss": 1.4429,
+ "theoretical_loss": 3.7821277992224043,
+ "tokens_seen": 749142016
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000624641925777332,
+ "loss": 1.3417,
+ "theoretical_loss": 3.7820948612977663,
+ "tokens_seen": 749207552
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006246258776328987,
+ "loss": 1.4858,
+ "theoretical_loss": 3.7820619270608624,
+ "tokens_seen": 749273088
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006246098294884655,
+ "loss": 1.3289,
+ "theoretical_loss": 3.782028996510958,
+ "tokens_seen": 749338624
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245937813440321,
+ "loss": 1.4173,
+ "theoretical_loss": 3.781996069647318,
+ "tokens_seen": 749404160
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245777331995989,
+ "loss": 1.2815,
+ "theoretical_loss": 3.781963146469207,
+ "tokens_seen": 749469696
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245616850551655,
+ "loss": 1.3905,
+ "theoretical_loss": 3.7819302269758905,
+ "tokens_seen": 749535232
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245456369107323,
+ "loss": 1.4193,
+ "theoretical_loss": 3.7818973111666345,
+ "tokens_seen": 749600768
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245295887662989,
+ "loss": 1.4555,
+ "theoretical_loss": 3.7818643990407037,
+ "tokens_seen": 749666304
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006245135406218657,
+ "loss": 1.4173,
+ "theoretical_loss": 3.7818314905973645,
+ "tokens_seen": 749731840
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244974924774323,
+ "loss": 1.4484,
+ "theoretical_loss": 3.7817985858358827,
+ "tokens_seen": 749797376
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244814443329991,
+ "loss": 1.4426,
+ "theoretical_loss": 3.781765684755525,
+ "tokens_seen": 749862912
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244653961885657,
+ "loss": 1.4372,
+ "theoretical_loss": 3.7817327873555575,
+ "tokens_seen": 749928448
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244493480441325,
+ "loss": 1.5416,
+ "theoretical_loss": 3.7816998936352473,
+ "tokens_seen": 749993984
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244332998996991,
+ "loss": 1.4446,
+ "theoretical_loss": 3.7816670035938618,
+ "tokens_seen": 750059520
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244172517552659,
+ "loss": 1.3866,
+ "theoretical_loss": 3.7816341172306664,
+ "tokens_seen": 750125056
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006244012036108325,
+ "loss": 1.4924,
+ "theoretical_loss": 3.781601234544931,
+ "tokens_seen": 750190592
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243851554663993,
+ "loss": 1.4214,
+ "theoretical_loss": 3.7815683555359207,
+ "tokens_seen": 750256128
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243691073219659,
+ "loss": 1.4347,
+ "theoretical_loss": 3.781535480202905,
+ "tokens_seen": 750321664
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 255521,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4189245700836182,
+ "objective/train/theoretical_loss": 3.781502608545151,
+ "objective/train/tokens_used": 770847200,
+ "theoretical_loss": 3.781502608545151,
+ "tokens_seen": 750387200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243530591775327,
+ "loss": 1.3877,
+ "theoretical_loss": 3.781502608545151,
+ "tokens_seen": 750387200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243370110330993,
+ "loss": 1.4554,
+ "theoretical_loss": 3.781469740561927,
+ "tokens_seen": 750452736
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243209628886661,
+ "loss": 1.3596,
+ "theoretical_loss": 3.7814368762525015,
+ "tokens_seen": 750518272
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006243049147442327,
+ "loss": 1.4392,
+ "theoretical_loss": 3.7814040156161433,
+ "tokens_seen": 750583808
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242888665997995,
+ "loss": 1.4331,
+ "theoretical_loss": 3.7813711586521217,
+ "tokens_seen": 750649344
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242728184553661,
+ "loss": 1.4317,
+ "theoretical_loss": 3.7813383053597045,
+ "tokens_seen": 750714880
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242567703109329,
+ "loss": 1.4442,
+ "theoretical_loss": 3.7813054557381616,
+ "tokens_seen": 750780416
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242407221664995,
+ "loss": 1.3416,
+ "theoretical_loss": 3.7812726097867624,
+ "tokens_seen": 750845952
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242246740220663,
+ "loss": 1.4734,
+ "theoretical_loss": 3.781239767504777,
+ "tokens_seen": 750911488
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006242086258776329,
+ "loss": 1.4555,
+ "theoretical_loss": 3.781206928891475,
+ "tokens_seen": 750977024
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241925777331997,
+ "loss": 1.3431,
+ "theoretical_loss": 3.7811740939461256,
+ "tokens_seen": 751042560
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241765295887663,
+ "loss": 1.4286,
+ "theoretical_loss": 3.781141262668,
+ "tokens_seen": 751108096
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241604814443331,
+ "loss": 1.4231,
+ "theoretical_loss": 3.7811084350563693,
+ "tokens_seen": 751173632
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241444332998997,
+ "loss": 1.3431,
+ "theoretical_loss": 3.781075611110503,
+ "tokens_seen": 751239168
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241283851554665,
+ "loss": 1.4482,
+ "theoretical_loss": 3.781042790829672,
+ "tokens_seen": 751304704
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006241123370110331,
+ "loss": 1.4495,
+ "theoretical_loss": 3.7810099742131484,
+ "tokens_seen": 751370240
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240962888665999,
+ "loss": 1.4483,
+ "theoretical_loss": 3.7809771612602034,
+ "tokens_seen": 751435776
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240802407221665,
+ "loss": 1.514,
+ "theoretical_loss": 3.780944351970107,
+ "tokens_seen": 751501312
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240641925777333,
+ "loss": 1.4726,
+ "theoretical_loss": 3.7809115463421334,
+ "tokens_seen": 751566848
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240481444332999,
+ "loss": 1.3094,
+ "theoretical_loss": 3.7808787443755527,
+ "tokens_seen": 751632384
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240320962888667,
+ "loss": 1.3706,
+ "theoretical_loss": 3.780845946069638,
+ "tokens_seen": 751697920
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240160481444333,
+ "loss": 1.5188,
+ "theoretical_loss": 3.7808131514236614,
+ "tokens_seen": 751763456
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006240000000000001,
+ "loss": 1.387,
+ "theoretical_loss": 3.7807803604368946,
+ "tokens_seen": 751828992
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239839518555667,
+ "loss": 1.4186,
+ "theoretical_loss": 3.780747573108612,
+ "tokens_seen": 751894528
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239679037111335,
+ "loss": 1.3802,
+ "theoretical_loss": 3.7807147894380853,
+ "tokens_seen": 751960064
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239518555667001,
+ "loss": 1.4785,
+ "theoretical_loss": 3.7806820094245888,
+ "tokens_seen": 752025600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239358074222669,
+ "loss": 1.4123,
+ "theoretical_loss": 3.780649233067395,
+ "tokens_seen": 752091136
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239197592778335,
+ "loss": 1.4093,
+ "theoretical_loss": 3.7806164603657777,
+ "tokens_seen": 752156672
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006239037111334003,
+ "loss": 1.3978,
+ "theoretical_loss": 3.780583691319011,
+ "tokens_seen": 752222208
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623887662988967,
+ "loss": 1.3695,
+ "theoretical_loss": 3.7805509259263688,
+ "tokens_seen": 752287744
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006238716148445336,
+ "loss": 1.4512,
+ "theoretical_loss": 3.7805181641871255,
+ "tokens_seen": 752353280
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006238555667001003,
+ "loss": 1.5197,
+ "theoretical_loss": 3.7804854061005555,
+ "tokens_seen": 752418816
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623839518555667,
+ "loss": 1.3625,
+ "theoretical_loss": 3.780452651665933,
+ "tokens_seen": 752484352
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006238234704112337,
+ "loss": 1.3544,
+ "theoretical_loss": 3.780419900882533,
+ "tokens_seen": 752549888
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006238074222668004,
+ "loss": 1.4661,
+ "theoretical_loss": 3.780387153749632,
+ "tokens_seen": 752615424
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237913741223672,
+ "loss": 1.389,
+ "theoretical_loss": 3.780354410266503,
+ "tokens_seen": 752680960
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237753259779338,
+ "loss": 1.3227,
+ "theoretical_loss": 3.780321670432423,
+ "tokens_seen": 752746496
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237592778335006,
+ "loss": 1.3916,
+ "theoretical_loss": 3.7802889342466672,
+ "tokens_seen": 752812032
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237432296890672,
+ "loss": 1.4233,
+ "theoretical_loss": 3.7802562017085117,
+ "tokens_seen": 752877568
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237271815446338,
+ "loss": 1.3916,
+ "theoretical_loss": 3.780223472817233,
+ "tokens_seen": 752943104
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006237111334002006,
+ "loss": 1.4423,
+ "theoretical_loss": 3.7801907475721057,
+ "tokens_seen": 753008640
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006236950852557672,
+ "loss": 1.4797,
+ "theoretical_loss": 3.7801580259724084,
+ "tokens_seen": 753074176
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623679037111334,
+ "loss": 1.4956,
+ "theoretical_loss": 3.780125308017417,
+ "tokens_seen": 753139712
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006236629889669006,
+ "loss": 1.3677,
+ "theoretical_loss": 3.780092593706408,
+ "tokens_seen": 753205248
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006236469408224674,
+ "loss": 1.4972,
+ "theoretical_loss": 3.7800598830386587,
+ "tokens_seen": 753270784
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623630892678034,
+ "loss": 1.4059,
+ "theoretical_loss": 3.780027176013447,
+ "tokens_seen": 753336320
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006236148445336008,
+ "loss": 1.4172,
+ "theoretical_loss": 3.7799944726300505,
+ "tokens_seen": 753401856
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235987963891675,
+ "loss": 1.4357,
+ "theoretical_loss": 3.7799617728877464,
+ "tokens_seen": 753467392
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235827482447342,
+ "loss": 1.3688,
+ "theoretical_loss": 3.7799290767858125,
+ "tokens_seen": 753532928
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235667001003009,
+ "loss": 1.4547,
+ "theoretical_loss": 3.7798963843235276,
+ "tokens_seen": 753598464
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 256644,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4571774005889893,
+ "objective/train/theoretical_loss": 3.77986369550017,
+ "objective/train/tokens_used": 774124000,
+ "theoretical_loss": 3.77986369550017,
+ "tokens_seen": 753664000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235506519558676,
+ "loss": 1.448,
+ "theoretical_loss": 3.77986369550017,
+ "tokens_seen": 753664000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235346038114343,
+ "loss": 1.437,
+ "theoretical_loss": 3.779831010315018,
+ "tokens_seen": 753729536
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623518555667001,
+ "loss": 1.3776,
+ "theoretical_loss": 3.7797983287673502,
+ "tokens_seen": 753795072
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006235025075225677,
+ "loss": 1.4957,
+ "theoretical_loss": 3.7797656508564463,
+ "tokens_seen": 753860608
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006234864593781344,
+ "loss": 1.4569,
+ "theoretical_loss": 3.7797329765815846,
+ "tokens_seen": 753926144
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006234704112337011,
+ "loss": 1.4283,
+ "theoretical_loss": 3.779700305942045,
+ "tokens_seen": 753991680
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006234543630892678,
+ "loss": 1.3188,
+ "theoretical_loss": 3.7796676389371076,
+ "tokens_seen": 754057216
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006234383149448345,
+ "loss": 1.3921,
+ "theoretical_loss": 3.7796349755660517,
+ "tokens_seen": 754122752
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006234222668004012,
+ "loss": 1.4486,
+ "theoretical_loss": 3.7796023158281566,
+ "tokens_seen": 754188288
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623406218655968,
+ "loss": 1.3746,
+ "theoretical_loss": 3.7795696597227035,
+ "tokens_seen": 754253824
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006233901705115346,
+ "loss": 1.3456,
+ "theoretical_loss": 3.7795370072489733,
+ "tokens_seen": 754319360
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006233741223671014,
+ "loss": 1.4396,
+ "theoretical_loss": 3.7795043584062453,
+ "tokens_seen": 754384896
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623358074222668,
+ "loss": 1.4591,
+ "theoretical_loss": 3.779471713193801,
+ "tokens_seen": 754450432
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006233420260782348,
+ "loss": 1.4003,
+ "theoretical_loss": 3.7794390716109216,
+ "tokens_seen": 754515968
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006233259779338014,
+ "loss": 1.4405,
+ "theoretical_loss": 3.779406433656888,
+ "tokens_seen": 754581504
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006233099297893682,
+ "loss": 1.4479,
+ "theoretical_loss": 3.7793737993309824,
+ "tokens_seen": 754647040
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006232938816449348,
+ "loss": 1.2737,
+ "theoretical_loss": 3.7793411686324854,
+ "tokens_seen": 754712576
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006232778335005016,
+ "loss": 1.4082,
+ "theoretical_loss": 3.7793085415606793,
+ "tokens_seen": 754778112
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006232617853560682,
+ "loss": 1.4845,
+ "theoretical_loss": 3.7792759181148465,
+ "tokens_seen": 754843648
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623245737211635,
+ "loss": 1.2884,
+ "theoretical_loss": 3.7792432982942685,
+ "tokens_seen": 754909184
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006232296890672016,
+ "loss": 1.4281,
+ "theoretical_loss": 3.7792106820982285,
+ "tokens_seen": 754974720
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006232136409227684,
+ "loss": 1.4686,
+ "theoretical_loss": 3.779178069526009,
+ "tokens_seen": 755040256
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623197592778335,
+ "loss": 1.4515,
+ "theoretical_loss": 3.779145460576893,
+ "tokens_seen": 755105792
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231815446339018,
+ "loss": 1.4006,
+ "theoretical_loss": 3.7791128552501636,
+ "tokens_seen": 755171328
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231654964894684,
+ "loss": 1.5029,
+ "theoretical_loss": 3.7790802535451036,
+ "tokens_seen": 755236864
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231494483450352,
+ "loss": 1.4486,
+ "theoretical_loss": 3.779047655460997,
+ "tokens_seen": 755302400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231334002006018,
+ "loss": 1.4276,
+ "theoretical_loss": 3.779015060997127,
+ "tokens_seen": 755367936
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231173520561686,
+ "loss": 1.283,
+ "theoretical_loss": 3.778982470152778,
+ "tokens_seen": 755433472
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006231013039117352,
+ "loss": 1.5825,
+ "theoretical_loss": 3.7789498829272343,
+ "tokens_seen": 755499008
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623085255767302,
+ "loss": 1.4461,
+ "theoretical_loss": 3.7789172993197795,
+ "tokens_seen": 755564544
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006230692076228686,
+ "loss": 1.4566,
+ "theoretical_loss": 3.7788847193296986,
+ "tokens_seen": 755630080
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006230531594784354,
+ "loss": 1.3111,
+ "theoretical_loss": 3.7788521429562767,
+ "tokens_seen": 755695616
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000623037111334002,
+ "loss": 1.4424,
+ "theoretical_loss": 3.778819570198798,
+ "tokens_seen": 755761152
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006230210631895688,
+ "loss": 1.4257,
+ "theoretical_loss": 3.778787001056547,
+ "tokens_seen": 755826688
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006230050150451354,
+ "loss": 1.4097,
+ "theoretical_loss": 3.7787544355288105,
+ "tokens_seen": 755892224
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006229889669007022,
+ "loss": 1.3617,
+ "theoretical_loss": 3.7787218736148738,
+ "tokens_seen": 755957760
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006229729187562688,
+ "loss": 1.436,
+ "theoretical_loss": 3.778689315314022,
+ "tokens_seen": 756023296
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006229568706118356,
+ "loss": 1.2949,
+ "theoretical_loss": 3.778656760625541,
+ "tokens_seen": 756088832
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006229408224674022,
+ "loss": 1.4906,
+ "theoretical_loss": 3.7786242095487177,
+ "tokens_seen": 756154368
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622924774322969,
+ "loss": 1.4942,
+ "theoretical_loss": 3.778591662082838,
+ "tokens_seen": 756219904
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006229087261785356,
+ "loss": 1.3981,
+ "theoretical_loss": 3.7785591182271885,
+ "tokens_seen": 756285440
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006228926780341024,
+ "loss": 1.4899,
+ "theoretical_loss": 3.7785265779810553,
+ "tokens_seen": 756350976
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622876629889669,
+ "loss": 1.4776,
+ "theoretical_loss": 3.778494041343727,
+ "tokens_seen": 756416512
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006228605817452358,
+ "loss": 1.3764,
+ "theoretical_loss": 3.778461508314489,
+ "tokens_seen": 756482048
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006228445336008024,
+ "loss": 1.4456,
+ "theoretical_loss": 3.7784289788926295,
+ "tokens_seen": 756547584
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006228284854563692,
+ "loss": 1.4069,
+ "theoretical_loss": 3.778396453077436,
+ "tokens_seen": 756613120
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006228124373119358,
+ "loss": 1.4312,
+ "theoretical_loss": 3.778363930868196,
+ "tokens_seen": 756678656
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006227963891675026,
+ "loss": 1.4911,
+ "theoretical_loss": 3.778331412264198,
+ "tokens_seen": 756744192
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006227803410230692,
+ "loss": 1.4605,
+ "theoretical_loss": 3.7782988972647296,
+ "tokens_seen": 756809728
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622764292878636,
+ "loss": 1.3825,
+ "theoretical_loss": 3.7782663858690797,
+ "tokens_seen": 756875264
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 257583,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.471604824066162,
+ "objective/train/theoretical_loss": 3.778233878076536,
+ "objective/train/tokens_used": 777400800,
+ "theoretical_loss": 3.778233878076536,
+ "tokens_seen": 756940800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006227482447342026,
+ "loss": 1.4712,
+ "theoretical_loss": 3.778233878076536,
+ "tokens_seen": 756940800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006227321965897694,
+ "loss": 1.38,
+ "theoretical_loss": 3.7782013738863887,
+ "tokens_seen": 757006336
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622716148445336,
+ "loss": 1.2824,
+ "theoretical_loss": 3.7781688732979255,
+ "tokens_seen": 757071872
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006227001003009028,
+ "loss": 1.3731,
+ "theoretical_loss": 3.778136376310436,
+ "tokens_seen": 757137408
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006226840521564694,
+ "loss": 1.3583,
+ "theoretical_loss": 3.77810388292321,
+ "tokens_seen": 757202944
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006226680040120362,
+ "loss": 1.4256,
+ "theoretical_loss": 3.7780713931355367,
+ "tokens_seen": 757268480
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006226519558676028,
+ "loss": 1.446,
+ "theoretical_loss": 3.7780389069467057,
+ "tokens_seen": 757334016
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006226359077231696,
+ "loss": 1.3744,
+ "theoretical_loss": 3.778006424356007,
+ "tokens_seen": 757399552
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006226198595787362,
+ "loss": 1.4748,
+ "theoretical_loss": 3.7779739453627315,
+ "tokens_seen": 757465088
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622603811434303,
+ "loss": 1.304,
+ "theoretical_loss": 3.7779414699661684,
+ "tokens_seen": 757530624
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006225877632898696,
+ "loss": 1.3951,
+ "theoretical_loss": 3.7779089981656093,
+ "tokens_seen": 757596160
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006225717151454364,
+ "loss": 1.4539,
+ "theoretical_loss": 3.7778765299603445,
+ "tokens_seen": 757661696
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622555667001003,
+ "loss": 1.3827,
+ "theoretical_loss": 3.777844065349665,
+ "tokens_seen": 757727232
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006225396188565698,
+ "loss": 1.3948,
+ "theoretical_loss": 3.7778116043328627,
+ "tokens_seen": 757792768
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006225235707121364,
+ "loss": 1.4728,
+ "theoretical_loss": 3.777779146909228,
+ "tokens_seen": 757858304
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006225075225677032,
+ "loss": 1.4275,
+ "theoretical_loss": 3.777746693078053,
+ "tokens_seen": 757923840
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224914744232698,
+ "loss": 1.425,
+ "theoretical_loss": 3.7777142428386297,
+ "tokens_seen": 757989376
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224754262788365,
+ "loss": 1.3828,
+ "theoretical_loss": 3.777681796190249,
+ "tokens_seen": 758054912
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224593781344032,
+ "loss": 1.4134,
+ "theoretical_loss": 3.777649353132205,
+ "tokens_seen": 758120448
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224433299899699,
+ "loss": 1.4292,
+ "theoretical_loss": 3.7776169136637883,
+ "tokens_seen": 758185984
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224272818455367,
+ "loss": 1.4421,
+ "theoretical_loss": 3.7775844777842926,
+ "tokens_seen": 758251520
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006224112337011033,
+ "loss": 1.4663,
+ "theoretical_loss": 3.7775520454930103,
+ "tokens_seen": 758317056
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.00062239518555667,
+ "loss": 1.3652,
+ "theoretical_loss": 3.777519616789234,
+ "tokens_seen": 758382592
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006223791374122367,
+ "loss": 1.3367,
+ "theoretical_loss": 3.7774871916722574,
+ "tokens_seen": 758448128
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006223630892678035,
+ "loss": 1.5041,
+ "theoretical_loss": 3.7774547701413743,
+ "tokens_seen": 758513664
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006223470411233701,
+ "loss": 1.3251,
+ "theoretical_loss": 3.777422352195878,
+ "tokens_seen": 758579200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006223309929789367,
+ "loss": 1.3715,
+ "theoretical_loss": 3.7773899378350615,
+ "tokens_seen": 758644736
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006223149448345035,
+ "loss": 1.4444,
+ "theoretical_loss": 3.7773575270582196,
+ "tokens_seen": 758710272
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222988966900701,
+ "loss": 1.3804,
+ "theoretical_loss": 3.7773251198646465,
+ "tokens_seen": 758775808
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222828485456369,
+ "loss": 1.4785,
+ "theoretical_loss": 3.777292716253636,
+ "tokens_seen": 758841344
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222668004012035,
+ "loss": 1.3629,
+ "theoretical_loss": 3.777260316224484,
+ "tokens_seen": 758906880
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222507522567703,
+ "loss": 1.4352,
+ "theoretical_loss": 3.7772279197764838,
+ "tokens_seen": 758972416
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000622234704112337,
+ "loss": 1.423,
+ "theoretical_loss": 3.7771955269089315,
+ "tokens_seen": 759037952
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222186559679037,
+ "loss": 1.4486,
+ "theoretical_loss": 3.7771631376211214,
+ "tokens_seen": 759103488
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006222026078234705,
+ "loss": 1.4989,
+ "theoretical_loss": 3.7771307519123494,
+ "tokens_seen": 759169024
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221865596790371,
+ "loss": 1.4417,
+ "theoretical_loss": 3.7770983697819114,
+ "tokens_seen": 759234560
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221705115346039,
+ "loss": 1.4984,
+ "theoretical_loss": 3.7770659912291027,
+ "tokens_seen": 759300096
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221544633901705,
+ "loss": 1.4641,
+ "theoretical_loss": 3.7770336162532194,
+ "tokens_seen": 759365632
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221384152457373,
+ "loss": 1.3354,
+ "theoretical_loss": 3.7770012448535577,
+ "tokens_seen": 759431168
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221223671013039,
+ "loss": 1.4338,
+ "theoretical_loss": 3.7769688770294145,
+ "tokens_seen": 759496704
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006221063189568707,
+ "loss": 1.3725,
+ "theoretical_loss": 3.776936512780085,
+ "tokens_seen": 759562240
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220902708124373,
+ "loss": 1.3946,
+ "theoretical_loss": 3.776904152104868,
+ "tokens_seen": 759627776
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220742226680041,
+ "loss": 1.4352,
+ "theoretical_loss": 3.776871795003059,
+ "tokens_seen": 759693312
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220581745235707,
+ "loss": 1.5137,
+ "theoretical_loss": 3.7768394414739554,
+ "tokens_seen": 759758848
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220421263791375,
+ "loss": 1.4267,
+ "theoretical_loss": 3.7768070915168552,
+ "tokens_seen": 759824384
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220260782347041,
+ "loss": 1.4407,
+ "theoretical_loss": 3.7767747451310556,
+ "tokens_seen": 759889920
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006220100300902709,
+ "loss": 1.5487,
+ "theoretical_loss": 3.776742402315854,
+ "tokens_seen": 759955456
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219939819458375,
+ "loss": 1.3201,
+ "theoretical_loss": 3.7767100630705492,
+ "tokens_seen": 760020992
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219779338014043,
+ "loss": 1.4038,
+ "theoretical_loss": 3.776677727394439,
+ "tokens_seen": 760086528
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219618856569709,
+ "loss": 1.4596,
+ "theoretical_loss": 3.7766453952868217,
+ "tokens_seen": 760152064
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 258909,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.467252492904663,
+ "objective/train/theoretical_loss": 3.776613066746996,
+ "objective/train/tokens_used": 780677600,
+ "theoretical_loss": 3.776613066746996,
+ "tokens_seen": 760217600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219458375125377,
+ "loss": 1.5356,
+ "theoretical_loss": 3.776613066746996,
+ "tokens_seen": 760217600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219297893681043,
+ "loss": 1.3483,
+ "theoretical_loss": 3.7765807417742607,
+ "tokens_seen": 760283136
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006219137412236711,
+ "loss": 1.5033,
+ "theoretical_loss": 3.7765484203679147,
+ "tokens_seen": 760348672
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218976930792377,
+ "loss": 1.4532,
+ "theoretical_loss": 3.776516102527258,
+ "tokens_seen": 760414208
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218816449348045,
+ "loss": 1.4403,
+ "theoretical_loss": 3.7764837882515874,
+ "tokens_seen": 760479744
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218655967903711,
+ "loss": 1.4096,
+ "theoretical_loss": 3.7764514775402054,
+ "tokens_seen": 760545280
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218495486459379,
+ "loss": 1.471,
+ "theoretical_loss": 3.776419170392411,
+ "tokens_seen": 760610816
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218335005015045,
+ "loss": 1.4172,
+ "theoretical_loss": 3.7763868668075027,
+ "tokens_seen": 760676352
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218174523570713,
+ "loss": 1.4975,
+ "theoretical_loss": 3.776354566784783,
+ "tokens_seen": 760741888
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006218014042126379,
+ "loss": 1.5022,
+ "theoretical_loss": 3.77632227032355,
+ "tokens_seen": 760807424
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217853560682047,
+ "loss": 1.3702,
+ "theoretical_loss": 3.7762899774231062,
+ "tokens_seen": 760872960
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217693079237713,
+ "loss": 1.4161,
+ "theoretical_loss": 3.776257688082751,
+ "tokens_seen": 760938496
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217532597793381,
+ "loss": 1.4318,
+ "theoretical_loss": 3.776225402301786,
+ "tokens_seen": 761004032
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217372116349047,
+ "loss": 1.3119,
+ "theoretical_loss": 3.776193120079512,
+ "tokens_seen": 761069568
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217211634904715,
+ "loss": 1.4322,
+ "theoretical_loss": 3.7761608414152303,
+ "tokens_seen": 761135104
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006217051153460381,
+ "loss": 1.5481,
+ "theoretical_loss": 3.776128566308243,
+ "tokens_seen": 761200640
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216890672016049,
+ "loss": 1.4285,
+ "theoretical_loss": 3.776096294757852,
+ "tokens_seen": 761266176
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216730190571715,
+ "loss": 1.4636,
+ "theoretical_loss": 3.7760640267633576,
+ "tokens_seen": 761331712
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216569709127383,
+ "loss": 1.3541,
+ "theoretical_loss": 3.776031762324064,
+ "tokens_seen": 761397248
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216409227683049,
+ "loss": 1.3975,
+ "theoretical_loss": 3.775999501439273,
+ "tokens_seen": 761462784
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216248746238717,
+ "loss": 1.4355,
+ "theoretical_loss": 3.775967244108286,
+ "tokens_seen": 761528320
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006216088264794383,
+ "loss": 1.486,
+ "theoretical_loss": 3.775934990330407,
+ "tokens_seen": 761593856
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215927783350051,
+ "loss": 1.4797,
+ "theoretical_loss": 3.7759027401049385,
+ "tokens_seen": 761659392
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215767301905717,
+ "loss": 1.5027,
+ "theoretical_loss": 3.7758704934311833,
+ "tokens_seen": 761724928
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215606820461385,
+ "loss": 1.3778,
+ "theoretical_loss": 3.7758382503084453,
+ "tokens_seen": 761790464
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215446339017051,
+ "loss": 1.3964,
+ "theoretical_loss": 3.7758060107360274,
+ "tokens_seen": 761856000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215285857572719,
+ "loss": 1.4766,
+ "theoretical_loss": 3.775773774713234,
+ "tokens_seen": 761921536
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006215125376128385,
+ "loss": 1.5921,
+ "theoretical_loss": 3.775741542239369,
+ "tokens_seen": 761987072
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214964894684053,
+ "loss": 1.4706,
+ "theoretical_loss": 3.775709313313736,
+ "tokens_seen": 762052608
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214804413239719,
+ "loss": 1.4036,
+ "theoretical_loss": 3.775677087935639,
+ "tokens_seen": 762118144
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214643931795387,
+ "loss": 1.3538,
+ "theoretical_loss": 3.7756448661043835,
+ "tokens_seen": 762183680
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214483450351053,
+ "loss": 1.3597,
+ "theoretical_loss": 3.7756126478192735,
+ "tokens_seen": 762249216
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214322968906721,
+ "loss": 1.4321,
+ "theoretical_loss": 3.775580433079614,
+ "tokens_seen": 762314752
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214162487462387,
+ "loss": 1.3701,
+ "theoretical_loss": 3.7755482218847103,
+ "tokens_seen": 762380288
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006214002006018055,
+ "loss": 1.4022,
+ "theoretical_loss": 3.775516014233868,
+ "tokens_seen": 762445824
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213841524573721,
+ "loss": 1.4417,
+ "theoretical_loss": 3.7754838101263912,
+ "tokens_seen": 762511360
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213681043129389,
+ "loss": 1.3648,
+ "theoretical_loss": 3.7754516095615873,
+ "tokens_seen": 762576896
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213520561685055,
+ "loss": 1.375,
+ "theoretical_loss": 3.775419412538761,
+ "tokens_seen": 762642432
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213360080240723,
+ "loss": 1.3808,
+ "theoretical_loss": 3.7753872190572193,
+ "tokens_seen": 762707968
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213199598796389,
+ "loss": 1.4696,
+ "theoretical_loss": 3.775355029116268,
+ "tokens_seen": 762773504
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006213039117352057,
+ "loss": 1.465,
+ "theoretical_loss": 3.7753228427152123,
+ "tokens_seen": 762839040
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212878635907723,
+ "loss": 1.3666,
+ "theoretical_loss": 3.7752906598533613,
+ "tokens_seen": 762904576
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212718154463391,
+ "loss": 1.4051,
+ "theoretical_loss": 3.77525848053002,
+ "tokens_seen": 762970112
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212557673019057,
+ "loss": 1.4702,
+ "theoretical_loss": 3.7752263047444963,
+ "tokens_seen": 763035648
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212397191574725,
+ "loss": 1.4327,
+ "theoretical_loss": 3.7751941324960976,
+ "tokens_seen": 763101184
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212236710130391,
+ "loss": 1.4795,
+ "theoretical_loss": 3.7751619637841305,
+ "tokens_seen": 763166720
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006212076228686059,
+ "loss": 1.5067,
+ "theoretical_loss": 3.775129798607903,
+ "tokens_seen": 763232256
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006211915747241725,
+ "loss": 1.2731,
+ "theoretical_loss": 3.7750976369667226,
+ "tokens_seen": 763297792
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006211755265797393,
+ "loss": 1.4211,
+ "theoretical_loss": 3.7750654788598985,
+ "tokens_seen": 763363328
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000621159478435306,
+ "loss": 1.3284,
+ "theoretical_loss": 3.7750333242867375,
+ "tokens_seen": 763428864
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 259724,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3330668210983276,
+ "objective/train/theoretical_loss": 3.775001173246549,
+ "objective/train/tokens_used": 783954400,
+ "theoretical_loss": 3.775001173246549,
+ "tokens_seen": 763494400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006211434302908727,
+ "loss": 1.3791,
+ "theoretical_loss": 3.775001173246549,
+ "tokens_seen": 763494400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006211273821464393,
+ "loss": 1.3849,
+ "theoretical_loss": 3.774969025738642,
+ "tokens_seen": 763559936
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006211113340020061,
+ "loss": 1.4685,
+ "theoretical_loss": 3.7749368817623234,
+ "tokens_seen": 763625472
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210952858575727,
+ "loss": 1.3842,
+ "theoretical_loss": 3.774904741316904,
+ "tokens_seen": 763691008
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210792377131395,
+ "loss": 1.5269,
+ "theoretical_loss": 3.7748726044016916,
+ "tokens_seen": 763756544
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210631895687062,
+ "loss": 1.4646,
+ "theoretical_loss": 3.774840471015997,
+ "tokens_seen": 763822080
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210471414242728,
+ "loss": 1.3961,
+ "theoretical_loss": 3.774808341159129,
+ "tokens_seen": 763887616
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210310932798396,
+ "loss": 1.3616,
+ "theoretical_loss": 3.7747762148303976,
+ "tokens_seen": 763953152
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006210150451354062,
+ "loss": 1.3605,
+ "theoretical_loss": 3.7747440920291124,
+ "tokens_seen": 764018688
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620998996990973,
+ "loss": 1.4318,
+ "theoretical_loss": 3.774711972754584,
+ "tokens_seen": 764084224
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006209829488465396,
+ "loss": 1.4023,
+ "theoretical_loss": 3.7746798570061224,
+ "tokens_seen": 764149760
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006209669007021064,
+ "loss": 1.4544,
+ "theoretical_loss": 3.774647744783038,
+ "tokens_seen": 764215296
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620950852557673,
+ "loss": 1.4006,
+ "theoretical_loss": 3.7746156360846426,
+ "tokens_seen": 764280832
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006209348044132398,
+ "loss": 1.4134,
+ "theoretical_loss": 3.7745835309102462,
+ "tokens_seen": 764346368
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006209187562688064,
+ "loss": 1.4996,
+ "theoretical_loss": 3.7745514292591595,
+ "tokens_seen": 764411904
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006209027081243732,
+ "loss": 1.4596,
+ "theoretical_loss": 3.774519331130695,
+ "tokens_seen": 764477440
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006208866599799398,
+ "loss": 1.466,
+ "theoretical_loss": 3.7744872365241635,
+ "tokens_seen": 764542976
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006208706118355066,
+ "loss": 1.3785,
+ "theoretical_loss": 3.774455145438877,
+ "tokens_seen": 764608512
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006208545636910732,
+ "loss": 1.4326,
+ "theoretical_loss": 3.7744230578741473,
+ "tokens_seen": 764674048
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.00062083851554664,
+ "loss": 1.497,
+ "theoretical_loss": 3.7743909738292865,
+ "tokens_seen": 764739584
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006208224674022066,
+ "loss": 1.4737,
+ "theoretical_loss": 3.7743588933036065,
+ "tokens_seen": 764805120
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006208064192577734,
+ "loss": 1.3945,
+ "theoretical_loss": 3.774326816296421,
+ "tokens_seen": 764870656
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.00062079037111334,
+ "loss": 1.39,
+ "theoretical_loss": 3.774294742807041,
+ "tokens_seen": 764936192
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006207743229689068,
+ "loss": 1.4299,
+ "theoretical_loss": 3.774262672834781,
+ "tokens_seen": 765001728
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006207582748244734,
+ "loss": 1.3708,
+ "theoretical_loss": 3.7742306063789526,
+ "tokens_seen": 765067264
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006207422266800402,
+ "loss": 1.4407,
+ "theoretical_loss": 3.77419854343887,
+ "tokens_seen": 765132800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006207261785356068,
+ "loss": 1.4233,
+ "theoretical_loss": 3.774166484013846,
+ "tokens_seen": 765198336
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006207101303911736,
+ "loss": 1.455,
+ "theoretical_loss": 3.774134428103195,
+ "tokens_seen": 765263872
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006206940822467402,
+ "loss": 1.3979,
+ "theoretical_loss": 3.7741023757062306,
+ "tokens_seen": 765329408
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620678034102307,
+ "loss": 1.4455,
+ "theoretical_loss": 3.7740703268222666,
+ "tokens_seen": 765394944
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006206619859578736,
+ "loss": 1.4531,
+ "theoretical_loss": 3.7740382814506175,
+ "tokens_seen": 765460480
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006206459378134404,
+ "loss": 1.2891,
+ "theoretical_loss": 3.774006239590597,
+ "tokens_seen": 765526016
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620629889669007,
+ "loss": 1.3613,
+ "theoretical_loss": 3.773974201241521,
+ "tokens_seen": 765591552
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006206138415245738,
+ "loss": 1.3559,
+ "theoretical_loss": 3.773942166402703,
+ "tokens_seen": 765657088
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205977933801404,
+ "loss": 1.4377,
+ "theoretical_loss": 3.7739101350734585,
+ "tokens_seen": 765722624
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205817452357072,
+ "loss": 1.3769,
+ "theoretical_loss": 3.7738781072531027,
+ "tokens_seen": 765788160
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205656970912738,
+ "loss": 1.443,
+ "theoretical_loss": 3.7738460829409513,
+ "tokens_seen": 765853696
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205496489468406,
+ "loss": 1.2463,
+ "theoretical_loss": 3.773814062136319,
+ "tokens_seen": 765919232
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205336008024072,
+ "loss": 1.4535,
+ "theoretical_loss": 3.773782044838523,
+ "tokens_seen": 765984768
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620517552657974,
+ "loss": 1.3255,
+ "theoretical_loss": 3.7737500310468777,
+ "tokens_seen": 766050304
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006205015045135406,
+ "loss": 1.3543,
+ "theoretical_loss": 3.7737180207607004,
+ "tokens_seen": 766115840
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006204854563691074,
+ "loss": 1.4537,
+ "theoretical_loss": 3.7736860139793063,
+ "tokens_seen": 766181376
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620469408224674,
+ "loss": 1.428,
+ "theoretical_loss": 3.773654010702013,
+ "tokens_seen": 766246912
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006204533600802408,
+ "loss": 1.4351,
+ "theoretical_loss": 3.773622010928137,
+ "tokens_seen": 766312448
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006204373119358074,
+ "loss": 1.4944,
+ "theoretical_loss": 3.7735900146569947,
+ "tokens_seen": 766377984
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006204212637913742,
+ "loss": 1.5058,
+ "theoretical_loss": 3.7735580218879035,
+ "tokens_seen": 766443520
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006204052156469408,
+ "loss": 1.2958,
+ "theoretical_loss": 3.7735260326201807,
+ "tokens_seen": 766509056
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006203891675025076,
+ "loss": 1.2789,
+ "theoretical_loss": 3.773494046853144,
+ "tokens_seen": 766574592
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006203731193580742,
+ "loss": 1.4017,
+ "theoretical_loss": 3.773462064586111,
+ "tokens_seen": 766640128
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620357071213641,
+ "loss": 1.4233,
+ "theoretical_loss": 3.773430085818399,
+ "tokens_seen": 766705664
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 261084,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5048065185546875,
+ "objective/train/theoretical_loss": 3.7733981105493273,
+ "objective/train/tokens_used": 787231200,
+ "theoretical_loss": 3.7733981105493273,
+ "tokens_seen": 766771200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006203410230692076,
+ "loss": 1.4697,
+ "theoretical_loss": 3.7733981105493273,
+ "tokens_seen": 766771200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006203249749247744,
+ "loss": 1.3867,
+ "theoretical_loss": 3.7733661387782123,
+ "tokens_seen": 766836736
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620308926780341,
+ "loss": 1.4101,
+ "theoretical_loss": 3.7733341705043744,
+ "tokens_seen": 766902272
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202928786359078,
+ "loss": 1.3973,
+ "theoretical_loss": 3.7733022057271306,
+ "tokens_seen": 766967808
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202768304914744,
+ "loss": 1.4544,
+ "theoretical_loss": 3.7732702444458006,
+ "tokens_seen": 767033344
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202607823470412,
+ "loss": 1.5491,
+ "theoretical_loss": 3.773238286659704,
+ "tokens_seen": 767098880
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202447342026078,
+ "loss": 1.3937,
+ "theoretical_loss": 3.773206332368159,
+ "tokens_seen": 767164416
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202286860581746,
+ "loss": 1.3987,
+ "theoretical_loss": 3.7731743815704846,
+ "tokens_seen": 767229952
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006202126379137412,
+ "loss": 1.3838,
+ "theoretical_loss": 3.7731424342660014,
+ "tokens_seen": 767295488
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620196589769308,
+ "loss": 1.3372,
+ "theoretical_loss": 3.7731104904540294,
+ "tokens_seen": 767361024
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006201805416248746,
+ "loss": 1.3957,
+ "theoretical_loss": 3.7730785501338873,
+ "tokens_seen": 767426560
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006201644934804414,
+ "loss": 1.4313,
+ "theoretical_loss": 3.773046613304896,
+ "tokens_seen": 767492096
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620148445336008,
+ "loss": 1.3817,
+ "theoretical_loss": 3.7730146799663764,
+ "tokens_seen": 767557632
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006201323971915748,
+ "loss": 1.3762,
+ "theoretical_loss": 3.772982750117648,
+ "tokens_seen": 767623168
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006201163490471414,
+ "loss": 1.5095,
+ "theoretical_loss": 3.772950823758032,
+ "tokens_seen": 767688704
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006201003009027082,
+ "loss": 1.4444,
+ "theoretical_loss": 3.7729189008868493,
+ "tokens_seen": 767754240
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006200842527582748,
+ "loss": 1.4159,
+ "theoretical_loss": 3.7728869815034214,
+ "tokens_seen": 767819776
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006200682046138416,
+ "loss": 1.5091,
+ "theoretical_loss": 3.7728550656070685,
+ "tokens_seen": 767885312
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006200521564694082,
+ "loss": 1.4459,
+ "theoretical_loss": 3.7728231531971135,
+ "tokens_seen": 767950848
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000620036108324975,
+ "loss": 1.4411,
+ "theoretical_loss": 3.772791244272877,
+ "tokens_seen": 768016384
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006200200601805416,
+ "loss": 1.5223,
+ "theoretical_loss": 3.7727593388336813,
+ "tokens_seen": 768081920
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006200040120361084,
+ "loss": 1.4448,
+ "theoretical_loss": 3.7727274368788484,
+ "tokens_seen": 768147456
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000619987963891675,
+ "loss": 1.3049,
+ "theoretical_loss": 3.7726955384077008,
+ "tokens_seen": 768212992
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006199719157472418,
+ "loss": 1.4686,
+ "theoretical_loss": 3.7726636434195604,
+ "tokens_seen": 768278528
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006199558676028084,
+ "loss": 1.3818,
+ "theoretical_loss": 3.7726317519137504,
+ "tokens_seen": 768344064
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006199398194583752,
+ "loss": 1.401,
+ "theoretical_loss": 3.7725998638895932,
+ "tokens_seen": 768409600
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006199237713139418,
+ "loss": 1.4325,
+ "theoretical_loss": 3.772567979346412,
+ "tokens_seen": 768475136
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006199077231695086,
+ "loss": 1.43,
+ "theoretical_loss": 3.77253609828353,
+ "tokens_seen": 768540672
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006198916750250752,
+ "loss": 1.4822,
+ "theoretical_loss": 3.77250422070027,
+ "tokens_seen": 768606208
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000619875626880642,
+ "loss": 1.4113,
+ "theoretical_loss": 3.772472346595957,
+ "tokens_seen": 768671744
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006198595787362086,
+ "loss": 1.4157,
+ "theoretical_loss": 3.7724404759699137,
+ "tokens_seen": 768737280
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006198435305917754,
+ "loss": 1.369,
+ "theoretical_loss": 3.772408608821464,
+ "tokens_seen": 768802816
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000619827482447342,
+ "loss": 1.4812,
+ "theoretical_loss": 3.7723767451499324,
+ "tokens_seen": 768868352
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006198114343029088,
+ "loss": 1.4944,
+ "theoretical_loss": 3.772344884954643,
+ "tokens_seen": 768933888
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006197953861584754,
+ "loss": 1.4708,
+ "theoretical_loss": 3.77231302823492,
+ "tokens_seen": 768999424
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006197793380140422,
+ "loss": 1.4866,
+ "theoretical_loss": 3.772281174990089,
+ "tokens_seen": 769064960
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006197632898696088,
+ "loss": 1.3638,
+ "theoretical_loss": 3.772249325219475,
+ "tokens_seen": 769130496
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006197472417251756,
+ "loss": 1.5073,
+ "theoretical_loss": 3.7722174789224017,
+ "tokens_seen": 769196032
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006197311935807422,
+ "loss": 1.3497,
+ "theoretical_loss": 3.7721856360981953,
+ "tokens_seen": 769261568
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000619715145436309,
+ "loss": 1.3609,
+ "theoretical_loss": 3.7721537967461813,
+ "tokens_seen": 769327104
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196990972918757,
+ "loss": 1.4575,
+ "theoretical_loss": 3.7721219608656855,
+ "tokens_seen": 769392640
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196830491474424,
+ "loss": 1.4422,
+ "theoretical_loss": 3.7720901284560333,
+ "tokens_seen": 769458176
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000619667001003009,
+ "loss": 1.5332,
+ "theoretical_loss": 3.7720582995165506,
+ "tokens_seen": 769523712
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196509528585757,
+ "loss": 1.4122,
+ "theoretical_loss": 3.772026474046564,
+ "tokens_seen": 769589248
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196349047141425,
+ "loss": 1.3979,
+ "theoretical_loss": 3.7719946520454,
+ "tokens_seen": 769654784
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196188565697091,
+ "loss": 1.4553,
+ "theoretical_loss": 3.771962833512385,
+ "tokens_seen": 769720320
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006196028084252759,
+ "loss": 1.5624,
+ "theoretical_loss": 3.7719310184468458,
+ "tokens_seen": 769785856
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195867602808425,
+ "loss": 1.4628,
+ "theoretical_loss": 3.771899206848109,
+ "tokens_seen": 769851392
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195707121364093,
+ "loss": 1.4126,
+ "theoretical_loss": 3.7718673987155027,
+ "tokens_seen": 769916928
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195546639919759,
+ "loss": 1.3705,
+ "theoretical_loss": 3.771835594048353,
+ "tokens_seen": 769982464
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 261915,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4069135189056396,
+ "objective/train/theoretical_loss": 3.7718037928459887,
+ "objective/train/tokens_used": 790508000,
+ "theoretical_loss": 3.7718037928459887,
+ "tokens_seen": 770048000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195386158475427,
+ "loss": 1.4267,
+ "theoretical_loss": 3.7718037928459887,
+ "tokens_seen": 770048000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195225677031093,
+ "loss": 1.4309,
+ "theoretical_loss": 3.771771995107737,
+ "tokens_seen": 770113536
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006195065195586761,
+ "loss": 1.5265,
+ "theoretical_loss": 3.771740200832925,
+ "tokens_seen": 770179072
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194904714142427,
+ "loss": 1.4222,
+ "theoretical_loss": 3.771708410020882,
+ "tokens_seen": 770244608
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194744232698095,
+ "loss": 1.3883,
+ "theoretical_loss": 3.7716766226709355,
+ "tokens_seen": 770310144
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194583751253761,
+ "loss": 1.4449,
+ "theoretical_loss": 3.7716448387824144,
+ "tokens_seen": 770375680
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194423269809429,
+ "loss": 1.3884,
+ "theoretical_loss": 3.7716130583546477,
+ "tokens_seen": 770441216
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194262788365095,
+ "loss": 1.3459,
+ "theoretical_loss": 3.771581281386963,
+ "tokens_seen": 770506752
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006194102306920763,
+ "loss": 1.369,
+ "theoretical_loss": 3.7715495078786905,
+ "tokens_seen": 770572288
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193941825476429,
+ "loss": 1.3907,
+ "theoretical_loss": 3.771517737829159,
+ "tokens_seen": 770637824
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193781344032097,
+ "loss": 1.4249,
+ "theoretical_loss": 3.7714859712376985,
+ "tokens_seen": 770703360
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193620862587763,
+ "loss": 1.4423,
+ "theoretical_loss": 3.771454208103637,
+ "tokens_seen": 770768896
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193460381143431,
+ "loss": 1.4135,
+ "theoretical_loss": 3.771422448426306,
+ "tokens_seen": 770834432
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193299899699097,
+ "loss": 1.3792,
+ "theoretical_loss": 3.7713906922050344,
+ "tokens_seen": 770899968
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006193139418254765,
+ "loss": 1.4996,
+ "theoretical_loss": 3.771358939439153,
+ "tokens_seen": 770965504
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192978936810431,
+ "loss": 1.4811,
+ "theoretical_loss": 3.771327190127992,
+ "tokens_seen": 771031040
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192818455366099,
+ "loss": 1.4225,
+ "theoretical_loss": 3.7712954442708813,
+ "tokens_seen": 771096576
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192657973921765,
+ "loss": 1.3833,
+ "theoretical_loss": 3.7712637018671527,
+ "tokens_seen": 771162112
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192497492477433,
+ "loss": 1.3279,
+ "theoretical_loss": 3.7712319629161364,
+ "tokens_seen": 771227648
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192337011033099,
+ "loss": 1.4951,
+ "theoretical_loss": 3.771200227417163,
+ "tokens_seen": 771293184
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192176529588767,
+ "loss": 1.3993,
+ "theoretical_loss": 3.771168495369565,
+ "tokens_seen": 771358720
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006192016048144433,
+ "loss": 1.3692,
+ "theoretical_loss": 3.771136766772673,
+ "tokens_seen": 771424256
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191855566700101,
+ "loss": 1.4408,
+ "theoretical_loss": 3.7711050416258187,
+ "tokens_seen": 771489792
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191695085255767,
+ "loss": 1.3749,
+ "theoretical_loss": 3.771073319928335,
+ "tokens_seen": 771555328
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191534603811435,
+ "loss": 1.4492,
+ "theoretical_loss": 3.7710416016795523,
+ "tokens_seen": 771620864
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191374122367101,
+ "loss": 1.3585,
+ "theoretical_loss": 3.7710098868788036,
+ "tokens_seen": 771686400
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191213640922769,
+ "loss": 1.3964,
+ "theoretical_loss": 3.7709781755254212,
+ "tokens_seen": 771751936
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006191053159478435,
+ "loss": 1.3929,
+ "theoretical_loss": 3.770946467618738,
+ "tokens_seen": 771817472
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190892678034103,
+ "loss": 1.3236,
+ "theoretical_loss": 3.7709147631580864,
+ "tokens_seen": 771883008
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190732196589769,
+ "loss": 1.4498,
+ "theoretical_loss": 3.7708830621427993,
+ "tokens_seen": 771948544
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190571715145437,
+ "loss": 1.3861,
+ "theoretical_loss": 3.77085136457221,
+ "tokens_seen": 772014080
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190411233701103,
+ "loss": 1.3637,
+ "theoretical_loss": 3.7708196704456522,
+ "tokens_seen": 772079616
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190250752256771,
+ "loss": 1.415,
+ "theoretical_loss": 3.770787979762458,
+ "tokens_seen": 772145152
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006190090270812437,
+ "loss": 1.5101,
+ "theoretical_loss": 3.770756292521963,
+ "tokens_seen": 772210688
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189929789368105,
+ "loss": 1.4641,
+ "theoretical_loss": 3.7707246087235,
+ "tokens_seen": 772276224
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189769307923771,
+ "loss": 1.4672,
+ "theoretical_loss": 3.770692928366403,
+ "tokens_seen": 772341760
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189608826479439,
+ "loss": 1.5139,
+ "theoretical_loss": 3.7706612514500066,
+ "tokens_seen": 772407296
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189448345035105,
+ "loss": 1.363,
+ "theoretical_loss": 3.770629577973645,
+ "tokens_seen": 772472832
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189287863590773,
+ "loss": 1.5137,
+ "theoretical_loss": 3.7705979079366534,
+ "tokens_seen": 772538368
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006189127382146439,
+ "loss": 1.3931,
+ "theoretical_loss": 3.7705662413383654,
+ "tokens_seen": 772603904
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188966900702107,
+ "loss": 1.3853,
+ "theoretical_loss": 3.7705345781781174,
+ "tokens_seen": 772669440
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188806419257773,
+ "loss": 1.4165,
+ "theoretical_loss": 3.770502918455243,
+ "tokens_seen": 772734976
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188645937813441,
+ "loss": 1.5651,
+ "theoretical_loss": 3.770471262169079,
+ "tokens_seen": 772800512
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188485456369107,
+ "loss": 1.4451,
+ "theoretical_loss": 3.7704396093189607,
+ "tokens_seen": 772866048
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188324974924775,
+ "loss": 1.4238,
+ "theoretical_loss": 3.7704079599042233,
+ "tokens_seen": 772931584
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188164493480441,
+ "loss": 1.4405,
+ "theoretical_loss": 3.770376313924203,
+ "tokens_seen": 772997120
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006188004012036109,
+ "loss": 1.3301,
+ "theoretical_loss": 3.7703446713782354,
+ "tokens_seen": 773062656
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187843530591775,
+ "loss": 1.4101,
+ "theoretical_loss": 3.7703130322656575,
+ "tokens_seen": 773128192
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187683049147443,
+ "loss": 1.3621,
+ "theoretical_loss": 3.7702813965858053,
+ "tokens_seen": 773193728
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187522567703109,
+ "loss": 1.4221,
+ "theoretical_loss": 3.7702497643380157,
+ "tokens_seen": 773259264
+ },
+ {
+ "epoch": 0.23,
+ "objective/train/docs_used": 263140,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4101719856262207,
+ "objective/train/theoretical_loss": 3.7702181355216258,
+ "objective/train/tokens_used": 793784800,
+ "theoretical_loss": 3.7702181355216258,
+ "tokens_seen": 773324800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187362086258777,
+ "loss": 1.37,
+ "theoretical_loss": 3.7702181355216258,
+ "tokens_seen": 773324800
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187201604814443,
+ "loss": 1.3152,
+ "theoretical_loss": 3.770186510135972,
+ "tokens_seen": 773390336
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006187041123370111,
+ "loss": 1.4912,
+ "theoretical_loss": 3.7701548881803917,
+ "tokens_seen": 773455872
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186880641925777,
+ "loss": 1.3692,
+ "theoretical_loss": 3.7701232696542224,
+ "tokens_seen": 773521408
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186720160481445,
+ "loss": 1.4688,
+ "theoretical_loss": 3.7700916545568015,
+ "tokens_seen": 773586944
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186559679037111,
+ "loss": 1.55,
+ "theoretical_loss": 3.7700600428874678,
+ "tokens_seen": 773652480
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186399197592779,
+ "loss": 1.4596,
+ "theoretical_loss": 3.7700284346455577,
+ "tokens_seen": 773718016
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186238716148445,
+ "loss": 1.4967,
+ "theoretical_loss": 3.7699968298304096,
+ "tokens_seen": 773783552
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006186078234704113,
+ "loss": 1.4364,
+ "theoretical_loss": 3.7699652284413627,
+ "tokens_seen": 773849088
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000618591775325978,
+ "loss": 1.437,
+ "theoretical_loss": 3.769933630477755,
+ "tokens_seen": 773914624
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006185757271815447,
+ "loss": 1.4548,
+ "theoretical_loss": 3.769902035938925,
+ "tokens_seen": 773980160
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006185596790371114,
+ "loss": 1.4301,
+ "theoretical_loss": 3.7698704448242113,
+ "tokens_seen": 774045696
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006185436308926781,
+ "loss": 1.3961,
+ "theoretical_loss": 3.769838857132954,
+ "tokens_seen": 774111232
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006185275827482449,
+ "loss": 1.544,
+ "theoretical_loss": 3.769807272864491,
+ "tokens_seen": 774176768
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006185115346038115,
+ "loss": 1.5274,
+ "theoretical_loss": 3.7697756920181633,
+ "tokens_seen": 774242304
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184954864593783,
+ "loss": 1.4408,
+ "theoretical_loss": 3.769744114593309,
+ "tokens_seen": 774307840
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184794383149449,
+ "loss": 1.4132,
+ "theoretical_loss": 3.7697125405892686,
+ "tokens_seen": 774373376
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184633901705117,
+ "loss": 1.4891,
+ "theoretical_loss": 3.769680970005382,
+ "tokens_seen": 774438912
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184473420260783,
+ "loss": 1.4711,
+ "theoretical_loss": 3.769649402840989,
+ "tokens_seen": 774504448
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184312938816451,
+ "loss": 1.4862,
+ "theoretical_loss": 3.7696178390954302,
+ "tokens_seen": 774569984
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006184152457372117,
+ "loss": 1.396,
+ "theoretical_loss": 3.7695862787680463,
+ "tokens_seen": 774635520
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183991975927783,
+ "loss": 1.4648,
+ "theoretical_loss": 3.7695547218581775,
+ "tokens_seen": 774701056
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183831494483451,
+ "loss": 1.4278,
+ "theoretical_loss": 3.769523168365165,
+ "tokens_seen": 774766592
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183671013039117,
+ "loss": 1.3675,
+ "theoretical_loss": 3.7694916182883498,
+ "tokens_seen": 774832128
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183510531594785,
+ "loss": 1.3466,
+ "theoretical_loss": 3.7694600716270736,
+ "tokens_seen": 774897664
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183350050150452,
+ "loss": 1.5027,
+ "theoretical_loss": 3.7694285283806774,
+ "tokens_seen": 774963200
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183189568706119,
+ "loss": 1.3589,
+ "theoretical_loss": 3.769396988548502,
+ "tokens_seen": 775028736
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006183029087261786,
+ "loss": 1.4402,
+ "theoretical_loss": 3.7693654521298905,
+ "tokens_seen": 775094272
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006182868605817453,
+ "loss": 1.4952,
+ "theoretical_loss": 3.7693339191241844,
+ "tokens_seen": 775159808
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000618270812437312,
+ "loss": 1.3878,
+ "theoretical_loss": 3.769302389530726,
+ "tokens_seen": 775225344
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006182547642928786,
+ "loss": 1.4855,
+ "theoretical_loss": 3.769270863348857,
+ "tokens_seen": 775290880
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006182387161484454,
+ "loss": 1.4652,
+ "theoretical_loss": 3.7692393405779208,
+ "tokens_seen": 775356416
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.000618222668004012,
+ "loss": 1.4373,
+ "theoretical_loss": 3.7692078212172593,
+ "tokens_seen": 775421952
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 0.0006182066198595788,
+ "loss": 1.4672,
+ "theoretical_loss": 3.7691763052662157,
+ "tokens_seen": 775487488
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006181905717151454,
+ "loss": 1.4258,
+ "theoretical_loss": 3.7691447927241333,
+ "tokens_seen": 775553024
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006181745235707122,
+ "loss": 1.4884,
+ "theoretical_loss": 3.769113283590355,
+ "tokens_seen": 775618560
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006181584754262788,
+ "loss": 1.3293,
+ "theoretical_loss": 3.7690817778642245,
+ "tokens_seen": 775684096
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006181424272818456,
+ "loss": 1.3215,
+ "theoretical_loss": 3.7690502755450854,
+ "tokens_seen": 775749632
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006181263791374122,
+ "loss": 1.4058,
+ "theoretical_loss": 3.769018776632281,
+ "tokens_seen": 775815168
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000618110330992979,
+ "loss": 1.3601,
+ "theoretical_loss": 3.768987281125156,
+ "tokens_seen": 775880704
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006180942828485456,
+ "loss": 1.3727,
+ "theoretical_loss": 3.7689557890230545,
+ "tokens_seen": 775946240
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006180782347041124,
+ "loss": 1.4911,
+ "theoretical_loss": 3.7689243003253203,
+ "tokens_seen": 776011776
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000618062186559679,
+ "loss": 1.4389,
+ "theoretical_loss": 3.768892815031298,
+ "tokens_seen": 776077312
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006180461384152458,
+ "loss": 1.4177,
+ "theoretical_loss": 3.768861333140333,
+ "tokens_seen": 776142848
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006180300902708124,
+ "loss": 1.4762,
+ "theoretical_loss": 3.768829854651769,
+ "tokens_seen": 776208384
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006180140421263792,
+ "loss": 1.478,
+ "theoretical_loss": 3.7687983795649522,
+ "tokens_seen": 776273920
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006179979939819458,
+ "loss": 1.341,
+ "theoretical_loss": 3.768766907879227,
+ "tokens_seen": 776339456
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006179819458375126,
+ "loss": 1.4228,
+ "theoretical_loss": 3.7687354395939394,
+ "tokens_seen": 776404992
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006179658976930792,
+ "loss": 1.4807,
+ "theoretical_loss": 3.7687039747084348,
+ "tokens_seen": 776470528
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617949849548646,
+ "loss": 1.2837,
+ "theoretical_loss": 3.768672513222059,
+ "tokens_seen": 776536064
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 264403,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4671874046325684,
+ "objective/train/theoretical_loss": 3.7686410551341583,
+ "objective/train/tokens_used": 797061600,
+ "theoretical_loss": 3.7686410551341583,
+ "tokens_seen": 776601600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006179338014042126,
+ "loss": 1.3777,
+ "theoretical_loss": 3.7686410551341583,
+ "tokens_seen": 776601600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006179177532597794,
+ "loss": 1.4223,
+ "theoretical_loss": 3.768609600444078,
+ "tokens_seen": 776667136
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617901705115346,
+ "loss": 1.44,
+ "theoretical_loss": 3.768578149151165,
+ "tokens_seen": 776732672
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178856569709128,
+ "loss": 1.3614,
+ "theoretical_loss": 3.768546701254766,
+ "tokens_seen": 776798208
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178696088264794,
+ "loss": 1.3948,
+ "theoretical_loss": 3.7685152567542275,
+ "tokens_seen": 776863744
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178535606820462,
+ "loss": 1.467,
+ "theoretical_loss": 3.7684838156488962,
+ "tokens_seen": 776929280
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178375125376128,
+ "loss": 1.4427,
+ "theoretical_loss": 3.7684523779381194,
+ "tokens_seen": 776994816
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178214643931796,
+ "loss": 1.4314,
+ "theoretical_loss": 3.768420943621244,
+ "tokens_seen": 777060352
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006178054162487462,
+ "loss": 1.4692,
+ "theoretical_loss": 3.7683895126976177,
+ "tokens_seen": 777125888
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617789368104313,
+ "loss": 1.49,
+ "theoretical_loss": 3.7683580851665885,
+ "tokens_seen": 777191424
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006177733199598796,
+ "loss": 1.3269,
+ "theoretical_loss": 3.768326661027503,
+ "tokens_seen": 777256960
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006177572718154464,
+ "loss": 1.4427,
+ "theoretical_loss": 3.7682952402797105,
+ "tokens_seen": 777322496
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617741223671013,
+ "loss": 1.4641,
+ "theoretical_loss": 3.7682638229225582,
+ "tokens_seen": 777388032
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006177251755265798,
+ "loss": 1.3876,
+ "theoretical_loss": 3.7682324089553947,
+ "tokens_seen": 777453568
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006177091273821464,
+ "loss": 1.4079,
+ "theoretical_loss": 3.768200998377569,
+ "tokens_seen": 777519104
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006176930792377132,
+ "loss": 1.4262,
+ "theoretical_loss": 3.7681695911884288,
+ "tokens_seen": 777584640
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006176770310932798,
+ "loss": 1.353,
+ "theoretical_loss": 3.768138187387324,
+ "tokens_seen": 777650176
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006176609829488466,
+ "loss": 1.3663,
+ "theoretical_loss": 3.7681067869736022,
+ "tokens_seen": 777715712
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006176449348044132,
+ "loss": 1.4153,
+ "theoretical_loss": 3.768075389946614,
+ "tokens_seen": 777781248
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061762888665998,
+ "loss": 1.4123,
+ "theoretical_loss": 3.7680439963057086,
+ "tokens_seen": 777846784
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006176128385155466,
+ "loss": 1.5162,
+ "theoretical_loss": 3.7680126060502355,
+ "tokens_seen": 777912320
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175967903711134,
+ "loss": 1.41,
+ "theoretical_loss": 3.7679812191795437,
+ "tokens_seen": 777977856
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061758074222668,
+ "loss": 1.4208,
+ "theoretical_loss": 3.767949835692984,
+ "tokens_seen": 778043392
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175646940822468,
+ "loss": 1.4099,
+ "theoretical_loss": 3.767918455589906,
+ "tokens_seen": 778108928
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175486459378134,
+ "loss": 1.4533,
+ "theoretical_loss": 3.76788707886966,
+ "tokens_seen": 778174464
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175325977933802,
+ "loss": 1.3446,
+ "theoretical_loss": 3.7678557055315975,
+ "tokens_seen": 778240000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175165496489468,
+ "loss": 1.4287,
+ "theoretical_loss": 3.767824335575068,
+ "tokens_seen": 778305536
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006175005015045136,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7677929689994225,
+ "tokens_seen": 778371072
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006174844533600802,
+ "loss": 1.3957,
+ "theoretical_loss": 3.7677616058040124,
+ "tokens_seen": 778436608
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617468405215647,
+ "loss": 1.3862,
+ "theoretical_loss": 3.7677302459881883,
+ "tokens_seen": 778502144
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006174523570712136,
+ "loss": 1.3104,
+ "theoretical_loss": 3.767698889551302,
+ "tokens_seen": 778567680
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006174363089267804,
+ "loss": 1.3043,
+ "theoretical_loss": 3.7676675364927057,
+ "tokens_seen": 778633216
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617420260782347,
+ "loss": 1.4242,
+ "theoretical_loss": 3.76763618681175,
+ "tokens_seen": 778698752
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006174042126379138,
+ "loss": 1.3909,
+ "theoretical_loss": 3.7676048405077873,
+ "tokens_seen": 778764288
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006173881644934805,
+ "loss": 1.2704,
+ "theoretical_loss": 3.7675734975801696,
+ "tokens_seen": 778829824
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006173721163490472,
+ "loss": 1.513,
+ "theoretical_loss": 3.7675421580282498,
+ "tokens_seen": 778895360
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006173560682046139,
+ "loss": 1.3417,
+ "theoretical_loss": 3.7675108218513786,
+ "tokens_seen": 778960896
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006173400200601806,
+ "loss": 1.3729,
+ "theoretical_loss": 3.7674794890489105,
+ "tokens_seen": 779026432
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006173239719157473,
+ "loss": 1.3879,
+ "theoretical_loss": 3.767448159620198,
+ "tokens_seen": 779091968
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617307923771314,
+ "loss": 1.3719,
+ "theoretical_loss": 3.7674168335645932,
+ "tokens_seen": 779157504
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172918756268807,
+ "loss": 1.4471,
+ "theoretical_loss": 3.7673855108814496,
+ "tokens_seen": 779223040
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172758274824474,
+ "loss": 1.3563,
+ "theoretical_loss": 3.7673541915701207,
+ "tokens_seen": 779288576
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172597793380141,
+ "loss": 1.3779,
+ "theoretical_loss": 3.76732287562996,
+ "tokens_seen": 779354112
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172437311935808,
+ "loss": 1.3935,
+ "theoretical_loss": 3.767291563060321,
+ "tokens_seen": 779419648
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172276830491475,
+ "loss": 1.3951,
+ "theoretical_loss": 3.7672602538605586,
+ "tokens_seen": 779485184
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006172116349047142,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7672289480300254,
+ "tokens_seen": 779550720
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617195586760281,
+ "loss": 1.4581,
+ "theoretical_loss": 3.7671976455680767,
+ "tokens_seen": 779616256
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006171795386158476,
+ "loss": 1.4006,
+ "theoretical_loss": 3.767166346474066,
+ "tokens_seen": 779681792
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006171634904714144,
+ "loss": 1.4974,
+ "theoretical_loss": 3.7671350507473482,
+ "tokens_seen": 779747328
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617147442326981,
+ "loss": 1.4772,
+ "theoretical_loss": 3.7671037583872784,
+ "tokens_seen": 779812864
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 265321,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4230022430419922,
+ "objective/train/theoretical_loss": 3.767072469393211,
+ "objective/train/tokens_used": 800338400,
+ "theoretical_loss": 3.767072469393211,
+ "tokens_seen": 779878400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006171313941825478,
+ "loss": 1.4604,
+ "theoretical_loss": 3.767072469393211,
+ "tokens_seen": 779878400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006171153460381144,
+ "loss": 1.4413,
+ "theoretical_loss": 3.767041183764502,
+ "tokens_seen": 779943936
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170992978936812,
+ "loss": 1.3497,
+ "theoretical_loss": 3.767009901500506,
+ "tokens_seen": 780009472
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170832497492478,
+ "loss": 1.477,
+ "theoretical_loss": 3.766978622600579,
+ "tokens_seen": 780075008
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170672016048146,
+ "loss": 1.4941,
+ "theoretical_loss": 3.7669473470640757,
+ "tokens_seen": 780140544
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170511534603812,
+ "loss": 1.3191,
+ "theoretical_loss": 3.7669160748903527,
+ "tokens_seen": 780206080
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000617035105315948,
+ "loss": 1.3044,
+ "theoretical_loss": 3.766884806078766,
+ "tokens_seen": 780271616
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170190571715146,
+ "loss": 1.394,
+ "theoretical_loss": 3.766853540628672,
+ "tokens_seen": 780337152
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006170030090270812,
+ "loss": 1.3806,
+ "theoretical_loss": 3.7668222785394256,
+ "tokens_seen": 780402688
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000616986960882648,
+ "loss": 1.421,
+ "theoretical_loss": 3.766791019810385,
+ "tokens_seen": 780468224
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006169709127382146,
+ "loss": 1.3045,
+ "theoretical_loss": 3.7667597644409065,
+ "tokens_seen": 780533760
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006169548645937814,
+ "loss": 1.4526,
+ "theoretical_loss": 3.7667285124303467,
+ "tokens_seen": 780599296
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000616938816449348,
+ "loss": 1.4164,
+ "theoretical_loss": 3.7666972637780627,
+ "tokens_seen": 780664832
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006169227683049148,
+ "loss": 1.4292,
+ "theoretical_loss": 3.766666018483412,
+ "tokens_seen": 780730368
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006169067201604815,
+ "loss": 1.4096,
+ "theoretical_loss": 3.766634776545752,
+ "tokens_seen": 780795904
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168906720160482,
+ "loss": 1.431,
+ "theoretical_loss": 3.76660353796444,
+ "tokens_seen": 780861440
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168746238716149,
+ "loss": 1.3841,
+ "theoretical_loss": 3.7665723027388336,
+ "tokens_seen": 780926976
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168585757271815,
+ "loss": 1.3696,
+ "theoretical_loss": 3.7665410708682914,
+ "tokens_seen": 780992512
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168425275827483,
+ "loss": 1.3269,
+ "theoretical_loss": 3.7665098423521712,
+ "tokens_seen": 781058048
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168264794383149,
+ "loss": 1.2709,
+ "theoretical_loss": 3.766478617189832,
+ "tokens_seen": 781123584
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006168104312938817,
+ "loss": 1.3507,
+ "theoretical_loss": 3.766447395380631,
+ "tokens_seen": 781189120
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167943831494483,
+ "loss": 1.4879,
+ "theoretical_loss": 3.7664161769239275,
+ "tokens_seen": 781254656
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167783350050151,
+ "loss": 1.3632,
+ "theoretical_loss": 3.76638496181908,
+ "tokens_seen": 781320192
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167622868605817,
+ "loss": 1.3393,
+ "theoretical_loss": 3.7663537500654485,
+ "tokens_seen": 781385728
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167462387161485,
+ "loss": 1.5013,
+ "theoretical_loss": 3.766322541662391,
+ "tokens_seen": 781451264
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167301905717151,
+ "loss": 1.3433,
+ "theoretical_loss": 3.766291336609268,
+ "tokens_seen": 781516800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006167141424272819,
+ "loss": 1.3584,
+ "theoretical_loss": 3.766260134905438,
+ "tokens_seen": 781582336
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166980942828485,
+ "loss": 1.4709,
+ "theoretical_loss": 3.7662289365502617,
+ "tokens_seen": 781647872
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166820461384153,
+ "loss": 1.3859,
+ "theoretical_loss": 3.766197741543098,
+ "tokens_seen": 781713408
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166659979939819,
+ "loss": 1.3007,
+ "theoretical_loss": 3.766166549883308,
+ "tokens_seen": 781778944
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166499498495487,
+ "loss": 1.415,
+ "theoretical_loss": 3.7661353615702504,
+ "tokens_seen": 781844480
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166339017051153,
+ "loss": 1.3874,
+ "theoretical_loss": 3.7661041766032874,
+ "tokens_seen": 781910016
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166178535606821,
+ "loss": 1.4317,
+ "theoretical_loss": 3.7660729949817786,
+ "tokens_seen": 781975552
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006166018054162487,
+ "loss": 1.3863,
+ "theoretical_loss": 3.7660418167050853,
+ "tokens_seen": 782041088
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165857572718155,
+ "loss": 1.4142,
+ "theoretical_loss": 3.7660106417725676,
+ "tokens_seen": 782106624
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165697091273821,
+ "loss": 1.3271,
+ "theoretical_loss": 3.765979470183588,
+ "tokens_seen": 782172160
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165536609829489,
+ "loss": 1.4316,
+ "theoretical_loss": 3.765948301937506,
+ "tokens_seen": 782237696
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165376128385155,
+ "loss": 1.3685,
+ "theoretical_loss": 3.7659171370336844,
+ "tokens_seen": 782303232
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165215646940823,
+ "loss": 1.283,
+ "theoretical_loss": 3.765885975471485,
+ "tokens_seen": 782368768
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006165055165496489,
+ "loss": 1.4852,
+ "theoretical_loss": 3.7658548172502684,
+ "tokens_seen": 782434304
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164894684052157,
+ "loss": 1.4444,
+ "theoretical_loss": 3.7658236623693977,
+ "tokens_seen": 782499840
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164734202607823,
+ "loss": 1.4744,
+ "theoretical_loss": 3.765792510828234,
+ "tokens_seen": 782565376
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164573721163491,
+ "loss": 1.3762,
+ "theoretical_loss": 3.7657613626261415,
+ "tokens_seen": 782630912
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164413239719157,
+ "loss": 1.3975,
+ "theoretical_loss": 3.765730217762481,
+ "tokens_seen": 782696448
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164252758274825,
+ "loss": 1.3729,
+ "theoretical_loss": 3.765699076236616,
+ "tokens_seen": 782761984
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006164092276830491,
+ "loss": 1.4487,
+ "theoretical_loss": 3.7656679380479092,
+ "tokens_seen": 782827520
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163931795386159,
+ "loss": 1.3424,
+ "theoretical_loss": 3.7656368031957235,
+ "tokens_seen": 782893056
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163771313941825,
+ "loss": 1.4558,
+ "theoretical_loss": 3.765605671679422,
+ "tokens_seen": 782958592
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163610832497493,
+ "loss": 1.3998,
+ "theoretical_loss": 3.765574543498369,
+ "tokens_seen": 783024128
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163450351053159,
+ "loss": 1.3661,
+ "theoretical_loss": 3.765543418651927,
+ "tokens_seen": 783089664
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 266686,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4450281858444214,
+ "objective/train/theoretical_loss": 3.765512297139461,
+ "objective/train/tokens_used": 803615200,
+ "theoretical_loss": 3.765512297139461,
+ "tokens_seen": 783155200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163289869608827,
+ "loss": 1.3858,
+ "theoretical_loss": 3.765512297139461,
+ "tokens_seen": 783155200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006163129388164493,
+ "loss": 1.5334,
+ "theoretical_loss": 3.765481178960333,
+ "tokens_seen": 783220736
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162968906720161,
+ "loss": 1.3507,
+ "theoretical_loss": 3.765450064113909,
+ "tokens_seen": 783286272
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162808425275827,
+ "loss": 1.3889,
+ "theoretical_loss": 3.7654189525995516,
+ "tokens_seen": 783351808
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162647943831495,
+ "loss": 1.4549,
+ "theoretical_loss": 3.7653878444166273,
+ "tokens_seen": 783417344
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162487462387161,
+ "loss": 1.4686,
+ "theoretical_loss": 3.765356739564499,
+ "tokens_seen": 783482880
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162326980942829,
+ "loss": 1.4782,
+ "theoretical_loss": 3.7653256380425324,
+ "tokens_seen": 783548416
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162166499498495,
+ "loss": 1.3945,
+ "theoretical_loss": 3.7652945398500917,
+ "tokens_seen": 783613952
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006162006018054163,
+ "loss": 1.3612,
+ "theoretical_loss": 3.7652634449865428,
+ "tokens_seen": 783679488
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000616184553660983,
+ "loss": 1.4655,
+ "theoretical_loss": 3.7652323534512506,
+ "tokens_seen": 783745024
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006161685055165497,
+ "loss": 1.416,
+ "theoretical_loss": 3.765201265243581,
+ "tokens_seen": 783810560
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006161524573721164,
+ "loss": 1.4005,
+ "theoretical_loss": 3.765170180362899,
+ "tokens_seen": 783876096
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006161364092276831,
+ "loss": 1.4177,
+ "theoretical_loss": 3.7651390988085716,
+ "tokens_seen": 783941632
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006161203610832498,
+ "loss": 1.3948,
+ "theoretical_loss": 3.765108020579963,
+ "tokens_seen": 784007168
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006161043129388165,
+ "loss": 1.4268,
+ "theoretical_loss": 3.7650769456764417,
+ "tokens_seen": 784072704
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006160882647943832,
+ "loss": 1.3296,
+ "theoretical_loss": 3.765045874097372,
+ "tokens_seen": 784138240
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006160722166499499,
+ "loss": 1.5356,
+ "theoretical_loss": 3.7650148058421213,
+ "tokens_seen": 784203776
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006160561685055166,
+ "loss": 1.4808,
+ "theoretical_loss": 3.7649837409100564,
+ "tokens_seen": 784269312
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006160401203610833,
+ "loss": 1.4401,
+ "theoretical_loss": 3.7649526793005443,
+ "tokens_seen": 784334848
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061602407221665,
+ "loss": 1.4127,
+ "theoretical_loss": 3.7649216210129515,
+ "tokens_seen": 784400384
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006160080240722167,
+ "loss": 1.5829,
+ "theoretical_loss": 3.764890566046646,
+ "tokens_seen": 784465920
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159919759277834,
+ "loss": 1.3616,
+ "theoretical_loss": 3.764859514400994,
+ "tokens_seen": 784531456
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159759277833501,
+ "loss": 1.495,
+ "theoretical_loss": 3.7648284660753646,
+ "tokens_seen": 784596992
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159598796389168,
+ "loss": 1.45,
+ "theoretical_loss": 3.7647974210691246,
+ "tokens_seen": 784662528
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159438314944835,
+ "loss": 1.4143,
+ "theoretical_loss": 3.7647663793816424,
+ "tokens_seen": 784728064
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159277833500502,
+ "loss": 1.4681,
+ "theoretical_loss": 3.764735341012285,
+ "tokens_seen": 784793600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006159117352056169,
+ "loss": 1.4675,
+ "theoretical_loss": 3.7647043059604224,
+ "tokens_seen": 784859136
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006158956870611836,
+ "loss": 1.5275,
+ "theoretical_loss": 3.7646732742254216,
+ "tokens_seen": 784924672
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006158796389167503,
+ "loss": 1.3571,
+ "theoretical_loss": 3.764642245806652,
+ "tokens_seen": 784990208
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615863590772317,
+ "loss": 1.309,
+ "theoretical_loss": 3.7646112207034825,
+ "tokens_seen": 785055744
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006158475426278837,
+ "loss": 1.3369,
+ "theoretical_loss": 3.764580198915281,
+ "tokens_seen": 785121280
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006158314944834504,
+ "loss": 1.3798,
+ "theoretical_loss": 3.764549180441418,
+ "tokens_seen": 785186816
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006158154463390171,
+ "loss": 1.2188,
+ "theoretical_loss": 3.7645181652812623,
+ "tokens_seen": 785252352
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157993981945839,
+ "loss": 1.4434,
+ "theoretical_loss": 3.7644871534341826,
+ "tokens_seen": 785317888
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157833500501505,
+ "loss": 1.4183,
+ "theoretical_loss": 3.7644561448995497,
+ "tokens_seen": 785383424
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157673019057173,
+ "loss": 1.4196,
+ "theoretical_loss": 3.764425139676733,
+ "tokens_seen": 785448960
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157512537612839,
+ "loss": 1.3556,
+ "theoretical_loss": 3.7643941377651022,
+ "tokens_seen": 785514496
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157352056168507,
+ "loss": 1.4348,
+ "theoretical_loss": 3.7643631391640278,
+ "tokens_seen": 785580032
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006157191574724173,
+ "loss": 1.4476,
+ "theoretical_loss": 3.7643321438728803,
+ "tokens_seen": 785645568
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615703109327984,
+ "loss": 1.4187,
+ "theoretical_loss": 3.7643011518910297,
+ "tokens_seen": 785711104
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156870611835507,
+ "loss": 1.4248,
+ "theoretical_loss": 3.764270163217847,
+ "tokens_seen": 785776640
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156710130391175,
+ "loss": 1.5277,
+ "theoretical_loss": 3.764239177852703,
+ "tokens_seen": 785842176
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156549648946841,
+ "loss": 1.3589,
+ "theoretical_loss": 3.764208195794969,
+ "tokens_seen": 785907712
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156389167502509,
+ "loss": 1.4421,
+ "theoretical_loss": 3.7641772170440158,
+ "tokens_seen": 785973248
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156228686058175,
+ "loss": 1.3881,
+ "theoretical_loss": 3.7641462415992155,
+ "tokens_seen": 786038784
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006156068204613843,
+ "loss": 1.3179,
+ "theoretical_loss": 3.7641152694599382,
+ "tokens_seen": 786104320
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006155907723169509,
+ "loss": 1.4547,
+ "theoretical_loss": 3.764084300625557,
+ "tokens_seen": 786169856
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006155747241725176,
+ "loss": 1.3807,
+ "theoretical_loss": 3.7640533350954435,
+ "tokens_seen": 786235392
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006155586760280843,
+ "loss": 1.3404,
+ "theoretical_loss": 3.7640223728689692,
+ "tokens_seen": 786300928
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615542627883651,
+ "loss": 1.4187,
+ "theoretical_loss": 3.763991413945507,
+ "tokens_seen": 786366464
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 267627,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3336844444274902,
+ "objective/train/theoretical_loss": 3.763960458324429,
+ "objective/train/tokens_used": 806892000,
+ "theoretical_loss": 3.763960458324429,
+ "tokens_seen": 786432000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006155265797392177,
+ "loss": 1.3624,
+ "theoretical_loss": 3.763960458324429,
+ "tokens_seen": 786432000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006155105315947844,
+ "loss": 1.373,
+ "theoretical_loss": 3.763929506005108,
+ "tokens_seen": 786497536
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154944834503511,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7638985569869163,
+ "tokens_seen": 786563072
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154784353059178,
+ "loss": 1.4227,
+ "theoretical_loss": 3.7638676112692275,
+ "tokens_seen": 786628608
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154623871614844,
+ "loss": 1.4224,
+ "theoretical_loss": 3.7638366688514138,
+ "tokens_seen": 786694144
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154463390170512,
+ "loss": 1.3557,
+ "theoretical_loss": 3.7638057297328493,
+ "tokens_seen": 786759680
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154302908726178,
+ "loss": 1.4047,
+ "theoretical_loss": 3.7637747939129067,
+ "tokens_seen": 786825216
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006154142427281846,
+ "loss": 1.4713,
+ "theoretical_loss": 3.7637438613909606,
+ "tokens_seen": 786890752
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006153981945837512,
+ "loss": 1.4039,
+ "theoretical_loss": 3.7637129321663836,
+ "tokens_seen": 786956288
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615382146439318,
+ "loss": 1.4927,
+ "theoretical_loss": 3.7636820062385503,
+ "tokens_seen": 787021824
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006153660982948846,
+ "loss": 1.3567,
+ "theoretical_loss": 3.7636510836068355,
+ "tokens_seen": 787087360
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006153500501504514,
+ "loss": 1.4755,
+ "theoretical_loss": 3.7636201642706117,
+ "tokens_seen": 787152896
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615334002006018,
+ "loss": 1.4107,
+ "theoretical_loss": 3.7635892482292554,
+ "tokens_seen": 787218432
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006153179538615848,
+ "loss": 1.3844,
+ "theoretical_loss": 3.7635583354821396,
+ "tokens_seen": 787283968
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006153019057171514,
+ "loss": 1.4169,
+ "theoretical_loss": 3.7635274260286398,
+ "tokens_seen": 787349504
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006152858575727182,
+ "loss": 1.5145,
+ "theoretical_loss": 3.763496519868131,
+ "tokens_seen": 787415040
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006152698094282848,
+ "loss": 1.3417,
+ "theoretical_loss": 3.7634656169999885,
+ "tokens_seen": 787480576
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006152537612838516,
+ "loss": 1.4822,
+ "theoretical_loss": 3.7634347174235874,
+ "tokens_seen": 787546112
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006152377131394182,
+ "loss": 1.4409,
+ "theoretical_loss": 3.7634038211383025,
+ "tokens_seen": 787611648
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615221664994985,
+ "loss": 1.4063,
+ "theoretical_loss": 3.763372928143511,
+ "tokens_seen": 787677184
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006152056168505516,
+ "loss": 1.5267,
+ "theoretical_loss": 3.763342038438587,
+ "tokens_seen": 787742720
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006151895687061184,
+ "loss": 1.4283,
+ "theoretical_loss": 3.7633111520229083,
+ "tokens_seen": 787808256
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615173520561685,
+ "loss": 1.5008,
+ "theoretical_loss": 3.7632802688958495,
+ "tokens_seen": 787873792
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006151574724172518,
+ "loss": 1.4748,
+ "theoretical_loss": 3.7632493890567877,
+ "tokens_seen": 787939328
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006151414242728184,
+ "loss": 1.3924,
+ "theoretical_loss": 3.7632185125051,
+ "tokens_seen": 788004864
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006151253761283852,
+ "loss": 1.4558,
+ "theoretical_loss": 3.763187639240161,
+ "tokens_seen": 788070400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006151093279839518,
+ "loss": 1.4065,
+ "theoretical_loss": 3.7631567692613497,
+ "tokens_seen": 788135936
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006150932798395186,
+ "loss": 1.4183,
+ "theoretical_loss": 3.763125902568042,
+ "tokens_seen": 788201472
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006150772316950852,
+ "loss": 1.5066,
+ "theoretical_loss": 3.763095039159616,
+ "tokens_seen": 788267008
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000615061183550652,
+ "loss": 1.4107,
+ "theoretical_loss": 3.7630641790354478,
+ "tokens_seen": 788332544
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006150451354062186,
+ "loss": 1.3547,
+ "theoretical_loss": 3.763033322194916,
+ "tokens_seen": 788398080
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006150290872617854,
+ "loss": 1.4287,
+ "theoretical_loss": 3.763002468637398,
+ "tokens_seen": 788463616
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006150130391173521,
+ "loss": 1.3749,
+ "theoretical_loss": 3.762971618362271,
+ "tokens_seen": 788529152
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149969909729188,
+ "loss": 1.3979,
+ "theoretical_loss": 3.762940771368914,
+ "tokens_seen": 788594688
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149809428284855,
+ "loss": 1.4632,
+ "theoretical_loss": 3.762909927656705,
+ "tokens_seen": 788660224
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149648946840522,
+ "loss": 1.4155,
+ "theoretical_loss": 3.7628790872250217,
+ "tokens_seen": 788725760
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149488465396189,
+ "loss": 1.3951,
+ "theoretical_loss": 3.762848250073244,
+ "tokens_seen": 788791296
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149327983951856,
+ "loss": 1.2936,
+ "theoretical_loss": 3.7628174162007486,
+ "tokens_seen": 788856832
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006149167502507523,
+ "loss": 1.2792,
+ "theoretical_loss": 3.762786585606916,
+ "tokens_seen": 788922368
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614900702106319,
+ "loss": 1.3986,
+ "theoretical_loss": 3.7627557582911253,
+ "tokens_seen": 788987904
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148846539618857,
+ "loss": 1.3391,
+ "theoretical_loss": 3.7627249342527547,
+ "tokens_seen": 789053440
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148686058174524,
+ "loss": 1.4647,
+ "theoretical_loss": 3.762694113491184,
+ "tokens_seen": 789118976
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148525576730191,
+ "loss": 1.4506,
+ "theoretical_loss": 3.762663296005793,
+ "tokens_seen": 789184512
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148365095285858,
+ "loss": 1.5645,
+ "theoretical_loss": 3.7626324817959613,
+ "tokens_seen": 789250048
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148204613841525,
+ "loss": 1.399,
+ "theoretical_loss": 3.762601670861069,
+ "tokens_seen": 789315584
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006148044132397192,
+ "loss": 1.3029,
+ "theoretical_loss": 3.7625708632004953,
+ "tokens_seen": 789381120
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006147883650952859,
+ "loss": 1.3806,
+ "theoretical_loss": 3.762540058813622,
+ "tokens_seen": 789446656
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006147723169508526,
+ "loss": 1.3644,
+ "theoretical_loss": 3.762509257699828,
+ "tokens_seen": 789512192
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006147562688064193,
+ "loss": 1.3883,
+ "theoretical_loss": 3.7624784598584946,
+ "tokens_seen": 789577728
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614740220661986,
+ "loss": 1.5156,
+ "theoretical_loss": 3.7624476652890024,
+ "tokens_seen": 789643264
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 268814,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2551066875457764,
+ "objective/train/theoretical_loss": 3.762416873990732,
+ "objective/train/tokens_used": 810168800,
+ "theoretical_loss": 3.762416873990732,
+ "tokens_seen": 789708800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006147241725175527,
+ "loss": 1.3459,
+ "theoretical_loss": 3.762416873990732,
+ "tokens_seen": 789708800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006147081243731194,
+ "loss": 1.4716,
+ "theoretical_loss": 3.7623860859630653,
+ "tokens_seen": 789774336
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146920762286861,
+ "loss": 1.5881,
+ "theoretical_loss": 3.762355301205383,
+ "tokens_seen": 789839872
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146760280842528,
+ "loss": 1.4598,
+ "theoretical_loss": 3.7623245197170663,
+ "tokens_seen": 789905408
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146599799398195,
+ "loss": 1.2829,
+ "theoretical_loss": 3.7622937414974977,
+ "tokens_seen": 789970944
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146439317953862,
+ "loss": 1.3983,
+ "theoretical_loss": 3.762262966546058,
+ "tokens_seen": 790036480
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146278836509529,
+ "loss": 1.4769,
+ "theoretical_loss": 3.762232194862129,
+ "tokens_seen": 790102016
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006146118355065196,
+ "loss": 1.3794,
+ "theoretical_loss": 3.7622014264450936,
+ "tokens_seen": 790167552
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006145957873620863,
+ "loss": 1.3926,
+ "theoretical_loss": 3.7621706612943333,
+ "tokens_seen": 790233088
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614579739217653,
+ "loss": 1.4794,
+ "theoretical_loss": 3.7621398994092314,
+ "tokens_seen": 790298624
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006145636910732197,
+ "loss": 1.5054,
+ "theoretical_loss": 3.76210914078917,
+ "tokens_seen": 790364160
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006145476429287864,
+ "loss": 1.4222,
+ "theoretical_loss": 3.762078385433531,
+ "tokens_seen": 790429696
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006145315947843531,
+ "loss": 1.3232,
+ "theoretical_loss": 3.7620476333416994,
+ "tokens_seen": 790495232
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006145155466399198,
+ "loss": 1.4588,
+ "theoretical_loss": 3.7620168845130566,
+ "tokens_seen": 790560768
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006144994984954865,
+ "loss": 1.3283,
+ "theoretical_loss": 3.761986138946986,
+ "tokens_seen": 790626304
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006144834503510532,
+ "loss": 1.3703,
+ "theoretical_loss": 3.761955396642872,
+ "tokens_seen": 790691840
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061446740220662,
+ "loss": 1.3067,
+ "theoretical_loss": 3.761924657600097,
+ "tokens_seen": 790757376
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006144513540621866,
+ "loss": 1.2759,
+ "theoretical_loss": 3.7618939218180456,
+ "tokens_seen": 790822912
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006144353059177534,
+ "loss": 1.4833,
+ "theoretical_loss": 3.7618631892961014,
+ "tokens_seen": 790888448
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061441925777332,
+ "loss": 1.4231,
+ "theoretical_loss": 3.7618324600336486,
+ "tokens_seen": 790953984
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006144032096288868,
+ "loss": 1.3073,
+ "theoretical_loss": 3.7618017340300716,
+ "tokens_seen": 791019520
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006143871614844534,
+ "loss": 1.3236,
+ "theoretical_loss": 3.761771011284755,
+ "tokens_seen": 791085056
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006143711133400202,
+ "loss": 1.415,
+ "theoretical_loss": 3.761740291797082,
+ "tokens_seen": 791150592
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006143550651955868,
+ "loss": 1.3706,
+ "theoretical_loss": 3.7617095755664396,
+ "tokens_seen": 791216128
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006143390170511536,
+ "loss": 1.4312,
+ "theoretical_loss": 3.761678862592211,
+ "tokens_seen": 791281664
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006143229689067202,
+ "loss": 1.4508,
+ "theoretical_loss": 3.761648152873782,
+ "tokens_seen": 791347200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614306920762287,
+ "loss": 1.3756,
+ "theoretical_loss": 3.761617446410538,
+ "tokens_seen": 791412736
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006142908726178536,
+ "loss": 1.4036,
+ "theoretical_loss": 3.761586743201864,
+ "tokens_seen": 791478272
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006142748244734204,
+ "loss": 1.3711,
+ "theoretical_loss": 3.761556043247146,
+ "tokens_seen": 791543808
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614258776328987,
+ "loss": 1.4405,
+ "theoretical_loss": 3.7615253465457696,
+ "tokens_seen": 791609344
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006142427281845538,
+ "loss": 1.4056,
+ "theoretical_loss": 3.7614946530971207,
+ "tokens_seen": 791674880
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006142266800401204,
+ "loss": 1.4189,
+ "theoretical_loss": 3.7614639629005855,
+ "tokens_seen": 791740416
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006142106318956872,
+ "loss": 1.4482,
+ "theoretical_loss": 3.76143327595555,
+ "tokens_seen": 791805952
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141945837512538,
+ "loss": 1.4486,
+ "theoretical_loss": 3.761402592261401,
+ "tokens_seen": 791871488
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141785356068205,
+ "loss": 1.5224,
+ "theoretical_loss": 3.761371911817525,
+ "tokens_seen": 791937024
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141624874623872,
+ "loss": 1.3956,
+ "theoretical_loss": 3.761341234623309,
+ "tokens_seen": 792002560
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141464393179539,
+ "loss": 1.3544,
+ "theoretical_loss": 3.7613105606781394,
+ "tokens_seen": 792068096
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141303911735206,
+ "loss": 1.4256,
+ "theoretical_loss": 3.7612798899814037,
+ "tokens_seen": 792133632
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006141143430290873,
+ "loss": 1.3331,
+ "theoretical_loss": 3.7612492225324887,
+ "tokens_seen": 792199168
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000614098294884654,
+ "loss": 1.4264,
+ "theoretical_loss": 3.761218558330783,
+ "tokens_seen": 792264704
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140822467402207,
+ "loss": 1.4669,
+ "theoretical_loss": 3.761187897375673,
+ "tokens_seen": 792330240
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140661985957874,
+ "loss": 1.468,
+ "theoretical_loss": 3.761157239666547,
+ "tokens_seen": 792395776
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140501504513541,
+ "loss": 1.4272,
+ "theoretical_loss": 3.761126585202792,
+ "tokens_seen": 792461312
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140341023069207,
+ "loss": 1.4201,
+ "theoretical_loss": 3.761095933983798,
+ "tokens_seen": 792526848
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140180541624875,
+ "loss": 1.4428,
+ "theoretical_loss": 3.7610652860089515,
+ "tokens_seen": 792592384
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006140020060180541,
+ "loss": 1.3483,
+ "theoretical_loss": 3.761034641277642,
+ "tokens_seen": 792657920
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139859578736209,
+ "loss": 1.4491,
+ "theoretical_loss": 3.7610039997892577,
+ "tokens_seen": 792723456
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139699097291875,
+ "loss": 1.4242,
+ "theoretical_loss": 3.7609733615431873,
+ "tokens_seen": 792788992
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139538615847543,
+ "loss": 1.3877,
+ "theoretical_loss": 3.76094272653882,
+ "tokens_seen": 792854528
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139378134403209,
+ "loss": 1.3439,
+ "theoretical_loss": 3.760912094775544,
+ "tokens_seen": 792920064
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 269715,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.299852728843689,
+ "objective/train/theoretical_loss": 3.7608814662527505,
+ "objective/train/tokens_used": 813445600,
+ "theoretical_loss": 3.7608814662527505,
+ "tokens_seen": 792985600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139217652958877,
+ "loss": 1.391,
+ "theoretical_loss": 3.7608814662527505,
+ "tokens_seen": 792985600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006139057171514543,
+ "loss": 1.3127,
+ "theoretical_loss": 3.7608508409698267,
+ "tokens_seen": 793051136
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138896690070211,
+ "loss": 1.3984,
+ "theoretical_loss": 3.7608202189261637,
+ "tokens_seen": 793116672
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138736208625877,
+ "loss": 1.3009,
+ "theoretical_loss": 3.760789600121151,
+ "tokens_seen": 793182208
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138575727181545,
+ "loss": 1.4481,
+ "theoretical_loss": 3.760758984554178,
+ "tokens_seen": 793247744
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138415245737211,
+ "loss": 1.4492,
+ "theoretical_loss": 3.7607283722246345,
+ "tokens_seen": 793313280
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138254764292879,
+ "loss": 1.4088,
+ "theoretical_loss": 3.760697763131912,
+ "tokens_seen": 793378816
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006138094282848546,
+ "loss": 1.4589,
+ "theoretical_loss": 3.7606671572754005,
+ "tokens_seen": 793444352
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006137933801404213,
+ "loss": 1.3007,
+ "theoretical_loss": 3.7606365546544898,
+ "tokens_seen": 793509888
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000613777331995988,
+ "loss": 1.3536,
+ "theoretical_loss": 3.7606059552685713,
+ "tokens_seen": 793575424
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006137612838515547,
+ "loss": 1.5259,
+ "theoretical_loss": 3.760575359117036,
+ "tokens_seen": 793640960
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006137452357071214,
+ "loss": 1.3579,
+ "theoretical_loss": 3.760544766199275,
+ "tokens_seen": 793706496
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006137291875626881,
+ "loss": 1.3601,
+ "theoretical_loss": 3.760514176514679,
+ "tokens_seen": 793772032
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006137131394182548,
+ "loss": 1.3715,
+ "theoretical_loss": 3.7604835900626394,
+ "tokens_seen": 793837568
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136970912738215,
+ "loss": 1.3638,
+ "theoretical_loss": 3.760453006842549,
+ "tokens_seen": 793903104
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136810431293882,
+ "loss": 1.5297,
+ "theoretical_loss": 3.7604224268537982,
+ "tokens_seen": 793968640
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136649949849549,
+ "loss": 1.4525,
+ "theoretical_loss": 3.7603918500957794,
+ "tokens_seen": 794034176
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136489468405216,
+ "loss": 1.4023,
+ "theoretical_loss": 3.7603612765678847,
+ "tokens_seen": 794099712
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136328986960883,
+ "loss": 1.4335,
+ "theoretical_loss": 3.7603307062695066,
+ "tokens_seen": 794165248
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000613616850551655,
+ "loss": 1.4526,
+ "theoretical_loss": 3.7603001392000364,
+ "tokens_seen": 794230784
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006136008024072217,
+ "loss": 1.3654,
+ "theoretical_loss": 3.760269575358868,
+ "tokens_seen": 794296320
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135847542627884,
+ "loss": 1.3988,
+ "theoretical_loss": 3.7602390147453937,
+ "tokens_seen": 794361856
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135687061183551,
+ "loss": 1.4066,
+ "theoretical_loss": 3.760208457359006,
+ "tokens_seen": 794427392
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135526579739218,
+ "loss": 1.3623,
+ "theoretical_loss": 3.7601779031990983,
+ "tokens_seen": 794492928
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135366098294885,
+ "loss": 1.4173,
+ "theoretical_loss": 3.7601473522650637,
+ "tokens_seen": 794558464
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135205616850552,
+ "loss": 1.3934,
+ "theoretical_loss": 3.7601168045562954,
+ "tokens_seen": 794624000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006135045135406219,
+ "loss": 1.3386,
+ "theoretical_loss": 3.760086260072188,
+ "tokens_seen": 794689536
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006134884653961886,
+ "loss": 1.5199,
+ "theoretical_loss": 3.7600557188121337,
+ "tokens_seen": 794755072
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006134724172517553,
+ "loss": 1.4429,
+ "theoretical_loss": 3.7600251807755276,
+ "tokens_seen": 794820608
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000613456369107322,
+ "loss": 1.4271,
+ "theoretical_loss": 3.759994645961762,
+ "tokens_seen": 794886144
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006134403209628887,
+ "loss": 1.4422,
+ "theoretical_loss": 3.759964114370234,
+ "tokens_seen": 794951680
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006134242728184554,
+ "loss": 1.3336,
+ "theoretical_loss": 3.759933586000335,
+ "tokens_seen": 795017216
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006134082246740221,
+ "loss": 1.4427,
+ "theoretical_loss": 3.759903060851461,
+ "tokens_seen": 795082752
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133921765295888,
+ "loss": 1.3202,
+ "theoretical_loss": 3.759872538923007,
+ "tokens_seen": 795148288
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133761283851555,
+ "loss": 1.3186,
+ "theoretical_loss": 3.759842020214367,
+ "tokens_seen": 795213824
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133600802407222,
+ "loss": 1.332,
+ "theoretical_loss": 3.7598115047249365,
+ "tokens_seen": 795279360
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133440320962889,
+ "loss": 1.3651,
+ "theoretical_loss": 3.759780992454111,
+ "tokens_seen": 795344896
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133279839518556,
+ "loss": 1.4328,
+ "theoretical_loss": 3.7597504834012847,
+ "tokens_seen": 795410432
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006133119358074223,
+ "loss": 1.423,
+ "theoretical_loss": 3.7597199775658545,
+ "tokens_seen": 795475968
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000613295887662989,
+ "loss": 1.3496,
+ "theoretical_loss": 3.7596894749472147,
+ "tokens_seen": 795541504
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006132798395185557,
+ "loss": 1.4071,
+ "theoretical_loss": 3.7596589755447622,
+ "tokens_seen": 795607040
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006132637913741224,
+ "loss": 1.3789,
+ "theoretical_loss": 3.759628479357893,
+ "tokens_seen": 795672576
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006132477432296891,
+ "loss": 1.4153,
+ "theoretical_loss": 3.7595979863860025,
+ "tokens_seen": 795738112
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006132316950852558,
+ "loss": 1.4625,
+ "theoretical_loss": 3.759567496628488,
+ "tokens_seen": 795803648
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006132156469408225,
+ "loss": 1.4996,
+ "theoretical_loss": 3.759537010084745,
+ "tokens_seen": 795869184
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131995987963892,
+ "loss": 1.4998,
+ "theoretical_loss": 3.759506526754171,
+ "tokens_seen": 795934720
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131835506519559,
+ "loss": 1.3282,
+ "theoretical_loss": 3.759476046636162,
+ "tokens_seen": 796000256
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131675025075226,
+ "loss": 1.3505,
+ "theoretical_loss": 3.759445569730115,
+ "tokens_seen": 796065792
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131514543630893,
+ "loss": 1.4525,
+ "theoretical_loss": 3.7594150960354282,
+ "tokens_seen": 796131328
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000613135406218656,
+ "loss": 1.4226,
+ "theoretical_loss": 3.7593846255514984,
+ "tokens_seen": 796196864
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 270870,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3770694732666016,
+ "objective/train/theoretical_loss": 3.7593541582777226,
+ "objective/train/tokens_used": 816722400,
+ "theoretical_loss": 3.7593541582777226,
+ "tokens_seen": 796262400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131193580742227,
+ "loss": 1.3837,
+ "theoretical_loss": 3.7593541582777226,
+ "tokens_seen": 796262400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006131033099297894,
+ "loss": 1.2927,
+ "theoretical_loss": 3.7593236942134993,
+ "tokens_seen": 796327936
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130872617853561,
+ "loss": 1.3862,
+ "theoretical_loss": 3.7592932333582247,
+ "tokens_seen": 796393472
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130712136409228,
+ "loss": 1.4526,
+ "theoretical_loss": 3.759262775711299,
+ "tokens_seen": 796459008
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130551654964895,
+ "loss": 1.2557,
+ "theoretical_loss": 3.759232321272118,
+ "tokens_seen": 796524544
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130391173520563,
+ "loss": 1.3715,
+ "theoretical_loss": 3.7592018700400818,
+ "tokens_seen": 796590080
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130230692076229,
+ "loss": 1.32,
+ "theoretical_loss": 3.759171422014588,
+ "tokens_seen": 796655616
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006130070210631897,
+ "loss": 1.4967,
+ "theoretical_loss": 3.7591409771950355,
+ "tokens_seen": 796721152
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006129909729187563,
+ "loss": 1.4658,
+ "theoretical_loss": 3.759110535580823,
+ "tokens_seen": 796786688
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612974924774323,
+ "loss": 1.3089,
+ "theoretical_loss": 3.759080097171349,
+ "tokens_seen": 796852224
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006129588766298897,
+ "loss": 1.4287,
+ "theoretical_loss": 3.7590496619660136,
+ "tokens_seen": 796917760
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006129428284854565,
+ "loss": 1.4344,
+ "theoretical_loss": 3.759019229964215,
+ "tokens_seen": 796983296
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006129267803410231,
+ "loss": 1.3833,
+ "theoretical_loss": 3.758988801165353,
+ "tokens_seen": 797048832
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006129107321965899,
+ "loss": 1.4713,
+ "theoretical_loss": 3.7589583755688274,
+ "tokens_seen": 797114368
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128946840521565,
+ "loss": 1.4555,
+ "theoretical_loss": 3.7589279531740374,
+ "tokens_seen": 797179904
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128786359077233,
+ "loss": 1.2965,
+ "theoretical_loss": 3.7588975339803836,
+ "tokens_seen": 797245440
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128625877632899,
+ "loss": 1.4055,
+ "theoretical_loss": 3.7588671179872657,
+ "tokens_seen": 797310976
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128465396188567,
+ "loss": 1.3899,
+ "theoretical_loss": 3.7588367051940836,
+ "tokens_seen": 797376512
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128304914744233,
+ "loss": 1.3753,
+ "theoretical_loss": 3.7588062956002384,
+ "tokens_seen": 797442048
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006128144433299901,
+ "loss": 1.314,
+ "theoretical_loss": 3.75877588920513,
+ "tokens_seen": 797507584
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127983951855567,
+ "loss": 1.4176,
+ "theoretical_loss": 3.7587454860081597,
+ "tokens_seen": 797573120
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127823470411234,
+ "loss": 1.5011,
+ "theoretical_loss": 3.7587150860087277,
+ "tokens_seen": 797638656
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127662988966901,
+ "loss": 1.4468,
+ "theoretical_loss": 3.7586846892062358,
+ "tokens_seen": 797704192
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127502507522568,
+ "loss": 1.3313,
+ "theoretical_loss": 3.758654295600085,
+ "tokens_seen": 797769728
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127342026078235,
+ "loss": 1.3622,
+ "theoretical_loss": 3.758623905189676,
+ "tokens_seen": 797835264
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127181544633902,
+ "loss": 1.5191,
+ "theoretical_loss": 3.758593517974411,
+ "tokens_seen": 797900800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006127021063189569,
+ "loss": 1.2989,
+ "theoretical_loss": 3.7585631339536913,
+ "tokens_seen": 797966336
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006126860581745236,
+ "loss": 1.3811,
+ "theoretical_loss": 3.7585327531269197,
+ "tokens_seen": 798031872
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006126700100300903,
+ "loss": 1.4147,
+ "theoretical_loss": 3.7585023754934963,
+ "tokens_seen": 798097408
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612653961885657,
+ "loss": 1.3622,
+ "theoretical_loss": 3.7584720010528248,
+ "tokens_seen": 798162944
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006126379137412236,
+ "loss": 1.4507,
+ "theoretical_loss": 3.7584416298043073,
+ "tokens_seen": 798228480
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006126218655967904,
+ "loss": 1.3998,
+ "theoretical_loss": 3.758411261747346,
+ "tokens_seen": 798294016
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006126058174523571,
+ "loss": 1.3518,
+ "theoretical_loss": 3.7583808968813437,
+ "tokens_seen": 798359552
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125897693079238,
+ "loss": 1.3929,
+ "theoretical_loss": 3.758350535205703,
+ "tokens_seen": 798425088
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125737211634905,
+ "loss": 1.3411,
+ "theoretical_loss": 3.7583201767198267,
+ "tokens_seen": 798490624
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125576730190572,
+ "loss": 1.3693,
+ "theoretical_loss": 3.7582898214231184,
+ "tokens_seen": 798556160
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125416248746239,
+ "loss": 1.3981,
+ "theoretical_loss": 3.7582594693149813,
+ "tokens_seen": 798621696
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125255767301906,
+ "loss": 1.3908,
+ "theoretical_loss": 3.7582291203948195,
+ "tokens_seen": 798687232
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006125095285857573,
+ "loss": 1.4559,
+ "theoretical_loss": 3.758198774662035,
+ "tokens_seen": 798752768
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612493480441324,
+ "loss": 1.3672,
+ "theoretical_loss": 3.7581684321160322,
+ "tokens_seen": 798818304
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006124774322968907,
+ "loss": 1.3338,
+ "theoretical_loss": 3.758138092756216,
+ "tokens_seen": 798883840
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006124613841524574,
+ "loss": 1.3337,
+ "theoretical_loss": 3.75810775658199,
+ "tokens_seen": 798949376
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006124453360080241,
+ "loss": 1.3231,
+ "theoretical_loss": 3.7580774235927574,
+ "tokens_seen": 799014912
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006124292878635908,
+ "loss": 1.3307,
+ "theoretical_loss": 3.7580470937879236,
+ "tokens_seen": 799080448
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006124132397191575,
+ "loss": 1.3038,
+ "theoretical_loss": 3.758016767166893,
+ "tokens_seen": 799145984
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123971915747242,
+ "loss": 1.3889,
+ "theoretical_loss": 3.757986443729071,
+ "tokens_seen": 799211520
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123811434302909,
+ "loss": 1.4174,
+ "theoretical_loss": 3.7579561234738614,
+ "tokens_seen": 799277056
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123650952858576,
+ "loss": 1.4158,
+ "theoretical_loss": 3.75792580640067,
+ "tokens_seen": 799342592
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123490471414243,
+ "loss": 1.4023,
+ "theoretical_loss": 3.7578954925089008,
+ "tokens_seen": 799408128
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612332998996991,
+ "loss": 1.318,
+ "theoretical_loss": 3.757865181797961,
+ "tokens_seen": 799473664
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 271751,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4218894243240356,
+ "objective/train/theoretical_loss": 3.757834874267255,
+ "objective/train/tokens_used": 819999200,
+ "theoretical_loss": 3.757834874267255,
+ "tokens_seen": 799539200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123169508525577,
+ "loss": 1.4056,
+ "theoretical_loss": 3.757834874267255,
+ "tokens_seen": 799539200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006123009027081244,
+ "loss": 1.3216,
+ "theoretical_loss": 3.757804569916188,
+ "tokens_seen": 799604736
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122848545636911,
+ "loss": 1.3733,
+ "theoretical_loss": 3.7577742687441678,
+ "tokens_seen": 799670272
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122688064192578,
+ "loss": 1.3942,
+ "theoretical_loss": 3.757743970750598,
+ "tokens_seen": 799735808
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122527582748245,
+ "loss": 1.3493,
+ "theoretical_loss": 3.7577136759348866,
+ "tokens_seen": 799801344
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122367101303912,
+ "loss": 1.4403,
+ "theoretical_loss": 3.757683384296439,
+ "tokens_seen": 799866880
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122206619859579,
+ "loss": 1.4301,
+ "theoretical_loss": 3.7576530958346623,
+ "tokens_seen": 799932416
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006122046138415246,
+ "loss": 1.4529,
+ "theoretical_loss": 3.7576228105489626,
+ "tokens_seen": 799997952
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006121885656970913,
+ "loss": 1.4253,
+ "theoretical_loss": 3.757592528438747,
+ "tokens_seen": 800063488
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612172517552658,
+ "loss": 1.4993,
+ "theoretical_loss": 3.7575622495034224,
+ "tokens_seen": 800129024
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006121564694082247,
+ "loss": 1.4272,
+ "theoretical_loss": 3.7575319737423953,
+ "tokens_seen": 800194560
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006121404212637914,
+ "loss": 1.3275,
+ "theoretical_loss": 3.757501701155074,
+ "tokens_seen": 800260096
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006121243731193581,
+ "loss": 1.4945,
+ "theoretical_loss": 3.757471431740866,
+ "tokens_seen": 800325632
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006121083249749248,
+ "loss": 1.4018,
+ "theoretical_loss": 3.757441165499178,
+ "tokens_seen": 800391168
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006120922768304915,
+ "loss": 1.5022,
+ "theoretical_loss": 3.7574109024294184,
+ "tokens_seen": 800456704
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006120762286860582,
+ "loss": 1.3525,
+ "theoretical_loss": 3.757380642530995,
+ "tokens_seen": 800522240
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006120601805416249,
+ "loss": 1.4784,
+ "theoretical_loss": 3.7573503858033157,
+ "tokens_seen": 800587776
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006120441323971916,
+ "loss": 1.5019,
+ "theoretical_loss": 3.7573201322457885,
+ "tokens_seen": 800653312
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006120280842527583,
+ "loss": 1.4038,
+ "theoretical_loss": 3.757289881857823,
+ "tokens_seen": 800718848
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000612012036108325,
+ "loss": 1.4068,
+ "theoretical_loss": 3.7572596346388263,
+ "tokens_seen": 800784384
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119959879638917,
+ "loss": 1.2788,
+ "theoretical_loss": 3.7572293905882077,
+ "tokens_seen": 800849920
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119799398194584,
+ "loss": 1.3467,
+ "theoretical_loss": 3.7571991497053765,
+ "tokens_seen": 800915456
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119638916750251,
+ "loss": 1.4457,
+ "theoretical_loss": 3.757168911989741,
+ "tokens_seen": 800980992
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119478435305918,
+ "loss": 1.3659,
+ "theoretical_loss": 3.7571386774407114,
+ "tokens_seen": 801046528
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119317953861585,
+ "loss": 1.3445,
+ "theoretical_loss": 3.757108446057696,
+ "tokens_seen": 801112064
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006119157472417252,
+ "loss": 1.3314,
+ "theoretical_loss": 3.7570782178401045,
+ "tokens_seen": 801177600
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118996990972919,
+ "loss": 1.4833,
+ "theoretical_loss": 3.757047992787347,
+ "tokens_seen": 801243136
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118836509528586,
+ "loss": 1.3912,
+ "theoretical_loss": 3.757017770898833,
+ "tokens_seen": 801308672
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118676028084253,
+ "loss": 1.3906,
+ "theoretical_loss": 3.756987552173973,
+ "tokens_seen": 801374208
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611851554663992,
+ "loss": 1.3971,
+ "theoretical_loss": 3.7569573366121762,
+ "tokens_seen": 801439744
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118355065195587,
+ "loss": 1.3192,
+ "theoretical_loss": 3.7569271242128535,
+ "tokens_seen": 801505280
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118194583751254,
+ "loss": 1.398,
+ "theoretical_loss": 3.7568969149754157,
+ "tokens_seen": 801570816
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006118034102306921,
+ "loss": 1.4638,
+ "theoretical_loss": 3.756866708899273,
+ "tokens_seen": 801636352
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006117873620862588,
+ "loss": 1.4404,
+ "theoretical_loss": 3.756836505983836,
+ "tokens_seen": 801701888
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006117713139418255,
+ "loss": 1.3786,
+ "theoretical_loss": 3.756806306228516,
+ "tokens_seen": 801767424
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006117552657973922,
+ "loss": 1.3963,
+ "theoretical_loss": 3.7567761096327237,
+ "tokens_seen": 801832960
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611739217652959,
+ "loss": 1.3476,
+ "theoretical_loss": 3.7567459161958707,
+ "tokens_seen": 801898496
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006117231695085256,
+ "loss": 1.4406,
+ "theoretical_loss": 3.7567157259173682,
+ "tokens_seen": 801964032
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006117071213640923,
+ "loss": 1.3238,
+ "theoretical_loss": 3.756685538796628,
+ "tokens_seen": 802029568
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611691073219659,
+ "loss": 1.337,
+ "theoretical_loss": 3.756655354833062,
+ "tokens_seen": 802095104
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006116750250752258,
+ "loss": 1.4085,
+ "theoretical_loss": 3.756625174026081,
+ "tokens_seen": 802160640
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006116589769307924,
+ "loss": 1.3654,
+ "theoretical_loss": 3.756594996375098,
+ "tokens_seen": 802226176
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006116429287863592,
+ "loss": 1.4613,
+ "theoretical_loss": 3.7565648218795253,
+ "tokens_seen": 802291712
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006116268806419258,
+ "loss": 1.3568,
+ "theoretical_loss": 3.756534650538775,
+ "tokens_seen": 802357248
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006116108324974926,
+ "loss": 1.348,
+ "theoretical_loss": 3.7565044823522595,
+ "tokens_seen": 802422784
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006115947843530592,
+ "loss": 1.363,
+ "theoretical_loss": 3.7564743173193915,
+ "tokens_seen": 802488320
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611578736208626,
+ "loss": 1.4044,
+ "theoretical_loss": 3.756444155439584,
+ "tokens_seen": 802553856
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006115626880641926,
+ "loss": 1.5253,
+ "theoretical_loss": 3.7564139967122494,
+ "tokens_seen": 802619392
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006115466399197594,
+ "loss": 1.3278,
+ "theoretical_loss": 3.7563838411368016,
+ "tokens_seen": 802684928
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611530591775326,
+ "loss": 1.4076,
+ "theoretical_loss": 3.756353688712654,
+ "tokens_seen": 802750464
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 272906,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2556822299957275,
+ "objective/train/theoretical_loss": 3.7563235394392187,
+ "objective/train/tokens_used": 823276000,
+ "theoretical_loss": 3.7563235394392187,
+ "tokens_seen": 802816000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006115145436308928,
+ "loss": 1.3715,
+ "theoretical_loss": 3.7563235394392187,
+ "tokens_seen": 802816000
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114984954864594,
+ "loss": 1.437,
+ "theoretical_loss": 3.756293393315911,
+ "tokens_seen": 802881536
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114824473420262,
+ "loss": 1.4397,
+ "theoretical_loss": 3.7562632503421436,
+ "tokens_seen": 802947072
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114663991975928,
+ "loss": 1.4542,
+ "theoretical_loss": 3.7562331105173308,
+ "tokens_seen": 803012608
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114503510531596,
+ "loss": 1.2796,
+ "theoretical_loss": 3.756202973840886,
+ "tokens_seen": 803078144
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114343029087262,
+ "loss": 1.4424,
+ "theoretical_loss": 3.7561728403122254,
+ "tokens_seen": 803143680
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611418254764293,
+ "loss": 1.3938,
+ "theoretical_loss": 3.7561427099307614,
+ "tokens_seen": 803209216
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006114022066198596,
+ "loss": 1.3613,
+ "theoretical_loss": 3.756112582695909,
+ "tokens_seen": 803274752
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006113861584754263,
+ "loss": 1.4798,
+ "theoretical_loss": 3.7560824586070836,
+ "tokens_seen": 803340288
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611370110330993,
+ "loss": 1.4933,
+ "theoretical_loss": 3.756052337663699,
+ "tokens_seen": 803405824
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006113540621865597,
+ "loss": 1.3328,
+ "theoretical_loss": 3.756022219865171,
+ "tokens_seen": 803471360
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006113380140421264,
+ "loss": 1.4218,
+ "theoretical_loss": 3.7559921052109155,
+ "tokens_seen": 803536896
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006113219658976931,
+ "loss": 1.4613,
+ "theoretical_loss": 3.7559619937003457,
+ "tokens_seen": 803602432
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006113059177532598,
+ "loss": 1.3645,
+ "theoretical_loss": 3.755931885332879,
+ "tokens_seen": 803667968
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006112898696088265,
+ "loss": 1.3942,
+ "theoretical_loss": 3.75590178010793,
+ "tokens_seen": 803733504
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006112738214643932,
+ "loss": 1.3609,
+ "theoretical_loss": 3.7558716780249153,
+ "tokens_seen": 803799040
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006112577733199599,
+ "loss": 1.3726,
+ "theoretical_loss": 3.75584157908325,
+ "tokens_seen": 803864576
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006112417251755266,
+ "loss": 1.3219,
+ "theoretical_loss": 3.755811483282351,
+ "tokens_seen": 803930112
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006112256770310933,
+ "loss": 1.3523,
+ "theoretical_loss": 3.7557813906216344,
+ "tokens_seen": 803995648
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.00061120962888666,
+ "loss": 1.4592,
+ "theoretical_loss": 3.755751301100516,
+ "tokens_seen": 804061184
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111935807422267,
+ "loss": 1.4307,
+ "theoretical_loss": 3.755721214718413,
+ "tokens_seen": 804126720
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111775325977934,
+ "loss": 1.3931,
+ "theoretical_loss": 3.755691131474742,
+ "tokens_seen": 804192256
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111614844533601,
+ "loss": 1.3656,
+ "theoretical_loss": 3.7556610513689197,
+ "tokens_seen": 804257792
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111454363089268,
+ "loss": 1.3319,
+ "theoretical_loss": 3.755630974400364,
+ "tokens_seen": 804323328
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111293881644935,
+ "loss": 1.3791,
+ "theoretical_loss": 3.7556009005684907,
+ "tokens_seen": 804388864
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006111133400200602,
+ "loss": 1.3199,
+ "theoretical_loss": 3.7555708298727186,
+ "tokens_seen": 804454400
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110972918756269,
+ "loss": 1.2016,
+ "theoretical_loss": 3.7555407623124637,
+ "tokens_seen": 804519936
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110812437311936,
+ "loss": 1.3712,
+ "theoretical_loss": 3.755510697887145,
+ "tokens_seen": 804585472
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110651955867603,
+ "loss": 1.3483,
+ "theoretical_loss": 3.75548063659618,
+ "tokens_seen": 804651008
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000611049147442327,
+ "loss": 1.5265,
+ "theoretical_loss": 3.755450578438986,
+ "tokens_seen": 804716544
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110330992978937,
+ "loss": 1.304,
+ "theoretical_loss": 3.7554205234149824,
+ "tokens_seen": 804782080
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110170511534604,
+ "loss": 1.3313,
+ "theoretical_loss": 3.755390471523586,
+ "tokens_seen": 804847616
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006110010030090271,
+ "loss": 1.3138,
+ "theoretical_loss": 3.755360422764217,
+ "tokens_seen": 804913152
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109849548645938,
+ "loss": 1.38,
+ "theoretical_loss": 3.755330377136292,
+ "tokens_seen": 804978688
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109689067201605,
+ "loss": 1.3346,
+ "theoretical_loss": 3.7553003346392315,
+ "tokens_seen": 805044224
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109528585757272,
+ "loss": 1.3258,
+ "theoretical_loss": 3.7552702952724535,
+ "tokens_seen": 805109760
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109368104312939,
+ "loss": 1.411,
+ "theoretical_loss": 3.755240259035377,
+ "tokens_seen": 805175296
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109207622868606,
+ "loss": 1.472,
+ "theoretical_loss": 3.755210225927422,
+ "tokens_seen": 805240832
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006109047141424273,
+ "loss": 1.4201,
+ "theoretical_loss": 3.7551801959480073,
+ "tokens_seen": 805306368
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610888665997994,
+ "loss": 1.3593,
+ "theoretical_loss": 3.7551501690965523,
+ "tokens_seen": 805371904
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006108726178535607,
+ "loss": 1.3621,
+ "theoretical_loss": 3.755120145372477,
+ "tokens_seen": 805437440
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006108565697091274,
+ "loss": 1.4013,
+ "theoretical_loss": 3.7550901247752013,
+ "tokens_seen": 805502976
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006108405215646941,
+ "loss": 1.3432,
+ "theoretical_loss": 3.755060107304145,
+ "tokens_seen": 805568512
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006108244734202608,
+ "loss": 1.4648,
+ "theoretical_loss": 3.7550300929587284,
+ "tokens_seen": 805634048
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006108084252758275,
+ "loss": 1.3331,
+ "theoretical_loss": 3.7550000817383715,
+ "tokens_seen": 805699584
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006107923771313942,
+ "loss": 1.4054,
+ "theoretical_loss": 3.754970073642496,
+ "tokens_seen": 805765120
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006107763289869609,
+ "loss": 1.4175,
+ "theoretical_loss": 3.7549400686705203,
+ "tokens_seen": 805830656
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006107602808425276,
+ "loss": 1.4496,
+ "theoretical_loss": 3.754910066821867,
+ "tokens_seen": 805896192
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006107442326980943,
+ "loss": 1.3061,
+ "theoretical_loss": 3.7548800680959564,
+ "tokens_seen": 805961728
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610728184553661,
+ "loss": 1.3899,
+ "theoretical_loss": 3.75485007249221,
+ "tokens_seen": 806027264
+ },
+ {
+ "epoch": 0.24,
+ "objective/train/docs_used": 274160,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.31361985206604,
+ "objective/train/theoretical_loss": 3.754820080010048,
+ "objective/train/tokens_used": 826552800,
+ "theoretical_loss": 3.754820080010048,
+ "tokens_seen": 806092800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006107121364092277,
+ "loss": 1.3446,
+ "theoretical_loss": 3.754820080010048,
+ "tokens_seen": 806092800
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106960882647944,
+ "loss": 1.4198,
+ "theoretical_loss": 3.7547900906488927,
+ "tokens_seen": 806158336
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106800401203611,
+ "loss": 1.3948,
+ "theoretical_loss": 3.7547601044081658,
+ "tokens_seen": 806223872
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106639919759278,
+ "loss": 1.3733,
+ "theoretical_loss": 3.7547301212872886,
+ "tokens_seen": 806289408
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106479438314945,
+ "loss": 1.41,
+ "theoretical_loss": 3.7547001412856824,
+ "tokens_seen": 806354944
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106318956870612,
+ "loss": 1.3092,
+ "theoretical_loss": 3.7546701644027705,
+ "tokens_seen": 806420480
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006106158475426279,
+ "loss": 1.3469,
+ "theoretical_loss": 3.754640190637974,
+ "tokens_seen": 806486016
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105997993981946,
+ "loss": 1.3592,
+ "theoretical_loss": 3.754610219990716,
+ "tokens_seen": 806551552
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105837512537613,
+ "loss": 1.426,
+ "theoretical_loss": 3.7545802524604177,
+ "tokens_seen": 806617088
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610567703109328,
+ "loss": 1.4712,
+ "theoretical_loss": 3.7545502880465036,
+ "tokens_seen": 806682624
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105516549648947,
+ "loss": 1.3731,
+ "theoretical_loss": 3.754520326748395,
+ "tokens_seen": 806748160
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105356068204614,
+ "loss": 1.3943,
+ "theoretical_loss": 3.7544903685655155,
+ "tokens_seen": 806813696
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105195586760281,
+ "loss": 1.2324,
+ "theoretical_loss": 3.754460413497288,
+ "tokens_seen": 806879232
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006105035105315948,
+ "loss": 1.2384,
+ "theoretical_loss": 3.7544304615431354,
+ "tokens_seen": 806944768
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006104874623871615,
+ "loss": 1.3791,
+ "theoretical_loss": 3.7544005127024818,
+ "tokens_seen": 807010304
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006104714142427282,
+ "loss": 1.3843,
+ "theoretical_loss": 3.7543705669747496,
+ "tokens_seen": 807075840
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006104553660982949,
+ "loss": 1.291,
+ "theoretical_loss": 3.754340624359364,
+ "tokens_seen": 807141376
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006104393179538616,
+ "loss": 1.3835,
+ "theoretical_loss": 3.7543106848557484,
+ "tokens_seen": 807206912
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006104232698094283,
+ "loss": 1.3209,
+ "theoretical_loss": 3.754280748463326,
+ "tokens_seen": 807272448
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610407221664995,
+ "loss": 1.378,
+ "theoretical_loss": 3.7542508151815213,
+ "tokens_seen": 807337984
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103911735205617,
+ "loss": 1.3738,
+ "theoretical_loss": 3.75422088500976,
+ "tokens_seen": 807403520
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103751253761284,
+ "loss": 1.4543,
+ "theoretical_loss": 3.7541909579474644,
+ "tokens_seen": 807469056
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103590772316951,
+ "loss": 1.4103,
+ "theoretical_loss": 3.7541610339940603,
+ "tokens_seen": 807534592
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103430290872618,
+ "loss": 1.4083,
+ "theoretical_loss": 3.754131113148972,
+ "tokens_seen": 807600128
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103269809428285,
+ "loss": 1.344,
+ "theoretical_loss": 3.7541011954116255,
+ "tokens_seen": 807665664
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006103109327983953,
+ "loss": 1.328,
+ "theoretical_loss": 3.754071280781445,
+ "tokens_seen": 807731200
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006102948846539619,
+ "loss": 1.4778,
+ "theoretical_loss": 3.7540413692578554,
+ "tokens_seen": 807796736
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006102788365095287,
+ "loss": 1.4331,
+ "theoretical_loss": 3.7540114608402826,
+ "tokens_seen": 807862272
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006102627883650953,
+ "loss": 1.3379,
+ "theoretical_loss": 3.753981555528152,
+ "tokens_seen": 807927808
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610246740220662,
+ "loss": 1.4679,
+ "theoretical_loss": 3.7539516533208896,
+ "tokens_seen": 807993344
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006102306920762287,
+ "loss": 1.3643,
+ "theoretical_loss": 3.753921754217921,
+ "tokens_seen": 808058880
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006102146439317955,
+ "loss": 1.4234,
+ "theoretical_loss": 3.7538918582186724,
+ "tokens_seen": 808124416
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006101985957873622,
+ "loss": 1.3992,
+ "theoretical_loss": 3.75386196532257,
+ "tokens_seen": 808189952
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006101825476429289,
+ "loss": 1.384,
+ "theoretical_loss": 3.753832075529039,
+ "tokens_seen": 808255488
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006101664994984956,
+ "loss": 1.289,
+ "theoretical_loss": 3.7538021888375077,
+ "tokens_seen": 808321024
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.0006101504513540623,
+ "loss": 1.4012,
+ "theoretical_loss": 3.7537723052474012,
+ "tokens_seen": 808386560
+ },
+ {
+ "epoch": 0.24,
+ "learning_rate": 0.000610134403209629,
+ "loss": 1.4271,
+ "theoretical_loss": 3.753742424758147,
+ "tokens_seen": 808452096
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006101183550651957,
+ "loss": 1.2788,
+ "theoretical_loss": 3.7537125473691715,
+ "tokens_seen": 808517632
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006101023069207623,
+ "loss": 1.3695,
+ "theoretical_loss": 3.7536826730799024,
+ "tokens_seen": 808583168
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100862587763291,
+ "loss": 1.416,
+ "theoretical_loss": 3.753652801889767,
+ "tokens_seen": 808648704
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100702106318957,
+ "loss": 1.5054,
+ "theoretical_loss": 3.7536229337981912,
+ "tokens_seen": 808714240
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100541624874625,
+ "loss": 1.401,
+ "theoretical_loss": 3.753593068804604,
+ "tokens_seen": 808779776
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100381143430291,
+ "loss": 1.3745,
+ "theoretical_loss": 3.7535632069084333,
+ "tokens_seen": 808845312
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100220661985959,
+ "loss": 1.3839,
+ "theoretical_loss": 3.753533348109106,
+ "tokens_seen": 808910848
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006100060180541625,
+ "loss": 1.4323,
+ "theoretical_loss": 3.7535034924060504,
+ "tokens_seen": 808976384
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006099899699097292,
+ "loss": 1.4034,
+ "theoretical_loss": 3.7534736397986945,
+ "tokens_seen": 809041920
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006099739217652959,
+ "loss": 1.4289,
+ "theoretical_loss": 3.753443790286467,
+ "tokens_seen": 809107456
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006099578736208626,
+ "loss": 1.3386,
+ "theoretical_loss": 3.7534139438687957,
+ "tokens_seen": 809172992
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006099418254764293,
+ "loss": 1.3465,
+ "theoretical_loss": 3.7533841005451096,
+ "tokens_seen": 809238528
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609925777331996,
+ "loss": 1.3349,
+ "theoretical_loss": 3.7533542603148375,
+ "tokens_seen": 809304064
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 275041,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4928539991378784,
+ "objective/train/theoretical_loss": 3.7533244231774083,
+ "objective/train/tokens_used": 829829600,
+ "theoretical_loss": 3.7533244231774083,
+ "tokens_seen": 809369600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006099097291875627,
+ "loss": 1.41,
+ "theoretical_loss": 3.7533244231774083,
+ "tokens_seen": 809369600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098936810431294,
+ "loss": 1.3372,
+ "theoretical_loss": 3.753294589132251,
+ "tokens_seen": 809435136
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098776328986961,
+ "loss": 1.3673,
+ "theoretical_loss": 3.753264758178794,
+ "tokens_seen": 809500672
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098615847542628,
+ "loss": 1.437,
+ "theoretical_loss": 3.7532349303164683,
+ "tokens_seen": 809566208
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098455366098295,
+ "loss": 1.3952,
+ "theoretical_loss": 3.7532051055447018,
+ "tokens_seen": 809631744
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098294884653962,
+ "loss": 1.3868,
+ "theoretical_loss": 3.753175283862925,
+ "tokens_seen": 809697280
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006098134403209629,
+ "loss": 1.4342,
+ "theoretical_loss": 3.753145465270568,
+ "tokens_seen": 809762816
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097973921765296,
+ "loss": 1.4018,
+ "theoretical_loss": 3.753115649767059,
+ "tokens_seen": 809828352
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097813440320963,
+ "loss": 1.4393,
+ "theoretical_loss": 3.75308583735183,
+ "tokens_seen": 809893888
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609765295887663,
+ "loss": 1.4589,
+ "theoretical_loss": 3.753056028024311,
+ "tokens_seen": 809959424
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097492477432297,
+ "loss": 1.4399,
+ "theoretical_loss": 3.753026221783932,
+ "tokens_seen": 810024960
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097331995987964,
+ "loss": 1.4689,
+ "theoretical_loss": 3.752996418630123,
+ "tokens_seen": 810090496
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097171514543631,
+ "loss": 1.3463,
+ "theoretical_loss": 3.7529666185623154,
+ "tokens_seen": 810156032
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006097011033099298,
+ "loss": 1.4034,
+ "theoretical_loss": 3.75293682157994,
+ "tokens_seen": 810221568
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006096850551654965,
+ "loss": 1.4199,
+ "theoretical_loss": 3.7529070276824283,
+ "tokens_seen": 810287104
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006096690070210632,
+ "loss": 1.433,
+ "theoretical_loss": 3.7528772368692103,
+ "tokens_seen": 810352640
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006096529588766299,
+ "loss": 1.4171,
+ "theoretical_loss": 3.7528474491397183,
+ "tokens_seen": 810418176
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006096369107321966,
+ "loss": 1.3973,
+ "theoretical_loss": 3.7528176644933833,
+ "tokens_seen": 810483712
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006096208625877633,
+ "loss": 1.3973,
+ "theoretical_loss": 3.7527878829296366,
+ "tokens_seen": 810549248
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.00060960481444333,
+ "loss": 1.3531,
+ "theoretical_loss": 3.752758104447911,
+ "tokens_seen": 810614784
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095887662988967,
+ "loss": 1.5146,
+ "theoretical_loss": 3.7527283290476374,
+ "tokens_seen": 810680320
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095727181544634,
+ "loss": 1.2868,
+ "theoretical_loss": 3.752698556728248,
+ "tokens_seen": 810745856
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095566700100301,
+ "loss": 1.2298,
+ "theoretical_loss": 3.7526687874891755,
+ "tokens_seen": 810811392
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095406218655968,
+ "loss": 1.4393,
+ "theoretical_loss": 3.752639021329852,
+ "tokens_seen": 810876928
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095245737211635,
+ "loss": 1.3986,
+ "theoretical_loss": 3.75260925824971,
+ "tokens_seen": 810942464
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006095085255767302,
+ "loss": 1.3188,
+ "theoretical_loss": 3.752579498248183,
+ "tokens_seen": 811008000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006094924774322969,
+ "loss": 1.3937,
+ "theoretical_loss": 3.752549741324702,
+ "tokens_seen": 811073536
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006094764292878636,
+ "loss": 1.4371,
+ "theoretical_loss": 3.752519987478702,
+ "tokens_seen": 811139072
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006094603811434303,
+ "loss": 1.391,
+ "theoretical_loss": 3.7524902367096145,
+ "tokens_seen": 811204608
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609444332998997,
+ "loss": 1.4319,
+ "theoretical_loss": 3.7524604890168733,
+ "tokens_seen": 811270144
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006094282848545637,
+ "loss": 1.3737,
+ "theoretical_loss": 3.752430744399912,
+ "tokens_seen": 811335680
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006094122367101304,
+ "loss": 1.4078,
+ "theoretical_loss": 3.7524010028581634,
+ "tokens_seen": 811401216
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093961885656971,
+ "loss": 1.3747,
+ "theoretical_loss": 3.752371264391063,
+ "tokens_seen": 811466752
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093801404212638,
+ "loss": 1.4404,
+ "theoretical_loss": 3.752341528998043,
+ "tokens_seen": 811532288
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093640922768305,
+ "loss": 1.3647,
+ "theoretical_loss": 3.7523117966785384,
+ "tokens_seen": 811597824
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093480441323972,
+ "loss": 1.4224,
+ "theoretical_loss": 3.752282067431983,
+ "tokens_seen": 811663360
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093319959879639,
+ "loss": 1.3438,
+ "theoretical_loss": 3.7522523412578104,
+ "tokens_seen": 811728896
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006093159478435306,
+ "loss": 1.3447,
+ "theoretical_loss": 3.7522226181554563,
+ "tokens_seen": 811794432
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092998996990973,
+ "loss": 1.2968,
+ "theoretical_loss": 3.7521928981243544,
+ "tokens_seen": 811859968
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609283851554664,
+ "loss": 1.3338,
+ "theoretical_loss": 3.7521631811639398,
+ "tokens_seen": 811925504
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092678034102307,
+ "loss": 1.3038,
+ "theoretical_loss": 3.752133467273648,
+ "tokens_seen": 811991040
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092517552657974,
+ "loss": 1.3154,
+ "theoretical_loss": 3.7521037564529127,
+ "tokens_seen": 812056576
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092357071213641,
+ "loss": 1.3167,
+ "theoretical_loss": 3.7520740487011706,
+ "tokens_seen": 812122112
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092196589769308,
+ "loss": 1.3733,
+ "theoretical_loss": 3.752044344017856,
+ "tokens_seen": 812187648
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006092036108324975,
+ "loss": 1.4768,
+ "theoretical_loss": 3.752014642402405,
+ "tokens_seen": 812253184
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006091875626880642,
+ "loss": 1.3319,
+ "theoretical_loss": 3.751984943854253,
+ "tokens_seen": 812318720
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006091715145436309,
+ "loss": 1.4028,
+ "theoretical_loss": 3.751955248372836,
+ "tokens_seen": 812384256
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006091554663991976,
+ "loss": 1.3679,
+ "theoretical_loss": 3.75192555595759,
+ "tokens_seen": 812449792
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006091394182547643,
+ "loss": 1.3209,
+ "theoretical_loss": 3.751895866607951,
+ "tokens_seen": 812515328
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609123370110331,
+ "loss": 1.3726,
+ "theoretical_loss": 3.7518661803233546,
+ "tokens_seen": 812580864
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 276340,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2950143814086914,
+ "objective/train/theoretical_loss": 3.7518364971032385,
+ "objective/train/tokens_used": 833106400,
+ "theoretical_loss": 3.7518364971032385,
+ "tokens_seen": 812646400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006091073219658977,
+ "loss": 1.3498,
+ "theoretical_loss": 3.7518364971032385,
+ "tokens_seen": 812646400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006090912738214644,
+ "loss": 1.3764,
+ "theoretical_loss": 3.751806816947038,
+ "tokens_seen": 812711936
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006090752256770311,
+ "loss": 1.4244,
+ "theoretical_loss": 3.751777139854191,
+ "tokens_seen": 812777472
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006090591775325978,
+ "loss": 1.4724,
+ "theoretical_loss": 3.7517474658241334,
+ "tokens_seen": 812843008
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006090431293881645,
+ "loss": 1.2368,
+ "theoretical_loss": 3.7517177948563027,
+ "tokens_seen": 812908544
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006090270812437312,
+ "loss": 1.4304,
+ "theoretical_loss": 3.7516881269501354,
+ "tokens_seen": 812974080
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000609011033099298,
+ "loss": 1.4232,
+ "theoretical_loss": 3.75165846210507,
+ "tokens_seen": 813039616
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089949849548647,
+ "loss": 1.4495,
+ "theoretical_loss": 3.7516288003205434,
+ "tokens_seen": 813105152
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089789368104313,
+ "loss": 1.3003,
+ "theoretical_loss": 3.7515991415959933,
+ "tokens_seen": 813170688
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089628886659981,
+ "loss": 1.4702,
+ "theoretical_loss": 3.7515694859308564,
+ "tokens_seen": 813236224
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089468405215648,
+ "loss": 1.5598,
+ "theoretical_loss": 3.7515398333245717,
+ "tokens_seen": 813301760
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089307923771315,
+ "loss": 1.4456,
+ "theoretical_loss": 3.751510183776577,
+ "tokens_seen": 813367296
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006089147442326982,
+ "loss": 1.5005,
+ "theoretical_loss": 3.7514805372863105,
+ "tokens_seen": 813432832
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088986960882649,
+ "loss": 1.3097,
+ "theoretical_loss": 3.7514508938532107,
+ "tokens_seen": 813498368
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088826479438316,
+ "loss": 1.4725,
+ "theoretical_loss": 3.7514212534767157,
+ "tokens_seen": 813563904
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088665997993983,
+ "loss": 1.4341,
+ "theoretical_loss": 3.7513916161562646,
+ "tokens_seen": 813629440
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608850551654965,
+ "loss": 1.4583,
+ "theoretical_loss": 3.7513619818912964,
+ "tokens_seen": 813694976
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088345035105317,
+ "loss": 1.3755,
+ "theoretical_loss": 3.751332350681249,
+ "tokens_seen": 813760512
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088184553660984,
+ "loss": 1.4159,
+ "theoretical_loss": 3.751302722525562,
+ "tokens_seen": 813826048
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006088024072216651,
+ "loss": 1.4529,
+ "theoretical_loss": 3.7512730974236748,
+ "tokens_seen": 813891584
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087863590772318,
+ "loss": 1.4448,
+ "theoretical_loss": 3.751243475375027,
+ "tokens_seen": 813957120
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087703109327985,
+ "loss": 1.4802,
+ "theoretical_loss": 3.7512138563790574,
+ "tokens_seen": 814022656
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087542627883652,
+ "loss": 1.3585,
+ "theoretical_loss": 3.7511842404352063,
+ "tokens_seen": 814088192
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087382146439319,
+ "loss": 1.4013,
+ "theoretical_loss": 3.7511546275429133,
+ "tokens_seen": 814153728
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087221664994986,
+ "loss": 1.3427,
+ "theoretical_loss": 3.7511250177016184,
+ "tokens_seen": 814219264
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006087061183550652,
+ "loss": 1.4272,
+ "theoretical_loss": 3.7510954109107617,
+ "tokens_seen": 814284800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608690070210632,
+ "loss": 1.3887,
+ "theoretical_loss": 3.751065807169784,
+ "tokens_seen": 814350336
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006086740220661986,
+ "loss": 1.6241,
+ "theoretical_loss": 3.7510362064781244,
+ "tokens_seen": 814415872
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006086579739217654,
+ "loss": 1.4205,
+ "theoretical_loss": 3.7510066088352247,
+ "tokens_seen": 814481408
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608641925777332,
+ "loss": 1.3721,
+ "theoretical_loss": 3.750977014240525,
+ "tokens_seen": 814546944
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006086258776328988,
+ "loss": 1.4522,
+ "theoretical_loss": 3.750947422693467,
+ "tokens_seen": 814612480
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006086098294884654,
+ "loss": 1.3763,
+ "theoretical_loss": 3.7509178341934906,
+ "tokens_seen": 814678016
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085937813440322,
+ "loss": 1.4083,
+ "theoretical_loss": 3.7508882487400372,
+ "tokens_seen": 814743552
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085777331995988,
+ "loss": 1.4799,
+ "theoretical_loss": 3.7508586663325483,
+ "tokens_seen": 814809088
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085616850551655,
+ "loss": 1.4582,
+ "theoretical_loss": 3.750829086970466,
+ "tokens_seen": 814874624
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085456369107322,
+ "loss": 1.3615,
+ "theoretical_loss": 3.750799510653231,
+ "tokens_seen": 814940160
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085295887662989,
+ "loss": 1.5135,
+ "theoretical_loss": 3.7507699373802854,
+ "tokens_seen": 815005696
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006085135406218656,
+ "loss": 1.4188,
+ "theoretical_loss": 3.750740367151071,
+ "tokens_seen": 815071232
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084974924774323,
+ "loss": 1.4552,
+ "theoretical_loss": 3.750710799965029,
+ "tokens_seen": 815136768
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608481444332999,
+ "loss": 1.4154,
+ "theoretical_loss": 3.7506812358216037,
+ "tokens_seen": 815202304
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084653961885657,
+ "loss": 1.4007,
+ "theoretical_loss": 3.7506516747202356,
+ "tokens_seen": 815267840
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084493480441324,
+ "loss": 1.3778,
+ "theoretical_loss": 3.7506221166603675,
+ "tokens_seen": 815333376
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084332998996991,
+ "loss": 1.5337,
+ "theoretical_loss": 3.750592561641443,
+ "tokens_seen": 815398912
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084172517552658,
+ "loss": 1.44,
+ "theoretical_loss": 3.750563009662904,
+ "tokens_seen": 815464448
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006084012036108325,
+ "loss": 1.3755,
+ "theoretical_loss": 3.750533460724193,
+ "tokens_seen": 815529984
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006083851554663992,
+ "loss": 1.3554,
+ "theoretical_loss": 3.750503914824754,
+ "tokens_seen": 815595520
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006083691073219659,
+ "loss": 1.3796,
+ "theoretical_loss": 3.75047437196403,
+ "tokens_seen": 815661056
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006083530591775326,
+ "loss": 1.4956,
+ "theoretical_loss": 3.750444832141464,
+ "tokens_seen": 815726592
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006083370110330993,
+ "loss": 1.4646,
+ "theoretical_loss": 3.7504152953564995,
+ "tokens_seen": 815792128
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608320962888666,
+ "loss": 1.2947,
+ "theoretical_loss": 3.750385761608581,
+ "tokens_seen": 815857664
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 277338,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.33406662940979,
+ "objective/train/theoretical_loss": 3.7503562308971516,
+ "objective/train/tokens_used": 836383200,
+ "theoretical_loss": 3.7503562308971516,
+ "tokens_seen": 815923200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006083049147442327,
+ "loss": 1.3607,
+ "theoretical_loss": 3.7503562308971516,
+ "tokens_seen": 815923200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082888665997994,
+ "loss": 1.5228,
+ "theoretical_loss": 3.7503267032216554,
+ "tokens_seen": 815988736
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082728184553661,
+ "loss": 1.2615,
+ "theoretical_loss": 3.750297178581536,
+ "tokens_seen": 816054272
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082567703109328,
+ "loss": 1.3789,
+ "theoretical_loss": 3.7502676569762383,
+ "tokens_seen": 816119808
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082407221664995,
+ "loss": 1.5057,
+ "theoretical_loss": 3.7502381384052064,
+ "tokens_seen": 816185344
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082246740220662,
+ "loss": 1.3762,
+ "theoretical_loss": 3.750208622867885,
+ "tokens_seen": 816250880
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006082086258776329,
+ "loss": 1.322,
+ "theoretical_loss": 3.7501791103637188,
+ "tokens_seen": 816316416
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006081925777331996,
+ "loss": 1.4761,
+ "theoretical_loss": 3.7501496008921524,
+ "tokens_seen": 816381952
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006081765295887663,
+ "loss": 1.3314,
+ "theoretical_loss": 3.750120094452631,
+ "tokens_seen": 816447488
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608160481444333,
+ "loss": 1.3851,
+ "theoretical_loss": 3.750090591044599,
+ "tokens_seen": 816513024
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006081444332998997,
+ "loss": 1.414,
+ "theoretical_loss": 3.750061090667503,
+ "tokens_seen": 816578560
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006081283851554664,
+ "loss": 1.3629,
+ "theoretical_loss": 3.750031593320787,
+ "tokens_seen": 816644096
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006081123370110331,
+ "loss": 1.3429,
+ "theoretical_loss": 3.750002099003898,
+ "tokens_seen": 816709632
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080962888665998,
+ "loss": 1.379,
+ "theoretical_loss": 3.7499726077162805,
+ "tokens_seen": 816775168
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080802407221665,
+ "loss": 1.3893,
+ "theoretical_loss": 3.749943119457381,
+ "tokens_seen": 816840704
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080641925777332,
+ "loss": 1.3895,
+ "theoretical_loss": 3.749913634226645,
+ "tokens_seen": 816906240
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080481444332999,
+ "loss": 1.3953,
+ "theoretical_loss": 3.7498841520235193,
+ "tokens_seen": 816971776
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080320962888666,
+ "loss": 1.3706,
+ "theoretical_loss": 3.7498546728474498,
+ "tokens_seen": 817037312
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006080160481444333,
+ "loss": 1.397,
+ "theoretical_loss": 3.7498251966978824,
+ "tokens_seen": 817102848
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000608,
+ "loss": 1.3084,
+ "theoretical_loss": 3.7497957235742647,
+ "tokens_seen": 817168384
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079839518555667,
+ "loss": 1.4022,
+ "theoretical_loss": 3.749766253476043,
+ "tokens_seen": 817233920
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079679037111334,
+ "loss": 1.3423,
+ "theoretical_loss": 3.7497367864026634,
+ "tokens_seen": 817299456
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079518555667001,
+ "loss": 1.4834,
+ "theoretical_loss": 3.749707322353575,
+ "tokens_seen": 817364992
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079358074222668,
+ "loss": 1.4279,
+ "theoretical_loss": 3.7496778613282222,
+ "tokens_seen": 817430528
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079197592778335,
+ "loss": 1.4755,
+ "theoretical_loss": 3.749648403326054,
+ "tokens_seen": 817496064
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006079037111334002,
+ "loss": 1.3799,
+ "theoretical_loss": 3.7496189483465177,
+ "tokens_seen": 817561600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006078876629889669,
+ "loss": 1.4539,
+ "theoretical_loss": 3.749589496389061,
+ "tokens_seen": 817627136
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006078716148445336,
+ "loss": 1.3667,
+ "theoretical_loss": 3.749560047453131,
+ "tokens_seen": 817692672
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006078555667001003,
+ "loss": 1.4008,
+ "theoretical_loss": 3.7495306015381766,
+ "tokens_seen": 817758208
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607839518555667,
+ "loss": 1.3677,
+ "theoretical_loss": 3.7495011586436444,
+ "tokens_seen": 817823744
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006078234704112337,
+ "loss": 1.4381,
+ "theoretical_loss": 3.7494717187689837,
+ "tokens_seen": 817889280
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006078074222668004,
+ "loss": 1.419,
+ "theoretical_loss": 3.7494422819136424,
+ "tokens_seen": 817954816
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006077913741223672,
+ "loss": 1.4178,
+ "theoretical_loss": 3.7494128480770694,
+ "tokens_seen": 818020352
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006077753259779338,
+ "loss": 1.3622,
+ "theoretical_loss": 3.749383417258713,
+ "tokens_seen": 818085888
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006077592778335006,
+ "loss": 1.4795,
+ "theoretical_loss": 3.7493539894580215,
+ "tokens_seen": 818151424
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006077432296890672,
+ "loss": 1.4322,
+ "theoretical_loss": 3.749324564674444,
+ "tokens_seen": 818216960
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607727181544634,
+ "loss": 1.3771,
+ "theoretical_loss": 3.74929514290743,
+ "tokens_seen": 818282496
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006077111334002006,
+ "loss": 1.3654,
+ "theoretical_loss": 3.749265724156429,
+ "tokens_seen": 818348032
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006076950852557674,
+ "loss": 1.3776,
+ "theoretical_loss": 3.7492363084208895,
+ "tokens_seen": 818413568
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607679037111334,
+ "loss": 1.4275,
+ "theoretical_loss": 3.7492068957002607,
+ "tokens_seen": 818479104
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006076629889669008,
+ "loss": 1.4517,
+ "theoretical_loss": 3.749177485993993,
+ "tokens_seen": 818544640
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006076469408224674,
+ "loss": 1.4664,
+ "theoretical_loss": 3.749148079301537,
+ "tokens_seen": 818610176
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006076308926780342,
+ "loss": 1.4065,
+ "theoretical_loss": 3.7491186756223405,
+ "tokens_seen": 818675712
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006076148445336008,
+ "loss": 1.3951,
+ "theoretical_loss": 3.749089274955855,
+ "tokens_seen": 818741248
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006075987963891676,
+ "loss": 1.5266,
+ "theoretical_loss": 3.74905987730153,
+ "tokens_seen": 818806784
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006075827482447342,
+ "loss": 1.317,
+ "theoretical_loss": 3.7490304826588163,
+ "tokens_seen": 818872320
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607566700100301,
+ "loss": 1.3463,
+ "theoretical_loss": 3.7490010910271647,
+ "tokens_seen": 818937856
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006075506519558677,
+ "loss": 1.4715,
+ "theoretical_loss": 3.748971702406025,
+ "tokens_seen": 819003392
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006075346038114344,
+ "loss": 1.3579,
+ "theoretical_loss": 3.7489423167948486,
+ "tokens_seen": 819068928
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607518555667001,
+ "loss": 1.444,
+ "theoretical_loss": 3.7489129341930862,
+ "tokens_seen": 819134464
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 278526,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3594393730163574,
+ "objective/train/theoretical_loss": 3.7488835546001886,
+ "objective/train/tokens_used": 839660000,
+ "theoretical_loss": 3.7488835546001886,
+ "tokens_seen": 819200000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006075025075225678,
+ "loss": 1.4002,
+ "theoretical_loss": 3.7488835546001886,
+ "tokens_seen": 819200000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006074864593781345,
+ "loss": 1.4299,
+ "theoretical_loss": 3.7488541780156077,
+ "tokens_seen": 819265536
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006074704112337012,
+ "loss": 1.4435,
+ "theoretical_loss": 3.7488248044387946,
+ "tokens_seen": 819331072
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006074543630892679,
+ "loss": 1.2144,
+ "theoretical_loss": 3.7487954338692,
+ "tokens_seen": 819396608
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006074383149448346,
+ "loss": 1.4401,
+ "theoretical_loss": 3.7487660663062767,
+ "tokens_seen": 819462144
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006074222668004013,
+ "loss": 1.4884,
+ "theoretical_loss": 3.7487367017494755,
+ "tokens_seen": 819527680
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607406218655968,
+ "loss": 1.3272,
+ "theoretical_loss": 3.7487073401982487,
+ "tokens_seen": 819593216
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073901705115347,
+ "loss": 1.4158,
+ "theoretical_loss": 3.7486779816520492,
+ "tokens_seen": 819658752
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073741223671014,
+ "loss": 1.3252,
+ "theoretical_loss": 3.7486486261103282,
+ "tokens_seen": 819724288
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073580742226681,
+ "loss": 1.4221,
+ "theoretical_loss": 3.748619273572538,
+ "tokens_seen": 819789824
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073420260782348,
+ "loss": 1.376,
+ "theoretical_loss": 3.748589924038132,
+ "tokens_seen": 819855360
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073259779338015,
+ "loss": 1.3883,
+ "theoretical_loss": 3.748560577506562,
+ "tokens_seen": 819920896
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006073099297893681,
+ "loss": 1.4445,
+ "theoretical_loss": 3.748531233977281,
+ "tokens_seen": 819986432
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072938816449349,
+ "loss": 1.4481,
+ "theoretical_loss": 3.7485018934497423,
+ "tokens_seen": 820051968
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072778335005015,
+ "loss": 1.3551,
+ "theoretical_loss": 3.7484725559233985,
+ "tokens_seen": 820117504
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072617853560683,
+ "loss": 1.4588,
+ "theoretical_loss": 3.7484432213977033,
+ "tokens_seen": 820183040
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072457372116349,
+ "loss": 1.4682,
+ "theoretical_loss": 3.7484138898721096,
+ "tokens_seen": 820248576
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072296890672017,
+ "loss": 1.4129,
+ "theoretical_loss": 3.748384561346071,
+ "tokens_seen": 820314112
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006072136409227683,
+ "loss": 1.4247,
+ "theoretical_loss": 3.7483552358190417,
+ "tokens_seen": 820379648
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071975927783351,
+ "loss": 1.3853,
+ "theoretical_loss": 3.7483259132904747,
+ "tokens_seen": 820445184
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071815446339017,
+ "loss": 1.3521,
+ "theoretical_loss": 3.7482965937598243,
+ "tokens_seen": 820510720
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071654964894684,
+ "loss": 1.4377,
+ "theoretical_loss": 3.748267277226545,
+ "tokens_seen": 820576256
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071494483450351,
+ "loss": 1.3544,
+ "theoretical_loss": 3.74823796369009,
+ "tokens_seen": 820641792
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071334002006018,
+ "loss": 1.3531,
+ "theoretical_loss": 3.7482086531499146,
+ "tokens_seen": 820707328
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071173520561685,
+ "loss": 1.3332,
+ "theoretical_loss": 3.748179345605473,
+ "tokens_seen": 820772864
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006071013039117352,
+ "loss": 1.4968,
+ "theoretical_loss": 3.74815004105622,
+ "tokens_seen": 820838400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006070852557673019,
+ "loss": 1.458,
+ "theoretical_loss": 3.7481207395016103,
+ "tokens_seen": 820903936
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006070692076228686,
+ "loss": 1.3335,
+ "theoretical_loss": 3.7480914409410984,
+ "tokens_seen": 820969472
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006070531594784353,
+ "loss": 1.3834,
+ "theoretical_loss": 3.7480621453741394,
+ "tokens_seen": 821035008
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000607037111334002,
+ "loss": 1.4099,
+ "theoretical_loss": 3.74803285280019,
+ "tokens_seen": 821100544
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006070210631895687,
+ "loss": 1.3673,
+ "theoretical_loss": 3.7480035632187034,
+ "tokens_seen": 821166080
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006070050150451354,
+ "loss": 1.3187,
+ "theoretical_loss": 3.7479742766291366,
+ "tokens_seen": 821231616
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069889669007021,
+ "loss": 1.5089,
+ "theoretical_loss": 3.747944993030945,
+ "tokens_seen": 821297152
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069729187562688,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7479157124235836,
+ "tokens_seen": 821362688
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069568706118355,
+ "loss": 1.4294,
+ "theoretical_loss": 3.7478864348065093,
+ "tokens_seen": 821428224
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069408224674022,
+ "loss": 1.3491,
+ "theoretical_loss": 3.747857160179178,
+ "tokens_seen": 821493760
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069247743229689,
+ "loss": 1.3862,
+ "theoretical_loss": 3.747827888541045,
+ "tokens_seen": 821559296
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006069087261785356,
+ "loss": 1.4502,
+ "theoretical_loss": 3.7477986198915683,
+ "tokens_seen": 821624832
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006068926780341023,
+ "loss": 1.3816,
+ "theoretical_loss": 3.7477693542302033,
+ "tokens_seen": 821690368
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000606876629889669,
+ "loss": 1.4689,
+ "theoretical_loss": 3.7477400915564063,
+ "tokens_seen": 821755904
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006068605817452357,
+ "loss": 1.4246,
+ "theoretical_loss": 3.747710831869635,
+ "tokens_seen": 821821440
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006068445336008024,
+ "loss": 1.3817,
+ "theoretical_loss": 3.747681575169346,
+ "tokens_seen": 821886976
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006068284854563691,
+ "loss": 1.2922,
+ "theoretical_loss": 3.747652321454997,
+ "tokens_seen": 821952512
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006068124373119358,
+ "loss": 1.374,
+ "theoretical_loss": 3.7476230707260436,
+ "tokens_seen": 822018048
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067963891675025,
+ "loss": 1.3978,
+ "theoretical_loss": 3.7475938229819445,
+ "tokens_seen": 822083584
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067803410230692,
+ "loss": 1.3212,
+ "theoretical_loss": 3.747564578222157,
+ "tokens_seen": 822149120
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067642928786359,
+ "loss": 1.395,
+ "theoretical_loss": 3.747535336446138,
+ "tokens_seen": 822214656
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067482447342026,
+ "loss": 1.4946,
+ "theoretical_loss": 3.7475060976533463,
+ "tokens_seen": 822280192
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067321965897693,
+ "loss": 1.4107,
+ "theoretical_loss": 3.7474768618432392,
+ "tokens_seen": 822345728
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000606716148445336,
+ "loss": 1.3553,
+ "theoretical_loss": 3.747447629015275,
+ "tokens_seen": 822411264
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 279519,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.31684410572052,
+ "objective/train/theoretical_loss": 3.747418399168912,
+ "objective/train/tokens_used": 842936800,
+ "theoretical_loss": 3.747418399168912,
+ "tokens_seen": 822476800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006067001003009027,
+ "loss": 1.357,
+ "theoretical_loss": 3.747418399168912,
+ "tokens_seen": 822476800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066840521564694,
+ "loss": 1.4719,
+ "theoretical_loss": 3.747389172303608,
+ "tokens_seen": 822542336
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066680040120361,
+ "loss": 1.3836,
+ "theoretical_loss": 3.7473599484188225,
+ "tokens_seen": 822607872
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066519558676028,
+ "loss": 1.4576,
+ "theoretical_loss": 3.7473307275140133,
+ "tokens_seen": 822673408
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066359077231695,
+ "loss": 1.3516,
+ "theoretical_loss": 3.747301509588639,
+ "tokens_seen": 822738944
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066198595787362,
+ "loss": 1.3676,
+ "theoretical_loss": 3.7472722946421593,
+ "tokens_seen": 822804480
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006066038114343029,
+ "loss": 1.3243,
+ "theoretical_loss": 3.747243082674033,
+ "tokens_seen": 822870016
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065877632898697,
+ "loss": 1.3562,
+ "theoretical_loss": 3.7472138736837186,
+ "tokens_seen": 822935552
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065717151454363,
+ "loss": 1.3552,
+ "theoretical_loss": 3.747184667670677,
+ "tokens_seen": 823001088
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065556670010031,
+ "loss": 1.4412,
+ "theoretical_loss": 3.747155464634366,
+ "tokens_seen": 823066624
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065396188565697,
+ "loss": 1.3834,
+ "theoretical_loss": 3.747126264574246,
+ "tokens_seen": 823132160
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065235707121365,
+ "loss": 1.398,
+ "theoretical_loss": 3.7470970674897766,
+ "tokens_seen": 823197696
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006065075225677031,
+ "loss": 1.4561,
+ "theoretical_loss": 3.7470678733804177,
+ "tokens_seen": 823263232
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064914744232699,
+ "loss": 1.4431,
+ "theoretical_loss": 3.7470386822456296,
+ "tokens_seen": 823328768
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064754262788365,
+ "loss": 1.4705,
+ "theoretical_loss": 3.7470094940848724,
+ "tokens_seen": 823394304
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064593781344033,
+ "loss": 1.3929,
+ "theoretical_loss": 3.7469803088976064,
+ "tokens_seen": 823459840
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064433299899699,
+ "loss": 1.3357,
+ "theoretical_loss": 3.7469511266832916,
+ "tokens_seen": 823525376
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064272818455367,
+ "loss": 1.437,
+ "theoretical_loss": 3.7469219474413893,
+ "tokens_seen": 823590912
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006064112337011033,
+ "loss": 1.4384,
+ "theoretical_loss": 3.74689277117136,
+ "tokens_seen": 823656448
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063951855566701,
+ "loss": 1.4605,
+ "theoretical_loss": 3.7468635978726645,
+ "tokens_seen": 823721984
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063791374122367,
+ "loss": 1.442,
+ "theoretical_loss": 3.746834427544764,
+ "tokens_seen": 823787520
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063630892678035,
+ "loss": 1.4305,
+ "theoretical_loss": 3.7468052601871196,
+ "tokens_seen": 823853056
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063470411233701,
+ "loss": 1.4194,
+ "theoretical_loss": 3.7467760957991922,
+ "tokens_seen": 823918592
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063309929789369,
+ "loss": 1.3391,
+ "theoretical_loss": 3.746746934380444,
+ "tokens_seen": 823984128
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006063149448345035,
+ "loss": 1.5232,
+ "theoretical_loss": 3.7467177759303363,
+ "tokens_seen": 824049664
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062988966900703,
+ "loss": 1.3503,
+ "theoretical_loss": 3.7466886204483307,
+ "tokens_seen": 824115200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000606282848545637,
+ "loss": 1.4009,
+ "theoretical_loss": 3.746659467933889,
+ "tokens_seen": 824180736
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062668004012037,
+ "loss": 1.4677,
+ "theoretical_loss": 3.7466303183864733,
+ "tokens_seen": 824246272
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062507522567703,
+ "loss": 1.4113,
+ "theoretical_loss": 3.746601171805546,
+ "tokens_seen": 824311808
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062347041123371,
+ "loss": 1.4142,
+ "theoretical_loss": 3.7465720281905694,
+ "tokens_seen": 824377344
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062186559679037,
+ "loss": 1.3534,
+ "theoretical_loss": 3.746542887541005,
+ "tokens_seen": 824442880
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006062026078234705,
+ "loss": 1.4531,
+ "theoretical_loss": 3.746513749856317,
+ "tokens_seen": 824508416
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006061865596790372,
+ "loss": 1.4338,
+ "theoretical_loss": 3.7464846151359668,
+ "tokens_seen": 824573952
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006061705115346039,
+ "loss": 1.3586,
+ "theoretical_loss": 3.7464554833794175,
+ "tokens_seen": 824639488
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006061544633901706,
+ "loss": 1.4736,
+ "theoretical_loss": 3.7464263545861325,
+ "tokens_seen": 824705024
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006061384152457373,
+ "loss": 1.4254,
+ "theoretical_loss": 3.7463972287555745,
+ "tokens_seen": 824770560
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000606122367101304,
+ "loss": 1.4723,
+ "theoretical_loss": 3.7463681058872074,
+ "tokens_seen": 824836096
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006061063189568707,
+ "loss": 1.3425,
+ "theoretical_loss": 3.746338985980494,
+ "tokens_seen": 824901632
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060902708124374,
+ "loss": 1.2868,
+ "theoretical_loss": 3.746309869034898,
+ "tokens_seen": 824967168
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060742226680041,
+ "loss": 1.3821,
+ "theoretical_loss": 3.746280755049883,
+ "tokens_seen": 825032704
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060581745235708,
+ "loss": 1.3976,
+ "theoretical_loss": 3.7462516440249134,
+ "tokens_seen": 825098240
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060421263791375,
+ "loss": 1.355,
+ "theoretical_loss": 3.7462225359594523,
+ "tokens_seen": 825163776
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060260782347042,
+ "loss": 1.3845,
+ "theoretical_loss": 3.7461934308529647,
+ "tokens_seen": 825229312
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006060100300902709,
+ "loss": 1.3716,
+ "theoretical_loss": 3.7461643287049142,
+ "tokens_seen": 825294848
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006059939819458376,
+ "loss": 1.368,
+ "theoretical_loss": 3.7461352295147656,
+ "tokens_seen": 825360384
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006059779338014043,
+ "loss": 1.4058,
+ "theoretical_loss": 3.746106133281983,
+ "tokens_seen": 825425920
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605961885656971,
+ "loss": 1.3171,
+ "theoretical_loss": 3.746077040006032,
+ "tokens_seen": 825491456
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006059458375125377,
+ "loss": 1.4918,
+ "theoretical_loss": 3.7460479496863757,
+ "tokens_seen": 825556992
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006059297893681044,
+ "loss": 1.2982,
+ "theoretical_loss": 3.746018862322481,
+ "tokens_seen": 825622528
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605913741223671,
+ "loss": 1.3822,
+ "theoretical_loss": 3.7459897779138114,
+ "tokens_seen": 825688064
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 280689,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4213380813598633,
+ "objective/train/theoretical_loss": 3.745960696459833,
+ "objective/train/tokens_used": 846213600,
+ "theoretical_loss": 3.745960696459833,
+ "tokens_seen": 825753600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058976930792378,
+ "loss": 1.4046,
+ "theoretical_loss": 3.745960696459833,
+ "tokens_seen": 825753600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058816449348044,
+ "loss": 1.4818,
+ "theoretical_loss": 3.7459316179600117,
+ "tokens_seen": 825819136
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058655967903712,
+ "loss": 1.3332,
+ "theoretical_loss": 3.7459025424138117,
+ "tokens_seen": 825884672
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058495486459378,
+ "loss": 1.4354,
+ "theoretical_loss": 3.7458734698206992,
+ "tokens_seen": 825950208
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058335005015046,
+ "loss": 1.4405,
+ "theoretical_loss": 3.74584440018014,
+ "tokens_seen": 826015744
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006058174523570712,
+ "loss": 1.3741,
+ "theoretical_loss": 3.7458153334916,
+ "tokens_seen": 826081280
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605801404212638,
+ "loss": 1.4119,
+ "theoretical_loss": 3.7457862697545456,
+ "tokens_seen": 826146816
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006057853560682046,
+ "loss": 1.4196,
+ "theoretical_loss": 3.7457572089684428,
+ "tokens_seen": 826212352
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006057693079237713,
+ "loss": 1.3461,
+ "theoretical_loss": 3.7457281511327576,
+ "tokens_seen": 826277888
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605753259779338,
+ "loss": 1.405,
+ "theoretical_loss": 3.7456990962469567,
+ "tokens_seen": 826343424
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006057372116349047,
+ "loss": 1.4363,
+ "theoretical_loss": 3.745670044310507,
+ "tokens_seen": 826408960
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006057211634904714,
+ "loss": 1.4599,
+ "theoretical_loss": 3.7456409953228746,
+ "tokens_seen": 826474496
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006057051153460381,
+ "loss": 1.4169,
+ "theoretical_loss": 3.745611949283527,
+ "tokens_seen": 826540032
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056890672016048,
+ "loss": 1.4954,
+ "theoretical_loss": 3.745582906191931,
+ "tokens_seen": 826605568
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056730190571715,
+ "loss": 1.4297,
+ "theoretical_loss": 3.7455538660475542,
+ "tokens_seen": 826671104
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056569709127382,
+ "loss": 1.3785,
+ "theoretical_loss": 3.7455248288498635,
+ "tokens_seen": 826736640
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056409227683049,
+ "loss": 1.4381,
+ "theoretical_loss": 3.745495794598326,
+ "tokens_seen": 826802176
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056248746238716,
+ "loss": 1.51,
+ "theoretical_loss": 3.74546676329241,
+ "tokens_seen": 826867712
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006056088264794383,
+ "loss": 1.4123,
+ "theoretical_loss": 3.7454377349315826,
+ "tokens_seen": 826933248
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605592778335005,
+ "loss": 1.4357,
+ "theoretical_loss": 3.7454087095153126,
+ "tokens_seen": 826998784
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006055767301905717,
+ "loss": 1.3916,
+ "theoretical_loss": 3.7453796870430667,
+ "tokens_seen": 827064320
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006055606820461384,
+ "loss": 1.3999,
+ "theoretical_loss": 3.7453506675143142,
+ "tokens_seen": 827129856
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006055446339017051,
+ "loss": 1.426,
+ "theoretical_loss": 3.7453216509285223,
+ "tokens_seen": 827195392
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006055285857572718,
+ "loss": 1.4631,
+ "theoretical_loss": 3.7452926372851607,
+ "tokens_seen": 827260928
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006055125376128385,
+ "loss": 1.5237,
+ "theoretical_loss": 3.7452636265836965,
+ "tokens_seen": 827326464
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054964894684052,
+ "loss": 1.4424,
+ "theoretical_loss": 3.7452346188236,
+ "tokens_seen": 827392000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054804413239719,
+ "loss": 1.3875,
+ "theoretical_loss": 3.745205614004339,
+ "tokens_seen": 827457536
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054643931795386,
+ "loss": 1.4605,
+ "theoretical_loss": 3.745176612125382,
+ "tokens_seen": 827523072
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054483450351053,
+ "loss": 1.3546,
+ "theoretical_loss": 3.7451476131861994,
+ "tokens_seen": 827588608
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605432296890672,
+ "loss": 1.303,
+ "theoretical_loss": 3.74511861718626,
+ "tokens_seen": 827654144
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054162487462387,
+ "loss": 1.2855,
+ "theoretical_loss": 3.7450896241250327,
+ "tokens_seen": 827719680
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006054002006018054,
+ "loss": 1.42,
+ "theoretical_loss": 3.7450606340019874,
+ "tokens_seen": 827785216
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006053841524573722,
+ "loss": 1.4237,
+ "theoretical_loss": 3.7450316468165936,
+ "tokens_seen": 827850752
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006053681043129388,
+ "loss": 1.4083,
+ "theoretical_loss": 3.7450026625683215,
+ "tokens_seen": 827916288
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006053520561685056,
+ "loss": 1.4614,
+ "theoretical_loss": 3.7449736812566403,
+ "tokens_seen": 827981824
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006053360080240722,
+ "loss": 1.3581,
+ "theoretical_loss": 3.744944702881021,
+ "tokens_seen": 828047360
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605319959879639,
+ "loss": 1.422,
+ "theoretical_loss": 3.744915727440933,
+ "tokens_seen": 828112896
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006053039117352056,
+ "loss": 1.5054,
+ "theoretical_loss": 3.7448867549358473,
+ "tokens_seen": 828178432
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006052878635907724,
+ "loss": 1.4287,
+ "theoretical_loss": 3.744857785365234,
+ "tokens_seen": 828243968
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605271815446339,
+ "loss": 1.3261,
+ "theoretical_loss": 3.744828818728563,
+ "tokens_seen": 828309504
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006052557673019058,
+ "loss": 1.4221,
+ "theoretical_loss": 3.744799855025307,
+ "tokens_seen": 828375040
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006052397191574724,
+ "loss": 1.286,
+ "theoretical_loss": 3.744770894254935,
+ "tokens_seen": 828440576
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006052236710130392,
+ "loss": 1.3079,
+ "theoretical_loss": 3.744741936416919,
+ "tokens_seen": 828506112
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006052076228686058,
+ "loss": 1.318,
+ "theoretical_loss": 3.74471298151073,
+ "tokens_seen": 828571648
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006051915747241726,
+ "loss": 1.4599,
+ "theoretical_loss": 3.744684029535839,
+ "tokens_seen": 828637184
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006051755265797392,
+ "loss": 1.3454,
+ "theoretical_loss": 3.744655080491718,
+ "tokens_seen": 828702720
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605159478435306,
+ "loss": 1.3969,
+ "theoretical_loss": 3.744626134377838,
+ "tokens_seen": 828768256
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006051434302908726,
+ "loss": 1.4725,
+ "theoretical_loss": 3.7445971911936713,
+ "tokens_seen": 828833792
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006051273821464394,
+ "loss": 1.4707,
+ "theoretical_loss": 3.7445682509386895,
+ "tokens_seen": 828899328
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000605111334002006,
+ "loss": 1.4693,
+ "theoretical_loss": 3.7445393136123646,
+ "tokens_seen": 828964864
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 281837,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4682958126068115,
+ "objective/train/theoretical_loss": 3.7445103792141685,
+ "objective/train/tokens_used": 849490400,
+ "theoretical_loss": 3.7445103792141685,
+ "tokens_seen": 829030400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050952858575728,
+ "loss": 1.4146,
+ "theoretical_loss": 3.7445103792141685,
+ "tokens_seen": 829030400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050792377131394,
+ "loss": 1.2954,
+ "theoretical_loss": 3.7444814477435737,
+ "tokens_seen": 829095936
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050631895687062,
+ "loss": 1.3835,
+ "theoretical_loss": 3.7444525192000526,
+ "tokens_seen": 829161472
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050471414242728,
+ "loss": 1.3096,
+ "theoretical_loss": 3.744423593583078,
+ "tokens_seen": 829227008
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050310932798396,
+ "loss": 1.518,
+ "theoretical_loss": 3.744394670892122,
+ "tokens_seen": 829292544
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006050150451354062,
+ "loss": 1.393,
+ "theoretical_loss": 3.744365751126658,
+ "tokens_seen": 829358080
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000604998996990973,
+ "loss": 1.3358,
+ "theoretical_loss": 3.744336834286158,
+ "tokens_seen": 829423616
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006049829488465396,
+ "loss": 1.4868,
+ "theoretical_loss": 3.7443079203700966,
+ "tokens_seen": 829489152
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006049669007021064,
+ "loss": 1.4051,
+ "theoretical_loss": 3.7442790093779452,
+ "tokens_seen": 829554688
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000604950852557673,
+ "loss": 1.3656,
+ "theoretical_loss": 3.744250101309179,
+ "tokens_seen": 829620224
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006049348044132398,
+ "loss": 1.4104,
+ "theoretical_loss": 3.74422119616327,
+ "tokens_seen": 829685760
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006049187562688064,
+ "loss": 1.3818,
+ "theoretical_loss": 3.744192293939693,
+ "tokens_seen": 829751296
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006049027081243732,
+ "loss": 1.4822,
+ "theoretical_loss": 3.744163394637921,
+ "tokens_seen": 829816832
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006048866599799398,
+ "loss": 1.412,
+ "theoretical_loss": 3.744134498257428,
+ "tokens_seen": 829882368
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006048706118355066,
+ "loss": 1.4419,
+ "theoretical_loss": 3.7441056047976886,
+ "tokens_seen": 829947904
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006048545636910732,
+ "loss": 1.3883,
+ "theoretical_loss": 3.744076714258177,
+ "tokens_seen": 830013440
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.00060483851554664,
+ "loss": 1.5272,
+ "theoretical_loss": 3.744047826638366,
+ "tokens_seen": 830078976
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006048224674022067,
+ "loss": 1.415,
+ "theoretical_loss": 3.7440189419377314,
+ "tokens_seen": 830144512
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006048064192577734,
+ "loss": 1.4296,
+ "theoretical_loss": 3.743990060155748,
+ "tokens_seen": 830210048
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.00060479037111334,
+ "loss": 1.4556,
+ "theoretical_loss": 3.7439611812918896,
+ "tokens_seen": 830275584
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006047743229689068,
+ "loss": 1.4024,
+ "theoretical_loss": 3.743932305345632,
+ "tokens_seen": 830341120
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006047582748244735,
+ "loss": 1.3821,
+ "theoretical_loss": 3.7439034323164493,
+ "tokens_seen": 830406656
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006047422266800402,
+ "loss": 1.3547,
+ "theoretical_loss": 3.7438745622038176,
+ "tokens_seen": 830472192
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006047261785356069,
+ "loss": 1.3311,
+ "theoretical_loss": 3.743845695007211,
+ "tokens_seen": 830537728
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006047101303911736,
+ "loss": 1.3625,
+ "theoretical_loss": 3.7438168307261055,
+ "tokens_seen": 830603264
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006046940822467403,
+ "loss": 1.3913,
+ "theoretical_loss": 3.7437879693599774,
+ "tokens_seen": 830668800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000604678034102307,
+ "loss": 1.4088,
+ "theoretical_loss": 3.743759110908301,
+ "tokens_seen": 830734336
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006046619859578737,
+ "loss": 1.4352,
+ "theoretical_loss": 3.743730255370553,
+ "tokens_seen": 830799872
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006046459378134404,
+ "loss": 1.4227,
+ "theoretical_loss": 3.743701402746209,
+ "tokens_seen": 830865408
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006046298896690071,
+ "loss": 1.3402,
+ "theoretical_loss": 3.743672553034745,
+ "tokens_seen": 830930944
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006046138415245738,
+ "loss": 1.3676,
+ "theoretical_loss": 3.743643706235638,
+ "tokens_seen": 830996480
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045977933801405,
+ "loss": 1.3668,
+ "theoretical_loss": 3.7436148623483634,
+ "tokens_seen": 831062016
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045817452357072,
+ "loss": 1.5535,
+ "theoretical_loss": 3.7435860213723977,
+ "tokens_seen": 831127552
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045656970912739,
+ "loss": 1.4146,
+ "theoretical_loss": 3.7435571833072183,
+ "tokens_seen": 831193088
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045496489468406,
+ "loss": 1.3039,
+ "theoretical_loss": 3.743528348152301,
+ "tokens_seen": 831258624
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045336008024073,
+ "loss": 1.3983,
+ "theoretical_loss": 3.743499515907124,
+ "tokens_seen": 831324160
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045175526579739,
+ "loss": 1.4228,
+ "theoretical_loss": 3.7434706865711633,
+ "tokens_seen": 831389696
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006045015045135407,
+ "loss": 1.3759,
+ "theoretical_loss": 3.7434418601438963,
+ "tokens_seen": 831455232
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044854563691073,
+ "loss": 1.4089,
+ "theoretical_loss": 3.7434130366248004,
+ "tokens_seen": 831520768
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044694082246741,
+ "loss": 1.3858,
+ "theoretical_loss": 3.743384216013353,
+ "tokens_seen": 831586304
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044533600802407,
+ "loss": 1.3688,
+ "theoretical_loss": 3.7433553983090313,
+ "tokens_seen": 831651840
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044373119358075,
+ "loss": 1.4525,
+ "theoretical_loss": 3.743326583511313,
+ "tokens_seen": 831717376
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044212637913741,
+ "loss": 1.4081,
+ "theoretical_loss": 3.7432977716196767,
+ "tokens_seen": 831782912
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006044052156469409,
+ "loss": 1.4298,
+ "theoretical_loss": 3.7432689626336,
+ "tokens_seen": 831848448
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006043891675025075,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7432401565525613,
+ "tokens_seen": 831913984
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006043731193580742,
+ "loss": 1.393,
+ "theoretical_loss": 3.743211353376038,
+ "tokens_seen": 831979520
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006043570712136409,
+ "loss": 1.3862,
+ "theoretical_loss": 3.743182553103509,
+ "tokens_seen": 832045056
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006043410230692076,
+ "loss": 1.4645,
+ "theoretical_loss": 3.7431537557344527,
+ "tokens_seen": 832110592
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006043249749247743,
+ "loss": 1.3851,
+ "theoretical_loss": 3.7431249612683484,
+ "tokens_seen": 832176128
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000604308926780341,
+ "loss": 1.4357,
+ "theoretical_loss": 3.743096169704674,
+ "tokens_seen": 832241664
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 282632,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4518851041793823,
+ "objective/train/theoretical_loss": 3.7430673810429083,
+ "objective/train/tokens_used": 852767200,
+ "theoretical_loss": 3.7430673810429083,
+ "tokens_seen": 832307200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042928786359077,
+ "loss": 1.4014,
+ "theoretical_loss": 3.7430673810429083,
+ "tokens_seen": 832307200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042768304914744,
+ "loss": 1.4634,
+ "theoretical_loss": 3.743038595282531,
+ "tokens_seen": 832372736
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042607823470411,
+ "loss": 1.4312,
+ "theoretical_loss": 3.7430098124230216,
+ "tokens_seen": 832438272
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042447342026078,
+ "loss": 1.4061,
+ "theoretical_loss": 3.7429810324638586,
+ "tokens_seen": 832503808
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042286860581745,
+ "loss": 1.4185,
+ "theoretical_loss": 3.7429522554045223,
+ "tokens_seen": 832569344
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006042126379137412,
+ "loss": 1.3821,
+ "theoretical_loss": 3.7429234812444907,
+ "tokens_seen": 832634880
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041965897693079,
+ "loss": 1.4198,
+ "theoretical_loss": 3.7428947099832452,
+ "tokens_seen": 832700416
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041805416248747,
+ "loss": 1.3011,
+ "theoretical_loss": 3.7428659416202654,
+ "tokens_seen": 832765952
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041644934804413,
+ "loss": 1.3807,
+ "theoretical_loss": 3.7428371761550303,
+ "tokens_seen": 832831488
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041484453360081,
+ "loss": 1.4291,
+ "theoretical_loss": 3.7428084135870208,
+ "tokens_seen": 832897024
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041323971915747,
+ "loss": 1.2841,
+ "theoretical_loss": 3.7427796539157177,
+ "tokens_seen": 832962560
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041163490471415,
+ "loss": 1.4131,
+ "theoretical_loss": 3.7427508971406,
+ "tokens_seen": 833028096
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006041003009027081,
+ "loss": 1.3391,
+ "theoretical_loss": 3.7427221432611493,
+ "tokens_seen": 833093632
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040842527582749,
+ "loss": 1.4184,
+ "theoretical_loss": 3.7426933922768457,
+ "tokens_seen": 833159168
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040682046138415,
+ "loss": 1.4064,
+ "theoretical_loss": 3.74266464418717,
+ "tokens_seen": 833224704
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040521564694083,
+ "loss": 1.3462,
+ "theoretical_loss": 3.7426358989916038,
+ "tokens_seen": 833290240
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040361083249749,
+ "loss": 1.4502,
+ "theoretical_loss": 3.742607156689628,
+ "tokens_seen": 833355776
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040200601805417,
+ "loss": 1.3462,
+ "theoretical_loss": 3.742578417280723,
+ "tokens_seen": 833421312
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006040040120361083,
+ "loss": 1.3551,
+ "theoretical_loss": 3.7425496807643706,
+ "tokens_seen": 833486848
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039879638916751,
+ "loss": 1.4537,
+ "theoretical_loss": 3.7425209471400525,
+ "tokens_seen": 833552384
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039719157472417,
+ "loss": 1.3427,
+ "theoretical_loss": 3.7424922164072503,
+ "tokens_seen": 833617920
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039558676028085,
+ "loss": 1.3983,
+ "theoretical_loss": 3.7424634885654453,
+ "tokens_seen": 833683456
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039398194583751,
+ "loss": 1.2434,
+ "theoretical_loss": 3.7424347636141198,
+ "tokens_seen": 833748992
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039237713139419,
+ "loss": 1.4972,
+ "theoretical_loss": 3.742406041552756,
+ "tokens_seen": 833814528
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006039077231695085,
+ "loss": 1.3897,
+ "theoretical_loss": 3.742377322380835,
+ "tokens_seen": 833880064
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038916750250753,
+ "loss": 1.3202,
+ "theoretical_loss": 3.74234860609784,
+ "tokens_seen": 833945600
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038756268806419,
+ "loss": 1.3765,
+ "theoretical_loss": 3.742319892703253,
+ "tokens_seen": 834011136
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038595787362087,
+ "loss": 1.4412,
+ "theoretical_loss": 3.7422911821965568,
+ "tokens_seen": 834076672
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038435305917753,
+ "loss": 1.3687,
+ "theoretical_loss": 3.7422624745772337,
+ "tokens_seen": 834142208
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038274824473421,
+ "loss": 1.3772,
+ "theoretical_loss": 3.742233769844767,
+ "tokens_seen": 834207744
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006038114343029087,
+ "loss": 1.3646,
+ "theoretical_loss": 3.74220506799864,
+ "tokens_seen": 834273280
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037953861584755,
+ "loss": 1.4583,
+ "theoretical_loss": 3.7421763690383347,
+ "tokens_seen": 834338816
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037793380140421,
+ "loss": 1.4014,
+ "theoretical_loss": 3.742147672963334,
+ "tokens_seen": 834404352
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037632898696089,
+ "loss": 1.4687,
+ "theoretical_loss": 3.742118979773123,
+ "tokens_seen": 834469888
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037472417251755,
+ "loss": 1.4343,
+ "theoretical_loss": 3.742090289467184,
+ "tokens_seen": 834535424
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037311935807423,
+ "loss": 1.3913,
+ "theoretical_loss": 3.7420616020450006,
+ "tokens_seen": 834600960
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006037151454363089,
+ "loss": 1.4343,
+ "theoretical_loss": 3.742032917506057,
+ "tokens_seen": 834666496
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036990972918757,
+ "loss": 1.4094,
+ "theoretical_loss": 3.742004235849836,
+ "tokens_seen": 834732032
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036830491474423,
+ "loss": 1.3836,
+ "theoretical_loss": 3.7419755570758233,
+ "tokens_seen": 834797568
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036670010030091,
+ "loss": 1.3085,
+ "theoretical_loss": 3.7419468811835017,
+ "tokens_seen": 834863104
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036509528585757,
+ "loss": 1.3428,
+ "theoretical_loss": 3.741918208172356,
+ "tokens_seen": 834928640
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036349047141425,
+ "loss": 1.461,
+ "theoretical_loss": 3.74188953804187,
+ "tokens_seen": 834994176
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036188565697091,
+ "loss": 1.3668,
+ "theoretical_loss": 3.7418608707915295,
+ "tokens_seen": 835059712
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006036028084252759,
+ "loss": 1.4113,
+ "theoretical_loss": 3.741832206420818,
+ "tokens_seen": 835125248
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006035867602808425,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7418035449292204,
+ "tokens_seen": 835190784
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006035707121364093,
+ "loss": 1.4598,
+ "theoretical_loss": 3.741774886316222,
+ "tokens_seen": 835256320
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000603554663991976,
+ "loss": 1.4639,
+ "theoretical_loss": 3.741746230581308,
+ "tokens_seen": 835321856
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006035386158475427,
+ "loss": 1.3883,
+ "theoretical_loss": 3.7417175777239633,
+ "tokens_seen": 835387392
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006035225677031093,
+ "loss": 1.4573,
+ "theoretical_loss": 3.741688927743673,
+ "tokens_seen": 835452928
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006035065195586761,
+ "loss": 1.5791,
+ "theoretical_loss": 3.7416602806399233,
+ "tokens_seen": 835518464
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 283853,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.289686679840088,
+ "objective/train/theoretical_loss": 3.7416316364121993,
+ "objective/train/tokens_used": 856044000,
+ "theoretical_loss": 3.7416316364121993,
+ "tokens_seen": 835584000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034904714142427,
+ "loss": 1.394,
+ "theoretical_loss": 3.7416316364121993,
+ "tokens_seen": 835584000
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034744232698095,
+ "loss": 1.412,
+ "theoretical_loss": 3.7416029950599867,
+ "tokens_seen": 835649536
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034583751253762,
+ "loss": 1.3217,
+ "theoretical_loss": 3.7415743565827713,
+ "tokens_seen": 835715072
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034423269809429,
+ "loss": 1.4093,
+ "theoretical_loss": 3.741545720980039,
+ "tokens_seen": 835780608
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034262788365096,
+ "loss": 1.3962,
+ "theoretical_loss": 3.7415170882512765,
+ "tokens_seen": 835846144
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006034102306920763,
+ "loss": 1.4114,
+ "theoretical_loss": 3.7414884583959696,
+ "tokens_seen": 835911680
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000603394182547643,
+ "loss": 1.4147,
+ "theoretical_loss": 3.7414598314136045,
+ "tokens_seen": 835977216
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006033781344032097,
+ "loss": 1.4463,
+ "theoretical_loss": 3.741431207303668,
+ "tokens_seen": 836042752
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006033620862587764,
+ "loss": 1.2938,
+ "theoretical_loss": 3.7414025860656475,
+ "tokens_seen": 836108288
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006033460381143431,
+ "loss": 1.6035,
+ "theoretical_loss": 3.7413739676990283,
+ "tokens_seen": 836173824
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006033299899699098,
+ "loss": 1.4507,
+ "theoretical_loss": 3.741345352203298,
+ "tokens_seen": 836239360
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006033139418254765,
+ "loss": 1.3425,
+ "theoretical_loss": 3.741316739577944,
+ "tokens_seen": 836304896
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032978936810432,
+ "loss": 1.4659,
+ "theoretical_loss": 3.741288129822453,
+ "tokens_seen": 836370432
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032818455366099,
+ "loss": 1.4809,
+ "theoretical_loss": 3.7412595229363124,
+ "tokens_seen": 836435968
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032657973921766,
+ "loss": 1.4753,
+ "theoretical_loss": 3.74123091891901,
+ "tokens_seen": 836501504
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032497492477433,
+ "loss": 1.3901,
+ "theoretical_loss": 3.741202317770033,
+ "tokens_seen": 836567040
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.00060323370110331,
+ "loss": 1.3956,
+ "theoretical_loss": 3.7411737194888692,
+ "tokens_seen": 836632576
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032176529588767,
+ "loss": 1.4074,
+ "theoretical_loss": 3.7411451240750067,
+ "tokens_seen": 836698112
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006032016048144434,
+ "loss": 1.4154,
+ "theoretical_loss": 3.7411165315279327,
+ "tokens_seen": 836763648
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031855566700101,
+ "loss": 1.4605,
+ "theoretical_loss": 3.7410879418471357,
+ "tokens_seen": 836829184
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031695085255768,
+ "loss": 1.4226,
+ "theoretical_loss": 3.741059355032104,
+ "tokens_seen": 836894720
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031534603811435,
+ "loss": 1.4547,
+ "theoretical_loss": 3.7410307710823263,
+ "tokens_seen": 836960256
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031374122367102,
+ "loss": 1.4513,
+ "theoretical_loss": 3.7410021899972907,
+ "tokens_seen": 837025792
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031213640922769,
+ "loss": 1.3366,
+ "theoretical_loss": 3.740973611776486,
+ "tokens_seen": 837091328
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006031053159478436,
+ "loss": 1.3743,
+ "theoretical_loss": 3.740945036419401,
+ "tokens_seen": 837156864
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006030892678034102,
+ "loss": 1.4312,
+ "theoretical_loss": 3.740916463925524,
+ "tokens_seen": 837222400
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000603073219658977,
+ "loss": 1.5133,
+ "theoretical_loss": 3.740887894294344,
+ "tokens_seen": 837287936
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006030571715145436,
+ "loss": 1.4867,
+ "theoretical_loss": 3.740859327525352,
+ "tokens_seen": 837353472
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006030411233701104,
+ "loss": 1.423,
+ "theoretical_loss": 3.740830763618035,
+ "tokens_seen": 837419008
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000603025075225677,
+ "loss": 1.5043,
+ "theoretical_loss": 3.7408022025718832,
+ "tokens_seen": 837484544
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006030090270812438,
+ "loss": 1.34,
+ "theoretical_loss": 3.740773644386387,
+ "tokens_seen": 837550080
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006029929789368104,
+ "loss": 1.3819,
+ "theoretical_loss": 3.7407450890610345,
+ "tokens_seen": 837615616
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006029769307923772,
+ "loss": 1.4022,
+ "theoretical_loss": 3.7407165365953166,
+ "tokens_seen": 837681152
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006029608826479438,
+ "loss": 1.4855,
+ "theoretical_loss": 3.740687986988723,
+ "tokens_seen": 837746688
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006029448345035106,
+ "loss": 1.3891,
+ "theoretical_loss": 3.740659440240744,
+ "tokens_seen": 837812224
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006029287863590772,
+ "loss": 1.358,
+ "theoretical_loss": 3.740630896350869,
+ "tokens_seen": 837877760
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602912738214644,
+ "loss": 1.3737,
+ "theoretical_loss": 3.740602355318589,
+ "tokens_seen": 837943296
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028966900702106,
+ "loss": 1.3636,
+ "theoretical_loss": 3.7405738171433947,
+ "tokens_seen": 838008832
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028806419257774,
+ "loss": 1.4324,
+ "theoretical_loss": 3.740545281824776,
+ "tokens_seen": 838074368
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602864593781344,
+ "loss": 1.3236,
+ "theoretical_loss": 3.7405167493622242,
+ "tokens_seen": 838139904
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028485456369108,
+ "loss": 1.3804,
+ "theoretical_loss": 3.740488219755229,
+ "tokens_seen": 838205440
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028324974924774,
+ "loss": 1.3289,
+ "theoretical_loss": 3.740459693003283,
+ "tokens_seen": 838270976
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028164493480442,
+ "loss": 1.4463,
+ "theoretical_loss": 3.7404311691058765,
+ "tokens_seen": 838336512
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006028004012036108,
+ "loss": 1.514,
+ "theoretical_loss": 3.740402648062501,
+ "tokens_seen": 838402048
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006027843530591776,
+ "loss": 1.4703,
+ "theoretical_loss": 3.740374129872647,
+ "tokens_seen": 838467584
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006027683049147442,
+ "loss": 1.4832,
+ "theoretical_loss": 3.7403456145358076,
+ "tokens_seen": 838533120
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602752256770311,
+ "loss": 1.3512,
+ "theoretical_loss": 3.740317102051473,
+ "tokens_seen": 838598656
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006027362086258776,
+ "loss": 1.3195,
+ "theoretical_loss": 3.7402885924191356,
+ "tokens_seen": 838664192
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006027201604814444,
+ "loss": 1.3966,
+ "theoretical_loss": 3.740260085638287,
+ "tokens_seen": 838729728
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602704112337011,
+ "loss": 1.3818,
+ "theoretical_loss": 3.7402315817084197,
+ "tokens_seen": 838795264
+ },
+ {
+ "epoch": 0.25,
+ "objective/train/docs_used": 284714,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.401942253112793,
+ "objective/train/theoretical_loss": 3.7402030806290254,
+ "objective/train/tokens_used": 859320800,
+ "theoretical_loss": 3.7402030806290254,
+ "tokens_seen": 838860800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026880641925778,
+ "loss": 1.3878,
+ "theoretical_loss": 3.7402030806290254,
+ "tokens_seen": 838860800
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026720160481444,
+ "loss": 1.4426,
+ "theoretical_loss": 3.7401745823995967,
+ "tokens_seen": 838926336
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026559679037112,
+ "loss": 1.4389,
+ "theoretical_loss": 3.7401460870196255,
+ "tokens_seen": 838991872
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026399197592778,
+ "loss": 1.4171,
+ "theoretical_loss": 3.7401175944886047,
+ "tokens_seen": 839057408
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026238716148446,
+ "loss": 1.3983,
+ "theoretical_loss": 3.740089104806027,
+ "tokens_seen": 839122944
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006026078234704112,
+ "loss": 1.5401,
+ "theoretical_loss": 3.7400606179713853,
+ "tokens_seen": 839188480
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602591775325978,
+ "loss": 1.4772,
+ "theoretical_loss": 3.7400321339841724,
+ "tokens_seen": 839254016
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006025757271815446,
+ "loss": 1.4685,
+ "theoretical_loss": 3.740003652843882,
+ "tokens_seen": 839319552
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006025596790371114,
+ "loss": 1.4278,
+ "theoretical_loss": 3.739975174550006,
+ "tokens_seen": 839385088
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602543630892678,
+ "loss": 1.3401,
+ "theoretical_loss": 3.7399466991020383,
+ "tokens_seen": 839450624
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006025275827482448,
+ "loss": 1.4309,
+ "theoretical_loss": 3.739918226499473,
+ "tokens_seen": 839516160
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006025115346038114,
+ "loss": 1.3303,
+ "theoretical_loss": 3.7398897567418024,
+ "tokens_seen": 839581696
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006024954864593782,
+ "loss": 1.351,
+ "theoretical_loss": 3.7398612898285215,
+ "tokens_seen": 839647232
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006024794383149448,
+ "loss": 1.3971,
+ "theoretical_loss": 3.7398328257591236,
+ "tokens_seen": 839712768
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006024633901705116,
+ "loss": 1.3685,
+ "theoretical_loss": 3.7398043645331027,
+ "tokens_seen": 839778304
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006024473420260782,
+ "loss": 1.4835,
+ "theoretical_loss": 3.7397759061499523,
+ "tokens_seen": 839843840
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602431293881645,
+ "loss": 1.3469,
+ "theoretical_loss": 3.7397474506091677,
+ "tokens_seen": 839909376
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006024152457372116,
+ "loss": 1.3865,
+ "theoretical_loss": 3.7397189979102428,
+ "tokens_seen": 839974912
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023991975927784,
+ "loss": 1.4223,
+ "theoretical_loss": 3.739690548052672,
+ "tokens_seen": 840040448
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602383149448345,
+ "loss": 1.3895,
+ "theoretical_loss": 3.7396621010359494,
+ "tokens_seen": 840105984
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023671013039118,
+ "loss": 1.3644,
+ "theoretical_loss": 3.739633656859571,
+ "tokens_seen": 840171520
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023510531594784,
+ "loss": 1.4034,
+ "theoretical_loss": 3.739605215523031,
+ "tokens_seen": 840237056
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023350050150452,
+ "loss": 1.4005,
+ "theoretical_loss": 3.7395767770258246,
+ "tokens_seen": 840302592
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023189568706118,
+ "loss": 1.4012,
+ "theoretical_loss": 3.739548341367446,
+ "tokens_seen": 840368128
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006023029087261786,
+ "loss": 1.3468,
+ "theoretical_loss": 3.7395199085473916,
+ "tokens_seen": 840433664
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006022868605817452,
+ "loss": 1.4116,
+ "theoretical_loss": 3.7394914785651565,
+ "tokens_seen": 840499200
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602270812437312,
+ "loss": 1.4516,
+ "theoretical_loss": 3.739463051420236,
+ "tokens_seen": 840564736
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006022547642928786,
+ "loss": 1.4447,
+ "theoretical_loss": 3.739434627112126,
+ "tokens_seen": 840630272
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006022387161484454,
+ "loss": 1.4481,
+ "theoretical_loss": 3.7394062056403223,
+ "tokens_seen": 840695808
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602222668004012,
+ "loss": 1.4441,
+ "theoretical_loss": 3.73937778700432,
+ "tokens_seen": 840761344
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006022066198595788,
+ "loss": 1.4476,
+ "theoretical_loss": 3.739349371203616,
+ "tokens_seen": 840826880
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006021905717151454,
+ "loss": 1.3999,
+ "theoretical_loss": 3.739320958237707,
+ "tokens_seen": 840892416
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006021745235707122,
+ "loss": 1.375,
+ "theoretical_loss": 3.7392925481060884,
+ "tokens_seen": 840957952
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006021584754262788,
+ "loss": 1.4156,
+ "theoretical_loss": 3.739264140808256,
+ "tokens_seen": 841023488
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006021424272818456,
+ "loss": 1.3174,
+ "theoretical_loss": 3.7392357363437077,
+ "tokens_seen": 841089024
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006021263791374122,
+ "loss": 1.3175,
+ "theoretical_loss": 3.73920733471194,
+ "tokens_seen": 841154560
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602110330992979,
+ "loss": 1.3119,
+ "theoretical_loss": 3.7391789359124488,
+ "tokens_seen": 841220096
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006020942828485457,
+ "loss": 1.218,
+ "theoretical_loss": 3.739150539944732,
+ "tokens_seen": 841285632
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006020782347041124,
+ "loss": 1.3743,
+ "theoretical_loss": 3.7391221468082856,
+ "tokens_seen": 841351168
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.000602062186559679,
+ "loss": 1.4401,
+ "theoretical_loss": 3.7390937565026086,
+ "tokens_seen": 841416704
+ },
+ {
+ "epoch": 0.25,
+ "learning_rate": 0.0006020461384152458,
+ "loss": 1.3299,
+ "theoretical_loss": 3.7390653690271964,
+ "tokens_seen": 841482240
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006020300902708125,
+ "loss": 1.334,
+ "theoretical_loss": 3.7390369843815474,
+ "tokens_seen": 841547776
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006020140421263792,
+ "loss": 1.4229,
+ "theoretical_loss": 3.7390086025651588,
+ "tokens_seen": 841613312
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019979939819459,
+ "loss": 1.4636,
+ "theoretical_loss": 3.7389802235775287,
+ "tokens_seen": 841678848
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019819458375126,
+ "loss": 1.432,
+ "theoretical_loss": 3.738951847418155,
+ "tokens_seen": 841744384
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019658976930793,
+ "loss": 1.4492,
+ "theoretical_loss": 3.738923474086535,
+ "tokens_seen": 841809920
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000601949849548646,
+ "loss": 1.4188,
+ "theoretical_loss": 3.7388951035821676,
+ "tokens_seen": 841875456
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019338014042127,
+ "loss": 1.3371,
+ "theoretical_loss": 3.7388667359045504,
+ "tokens_seen": 841940992
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019177532597794,
+ "loss": 1.3195,
+ "theoretical_loss": 3.7388383710531823,
+ "tokens_seen": 842006528
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006019017051153461,
+ "loss": 1.3625,
+ "theoretical_loss": 3.7388100090275618,
+ "tokens_seen": 842072064
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 285855,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4833232164382935,
+ "objective/train/theoretical_loss": 3.738781649827186,
+ "objective/train/tokens_used": 862597600,
+ "theoretical_loss": 3.738781649827186,
+ "tokens_seen": 842137600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018856569709128,
+ "loss": 1.4449,
+ "theoretical_loss": 3.738781649827186,
+ "tokens_seen": 842137600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018696088264795,
+ "loss": 1.4223,
+ "theoretical_loss": 3.7387532934515564,
+ "tokens_seen": 842203136
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018535606820462,
+ "loss": 1.5405,
+ "theoretical_loss": 3.738724939900169,
+ "tokens_seen": 842268672
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018375125376129,
+ "loss": 1.3723,
+ "theoretical_loss": 3.738696589172525,
+ "tokens_seen": 842334208
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018214643931796,
+ "loss": 1.3751,
+ "theoretical_loss": 3.7386682412681225,
+ "tokens_seen": 842399744
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006018054162487463,
+ "loss": 1.4065,
+ "theoretical_loss": 3.73863989618646,
+ "tokens_seen": 842465280
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000601789368104313,
+ "loss": 1.3013,
+ "theoretical_loss": 3.738611553927039,
+ "tokens_seen": 842530816
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006017733199598797,
+ "loss": 1.3471,
+ "theoretical_loss": 3.738583214489357,
+ "tokens_seen": 842596352
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006017572718154464,
+ "loss": 1.4147,
+ "theoretical_loss": 3.7385548778729145,
+ "tokens_seen": 842661888
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006017412236710131,
+ "loss": 1.3634,
+ "theoretical_loss": 3.738526544077211,
+ "tokens_seen": 842727424
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006017251755265798,
+ "loss": 1.4423,
+ "theoretical_loss": 3.738498213101747,
+ "tokens_seen": 842792960
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006017091273821465,
+ "loss": 1.4134,
+ "theoretical_loss": 3.7384698849460216,
+ "tokens_seen": 842858496
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016930792377131,
+ "loss": 1.3371,
+ "theoretical_loss": 3.7384415596095355,
+ "tokens_seen": 842924032
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016770310932799,
+ "loss": 1.362,
+ "theoretical_loss": 3.7384132370917884,
+ "tokens_seen": 842989568
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016609829488465,
+ "loss": 1.3966,
+ "theoretical_loss": 3.738384917392282,
+ "tokens_seen": 843055104
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016449348044133,
+ "loss": 1.3893,
+ "theoretical_loss": 3.7383566005105155,
+ "tokens_seen": 843120640
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016288866599799,
+ "loss": 1.3767,
+ "theoretical_loss": 3.7383282864459897,
+ "tokens_seen": 843186176
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006016128385155467,
+ "loss": 1.4102,
+ "theoretical_loss": 3.7382999751982062,
+ "tokens_seen": 843251712
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015967903711133,
+ "loss": 1.4282,
+ "theoretical_loss": 3.738271666766665,
+ "tokens_seen": 843317248
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015807422266801,
+ "loss": 1.4793,
+ "theoretical_loss": 3.738243361150868,
+ "tokens_seen": 843382784
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015646940822467,
+ "loss": 1.3901,
+ "theoretical_loss": 3.7382150583503155,
+ "tokens_seen": 843448320
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015486459378135,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7381867583645096,
+ "tokens_seen": 843513856
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015325977933801,
+ "loss": 1.5218,
+ "theoretical_loss": 3.738158461192951,
+ "tokens_seen": 843579392
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015165496489469,
+ "loss": 1.4485,
+ "theoretical_loss": 3.738130166835141,
+ "tokens_seen": 843644928
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006015005015045135,
+ "loss": 1.3031,
+ "theoretical_loss": 3.7381018752905826,
+ "tokens_seen": 843710464
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014844533600803,
+ "loss": 1.4174,
+ "theoretical_loss": 3.7380735865587766,
+ "tokens_seen": 843776000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014684052156469,
+ "loss": 1.3375,
+ "theoretical_loss": 3.738045300639225,
+ "tokens_seen": 843841536
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014523570712137,
+ "loss": 1.4707,
+ "theoretical_loss": 3.7380170175314302,
+ "tokens_seen": 843907072
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014363089267803,
+ "loss": 1.3438,
+ "theoretical_loss": 3.737988737234894,
+ "tokens_seen": 843972608
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014202607823471,
+ "loss": 1.39,
+ "theoretical_loss": 3.737960459749119,
+ "tokens_seen": 844038144
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006014042126379137,
+ "loss": 1.4359,
+ "theoretical_loss": 3.737932185073607,
+ "tokens_seen": 844103680
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013881644934805,
+ "loss": 1.534,
+ "theoretical_loss": 3.7379039132078615,
+ "tokens_seen": 844169216
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013721163490471,
+ "loss": 1.4283,
+ "theoretical_loss": 3.7378756441513845,
+ "tokens_seen": 844234752
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013560682046139,
+ "loss": 1.3978,
+ "theoretical_loss": 3.737847377903679,
+ "tokens_seen": 844300288
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013400200601805,
+ "loss": 1.4508,
+ "theoretical_loss": 3.737819114464248,
+ "tokens_seen": 844365824
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013239719157473,
+ "loss": 1.3086,
+ "theoretical_loss": 3.7377908538325944,
+ "tokens_seen": 844431360
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006013079237713139,
+ "loss": 1.4145,
+ "theoretical_loss": 3.7377625960082215,
+ "tokens_seen": 844496896
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012918756268807,
+ "loss": 1.2873,
+ "theoretical_loss": 3.7377343409906327,
+ "tokens_seen": 844562432
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012758274824473,
+ "loss": 1.3702,
+ "theoretical_loss": 3.7377060887793316,
+ "tokens_seen": 844627968
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012597793380141,
+ "loss": 1.3439,
+ "theoretical_loss": 3.7376778393738217,
+ "tokens_seen": 844693504
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012437311935807,
+ "loss": 1.4976,
+ "theoretical_loss": 3.737649592773606,
+ "tokens_seen": 844759040
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012276830491475,
+ "loss": 1.3224,
+ "theoretical_loss": 3.737621348978189,
+ "tokens_seen": 844824576
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006012116349047141,
+ "loss": 1.3752,
+ "theoretical_loss": 3.7375931079870743,
+ "tokens_seen": 844890112
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011955867602809,
+ "loss": 1.4234,
+ "theoretical_loss": 3.7375648697997663,
+ "tokens_seen": 844955648
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011795386158475,
+ "loss": 1.3911,
+ "theoretical_loss": 3.737536634415769,
+ "tokens_seen": 845021184
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011634904714143,
+ "loss": 1.3739,
+ "theoretical_loss": 3.737508401834587,
+ "tokens_seen": 845086720
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011474423269809,
+ "loss": 1.3997,
+ "theoretical_loss": 3.7374801720557245,
+ "tokens_seen": 845152256
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011313941825477,
+ "loss": 1.501,
+ "theoretical_loss": 3.737451945078686,
+ "tokens_seen": 845217792
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006011153460381143,
+ "loss": 1.2698,
+ "theoretical_loss": 3.737423720902976,
+ "tokens_seen": 845283328
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010992978936811,
+ "loss": 1.4256,
+ "theoretical_loss": 3.7373954995281005,
+ "tokens_seen": 845348864
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 286799,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.428529977798462,
+ "objective/train/theoretical_loss": 3.7373672809535634,
+ "objective/train/tokens_used": 865874400,
+ "theoretical_loss": 3.7373672809535634,
+ "tokens_seen": 845414400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010832497492477,
+ "loss": 1.3799,
+ "theoretical_loss": 3.7373672809535634,
+ "tokens_seen": 845414400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010672016048145,
+ "loss": 1.4102,
+ "theoretical_loss": 3.7373390651788694,
+ "tokens_seen": 845479936
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010511534603811,
+ "loss": 1.3426,
+ "theoretical_loss": 3.7373108522035245,
+ "tokens_seen": 845545472
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010351053159479,
+ "loss": 1.3097,
+ "theoretical_loss": 3.737282642027034,
+ "tokens_seen": 845611008
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010190571715145,
+ "loss": 1.2863,
+ "theoretical_loss": 3.737254434648903,
+ "tokens_seen": 845676544
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006010030090270813,
+ "loss": 1.3013,
+ "theoretical_loss": 3.7372262300686376,
+ "tokens_seen": 845742080
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009869608826479,
+ "loss": 1.4221,
+ "theoretical_loss": 3.737198028285743,
+ "tokens_seen": 845807616
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009709127382147,
+ "loss": 1.324,
+ "theoretical_loss": 3.7371698292997255,
+ "tokens_seen": 845873152
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009548645937813,
+ "loss": 1.3582,
+ "theoretical_loss": 3.73714163311009,
+ "tokens_seen": 845938688
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009388164493481,
+ "loss": 1.3429,
+ "theoretical_loss": 3.737113439716344,
+ "tokens_seen": 846004224
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009227683049147,
+ "loss": 1.4675,
+ "theoretical_loss": 3.737085249117993,
+ "tokens_seen": 846069760
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006009067201604815,
+ "loss": 1.4306,
+ "theoretical_loss": 3.737057061314543,
+ "tokens_seen": 846135296
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006008906720160481,
+ "loss": 1.5029,
+ "theoretical_loss": 3.7370288763055015,
+ "tokens_seen": 846200832
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006008746238716149,
+ "loss": 1.371,
+ "theoretical_loss": 3.7370006940903737,
+ "tokens_seen": 846266368
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006008585757271815,
+ "loss": 1.4102,
+ "theoretical_loss": 3.7369725146686674,
+ "tokens_seen": 846331904
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006008425275827483,
+ "loss": 1.4441,
+ "theoretical_loss": 3.7369443380398897,
+ "tokens_seen": 846397440
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600826479438315,
+ "loss": 1.4409,
+ "theoretical_loss": 3.736916164203546,
+ "tokens_seen": 846462976
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006008104312938817,
+ "loss": 1.6028,
+ "theoretical_loss": 3.7368879931591454,
+ "tokens_seen": 846528512
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007943831494483,
+ "loss": 1.3932,
+ "theoretical_loss": 3.736859824906193,
+ "tokens_seen": 846594048
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007783350050151,
+ "loss": 1.2522,
+ "theoretical_loss": 3.736831659444198,
+ "tokens_seen": 846659584
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007622868605817,
+ "loss": 1.3381,
+ "theoretical_loss": 3.7368034967726667,
+ "tokens_seen": 846725120
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007462387161485,
+ "loss": 1.3857,
+ "theoretical_loss": 3.736775336891107,
+ "tokens_seen": 846790656
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007301905717151,
+ "loss": 1.3398,
+ "theoretical_loss": 3.736747179799027,
+ "tokens_seen": 846856192
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006007141424272819,
+ "loss": 1.3893,
+ "theoretical_loss": 3.736719025495934,
+ "tokens_seen": 846921728
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006980942828486,
+ "loss": 1.4846,
+ "theoretical_loss": 3.7366908739813365,
+ "tokens_seen": 846987264
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006820461384153,
+ "loss": 1.3505,
+ "theoretical_loss": 3.736662725254742,
+ "tokens_seen": 847052800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600665997993982,
+ "loss": 1.3939,
+ "theoretical_loss": 3.736634579315659,
+ "tokens_seen": 847118336
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006499498495487,
+ "loss": 1.321,
+ "theoretical_loss": 3.736606436163596,
+ "tokens_seen": 847183872
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006339017051154,
+ "loss": 1.4235,
+ "theoretical_loss": 3.7365782957980613,
+ "tokens_seen": 847249408
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006178535606821,
+ "loss": 1.4006,
+ "theoretical_loss": 3.736550158218563,
+ "tokens_seen": 847314944
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006006018054162488,
+ "loss": 1.4018,
+ "theoretical_loss": 3.736522023424611,
+ "tokens_seen": 847380480
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005857572718155,
+ "loss": 1.3999,
+ "theoretical_loss": 3.736493891415713,
+ "tokens_seen": 847446016
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005697091273823,
+ "loss": 1.3375,
+ "theoretical_loss": 3.7364657621913784,
+ "tokens_seen": 847511552
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005536609829489,
+ "loss": 1.2714,
+ "theoretical_loss": 3.7364376357511166,
+ "tokens_seen": 847577088
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005376128385157,
+ "loss": 1.355,
+ "theoretical_loss": 3.7364095120944363,
+ "tokens_seen": 847642624
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005215646940823,
+ "loss": 1.2644,
+ "theoretical_loss": 3.736381391220847,
+ "tokens_seen": 847708160
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006005055165496491,
+ "loss": 1.469,
+ "theoretical_loss": 3.736353273129858,
+ "tokens_seen": 847773696
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004894684052157,
+ "loss": 1.397,
+ "theoretical_loss": 3.7363251578209797,
+ "tokens_seen": 847839232
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004734202607825,
+ "loss": 1.4887,
+ "theoretical_loss": 3.7362970452937208,
+ "tokens_seen": 847904768
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004573721163491,
+ "loss": 1.4575,
+ "theoretical_loss": 3.7362689355475913,
+ "tokens_seen": 847970304
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004413239719158,
+ "loss": 1.3535,
+ "theoretical_loss": 3.7362408285821016,
+ "tokens_seen": 848035840
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004252758274825,
+ "loss": 1.4311,
+ "theoretical_loss": 3.7362127243967613,
+ "tokens_seen": 848101376
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006004092276830492,
+ "loss": 1.4185,
+ "theoretical_loss": 3.7361846229910816,
+ "tokens_seen": 848166912
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006003931795386159,
+ "loss": 1.2694,
+ "theoretical_loss": 3.736156524364571,
+ "tokens_seen": 848232448
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006003771313941826,
+ "loss": 1.3745,
+ "theoretical_loss": 3.736128428516742,
+ "tokens_seen": 848297984
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006003610832497493,
+ "loss": 1.4684,
+ "theoretical_loss": 3.736100335447104,
+ "tokens_seen": 848363520
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600345035105316,
+ "loss": 1.3414,
+ "theoretical_loss": 3.736072245155168,
+ "tokens_seen": 848429056
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006003289869608827,
+ "loss": 1.4138,
+ "theoretical_loss": 3.7360441576404444,
+ "tokens_seen": 848494592
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006003129388164494,
+ "loss": 1.4227,
+ "theoretical_loss": 3.7360160729024443,
+ "tokens_seen": 848560128
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600296890672016,
+ "loss": 1.4908,
+ "theoretical_loss": 3.7359879909406795,
+ "tokens_seen": 848625664
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 287940,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3254786729812622,
+ "objective/train/theoretical_loss": 3.7359599117546605,
+ "objective/train/tokens_used": 869151200,
+ "theoretical_loss": 3.7359599117546605,
+ "tokens_seen": 848691200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002808425275828,
+ "loss": 1.35,
+ "theoretical_loss": 3.7359599117546605,
+ "tokens_seen": 848691200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002647943831494,
+ "loss": 1.4168,
+ "theoretical_loss": 3.735931835343899,
+ "tokens_seen": 848756736
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002487462387162,
+ "loss": 1.5306,
+ "theoretical_loss": 3.7359037617079056,
+ "tokens_seen": 848822272
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002326980942828,
+ "loss": 1.3725,
+ "theoretical_loss": 3.735875690846193,
+ "tokens_seen": 848887808
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002166499498496,
+ "loss": 1.3881,
+ "theoretical_loss": 3.735847622758272,
+ "tokens_seen": 848953344
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006002006018054162,
+ "loss": 1.4268,
+ "theoretical_loss": 3.7358195574436555,
+ "tokens_seen": 849018880
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600184553660983,
+ "loss": 1.3647,
+ "theoretical_loss": 3.7357914949018545,
+ "tokens_seen": 849084416
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006001685055165496,
+ "loss": 1.4098,
+ "theoretical_loss": 3.735763435132381,
+ "tokens_seen": 849149952
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006001524573721164,
+ "loss": 1.2927,
+ "theoretical_loss": 3.735735378134748,
+ "tokens_seen": 849215488
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000600136409227683,
+ "loss": 1.304,
+ "theoretical_loss": 3.735707323908467,
+ "tokens_seen": 849281024
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006001203610832498,
+ "loss": 1.3165,
+ "theoretical_loss": 3.7356792724530505,
+ "tokens_seen": 849346560
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006001043129388164,
+ "loss": 1.3498,
+ "theoretical_loss": 3.735651223768012,
+ "tokens_seen": 849412096
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006000882647943832,
+ "loss": 1.3796,
+ "theoretical_loss": 3.735623177852863,
+ "tokens_seen": 849477632
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006000722166499498,
+ "loss": 1.3351,
+ "theoretical_loss": 3.7355951347071166,
+ "tokens_seen": 849543168
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006000561685055166,
+ "loss": 1.4175,
+ "theoretical_loss": 3.7355670943302863,
+ "tokens_seen": 849608704
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006000401203610832,
+ "loss": 1.4171,
+ "theoretical_loss": 3.735539056721885,
+ "tokens_seen": 849674240
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.00060002407221665,
+ "loss": 1.3831,
+ "theoretical_loss": 3.7355110218814254,
+ "tokens_seen": 849739776
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0006000080240722166,
+ "loss": 1.324,
+ "theoretical_loss": 3.7354829898084208,
+ "tokens_seen": 849805312
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005999919759277834,
+ "loss": 1.4369,
+ "theoretical_loss": 3.735454960502385,
+ "tokens_seen": 849870848
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.00059997592778335,
+ "loss": 1.4079,
+ "theoretical_loss": 3.735426933962831,
+ "tokens_seen": 849936384
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005999598796389168,
+ "loss": 1.4077,
+ "theoretical_loss": 3.7353989101892733,
+ "tokens_seen": 850001920
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005999438314944834,
+ "loss": 1.4385,
+ "theoretical_loss": 3.7353708891812256,
+ "tokens_seen": 850067456
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005999277833500502,
+ "loss": 1.5027,
+ "theoretical_loss": 3.735342870938201,
+ "tokens_seen": 850132992
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005999117352056168,
+ "loss": 1.3286,
+ "theoretical_loss": 3.7353148554597144,
+ "tokens_seen": 850198528
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005998956870611836,
+ "loss": 1.4426,
+ "theoretical_loss": 3.7352868427452792,
+ "tokens_seen": 850264064
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005998796389167502,
+ "loss": 1.3627,
+ "theoretical_loss": 3.7352588327944103,
+ "tokens_seen": 850329600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599863590772317,
+ "loss": 1.423,
+ "theoretical_loss": 3.735230825606622,
+ "tokens_seen": 850395136
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005998475426278836,
+ "loss": 1.4013,
+ "theoretical_loss": 3.735202821181428,
+ "tokens_seen": 850460672
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005998314944834504,
+ "loss": 1.5009,
+ "theoretical_loss": 3.7351748195183436,
+ "tokens_seen": 850526208
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599815446339017,
+ "loss": 1.4742,
+ "theoretical_loss": 3.735146820616884,
+ "tokens_seen": 850591744
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997993981945838,
+ "loss": 1.4065,
+ "theoretical_loss": 3.735118824476564,
+ "tokens_seen": 850657280
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997833500501504,
+ "loss": 1.4179,
+ "theoretical_loss": 3.7350908310968975,
+ "tokens_seen": 850722816
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997673019057172,
+ "loss": 1.4582,
+ "theoretical_loss": 3.735062840477401,
+ "tokens_seen": 850788352
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997512537612838,
+ "loss": 1.4548,
+ "theoretical_loss": 3.735034852617589,
+ "tokens_seen": 850853888
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997352056168506,
+ "loss": 1.3788,
+ "theoretical_loss": 3.7350068675169776,
+ "tokens_seen": 850919424
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005997191574724172,
+ "loss": 1.4552,
+ "theoretical_loss": 3.734978885175081,
+ "tokens_seen": 850984960
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599703109327984,
+ "loss": 1.4249,
+ "theoretical_loss": 3.734950905591416,
+ "tokens_seen": 851050496
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005996870611835506,
+ "loss": 1.3944,
+ "theoretical_loss": 3.7349229287654984,
+ "tokens_seen": 851116032
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005996710130391174,
+ "loss": 1.3482,
+ "theoretical_loss": 3.734894954696843,
+ "tokens_seen": 851181568
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599654964894684,
+ "loss": 1.4952,
+ "theoretical_loss": 3.7348669833849666,
+ "tokens_seen": 851247104
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005996389167502508,
+ "loss": 1.3647,
+ "theoretical_loss": 3.734839014829385,
+ "tokens_seen": 851312640
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005996228686058174,
+ "loss": 1.3801,
+ "theoretical_loss": 3.7348110490296147,
+ "tokens_seen": 851378176
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005996068204613842,
+ "loss": 1.4457,
+ "theoretical_loss": 3.734783085985172,
+ "tokens_seen": 851443712
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005995907723169508,
+ "loss": 1.3893,
+ "theoretical_loss": 3.7347551256955738,
+ "tokens_seen": 851509248
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005995747241725176,
+ "loss": 1.3184,
+ "theoretical_loss": 3.734727168160336,
+ "tokens_seen": 851574784
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005995586760280842,
+ "loss": 1.4529,
+ "theoretical_loss": 3.734699213378975,
+ "tokens_seen": 851640320
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599542627883651,
+ "loss": 1.4379,
+ "theoretical_loss": 3.734671261351009,
+ "tokens_seen": 851705856
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005995265797392176,
+ "loss": 1.3981,
+ "theoretical_loss": 3.7346433120759537,
+ "tokens_seen": 851771392
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005995105315947844,
+ "loss": 1.422,
+ "theoretical_loss": 3.734615365553327,
+ "tokens_seen": 851836928
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599494483450351,
+ "loss": 1.4512,
+ "theoretical_loss": 3.7345874217826456,
+ "tokens_seen": 851902464
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 288857,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3938164710998535,
+ "objective/train/theoretical_loss": 3.734559480763427,
+ "objective/train/tokens_used": 872428000,
+ "theoretical_loss": 3.734559480763427,
+ "tokens_seen": 851968000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005994784353059178,
+ "loss": 1.3835,
+ "theoretical_loss": 3.734559480763427,
+ "tokens_seen": 851968000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005994623871614844,
+ "loss": 1.3932,
+ "theoretical_loss": 3.734531542495189,
+ "tokens_seen": 852033536
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005994463390170512,
+ "loss": 1.2591,
+ "theoretical_loss": 3.7345036069774484,
+ "tokens_seen": 852099072
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005994302908726178,
+ "loss": 1.3624,
+ "theoretical_loss": 3.734475674209724,
+ "tokens_seen": 852164608
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005994142427281846,
+ "loss": 1.4323,
+ "theoretical_loss": 3.7344477441915322,
+ "tokens_seen": 852230144
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993981945837514,
+ "loss": 1.4479,
+ "theoretical_loss": 3.7344198169223928,
+ "tokens_seen": 852295680
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599382146439318,
+ "loss": 1.4421,
+ "theoretical_loss": 3.734391892401822,
+ "tokens_seen": 852361216
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993660982948848,
+ "loss": 1.3885,
+ "theoretical_loss": 3.7343639706293392,
+ "tokens_seen": 852426752
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993500501504514,
+ "loss": 1.3702,
+ "theoretical_loss": 3.734336051604462,
+ "tokens_seen": 852492288
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993340020060182,
+ "loss": 1.3556,
+ "theoretical_loss": 3.7343081353267094,
+ "tokens_seen": 852557824
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993179538615848,
+ "loss": 1.427,
+ "theoretical_loss": 3.7342802217955997,
+ "tokens_seen": 852623360
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005993019057171516,
+ "loss": 1.4236,
+ "theoretical_loss": 3.7342523110106516,
+ "tokens_seen": 852688896
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005992858575727182,
+ "loss": 1.3168,
+ "theoretical_loss": 3.7342244029713845,
+ "tokens_seen": 852754432
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599269809428285,
+ "loss": 1.4055,
+ "theoretical_loss": 3.7341964976773157,
+ "tokens_seen": 852819968
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005992537612838516,
+ "loss": 1.3059,
+ "theoretical_loss": 3.734168595127966,
+ "tokens_seen": 852885504
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005992377131394184,
+ "loss": 1.3196,
+ "theoretical_loss": 3.7341406953228535,
+ "tokens_seen": 852951040
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599221664994985,
+ "loss": 1.3316,
+ "theoretical_loss": 3.7341127982614974,
+ "tokens_seen": 853016576
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005992056168505518,
+ "loss": 1.3972,
+ "theoretical_loss": 3.734084903943418,
+ "tokens_seen": 853082112
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005991895687061184,
+ "loss": 1.4237,
+ "theoretical_loss": 3.734057012368134,
+ "tokens_seen": 853147648
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005991735205616852,
+ "loss": 1.4201,
+ "theoretical_loss": 3.7340291235351657,
+ "tokens_seen": 853213184
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005991574724172518,
+ "loss": 1.4324,
+ "theoretical_loss": 3.734001237444032,
+ "tokens_seen": 853278720
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005991414242728186,
+ "loss": 1.4876,
+ "theoretical_loss": 3.7339733540942537,
+ "tokens_seen": 853344256
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005991253761283852,
+ "loss": 1.2994,
+ "theoretical_loss": 3.7339454734853508,
+ "tokens_seen": 853409792
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599109327983952,
+ "loss": 1.342,
+ "theoretical_loss": 3.733917595616842,
+ "tokens_seen": 853475328
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005990932798395186,
+ "loss": 1.4272,
+ "theoretical_loss": 3.7338897204882495,
+ "tokens_seen": 853540864
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005990772316950854,
+ "loss": 1.3983,
+ "theoretical_loss": 3.7338618480990924,
+ "tokens_seen": 853606400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000599061183550652,
+ "loss": 1.43,
+ "theoretical_loss": 3.7338339784488914,
+ "tokens_seen": 853671936
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005990451354062187,
+ "loss": 1.3391,
+ "theoretical_loss": 3.7338061115371675,
+ "tokens_seen": 853737472
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005990290872617854,
+ "loss": 1.2993,
+ "theoretical_loss": 3.7337782473634413,
+ "tokens_seen": 853803008
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005990130391173521,
+ "loss": 1.3782,
+ "theoretical_loss": 3.733750385927234,
+ "tokens_seen": 853868544
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989969909729188,
+ "loss": 1.4878,
+ "theoretical_loss": 3.733722527228065,
+ "tokens_seen": 853934080
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989809428284855,
+ "loss": 1.4,
+ "theoretical_loss": 3.7336946712654573,
+ "tokens_seen": 853999616
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989648946840522,
+ "loss": 1.3102,
+ "theoretical_loss": 3.733666818038931,
+ "tokens_seen": 854065152
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989488465396189,
+ "loss": 1.2489,
+ "theoretical_loss": 3.733638967548008,
+ "tokens_seen": 854130688
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989327983951856,
+ "loss": 1.4477,
+ "theoretical_loss": 3.733611119792209,
+ "tokens_seen": 854196224
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989167502507523,
+ "loss": 1.3739,
+ "theoretical_loss": 3.733583274771057,
+ "tokens_seen": 854261760
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005989007021063189,
+ "loss": 1.4298,
+ "theoretical_loss": 3.733555432484072,
+ "tokens_seen": 854327296
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988846539618857,
+ "loss": 1.3673,
+ "theoretical_loss": 3.733527592930777,
+ "tokens_seen": 854392832
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988686058174523,
+ "loss": 1.2787,
+ "theoretical_loss": 3.7334997561106933,
+ "tokens_seen": 854458368
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988525576730191,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7334719220233437,
+ "tokens_seen": 854523904
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988365095285857,
+ "loss": 1.3878,
+ "theoretical_loss": 3.7334440906682493,
+ "tokens_seen": 854589440
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988204613841525,
+ "loss": 1.4067,
+ "theoretical_loss": 3.7334162620449334,
+ "tokens_seen": 854654976
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005988044132397191,
+ "loss": 1.2822,
+ "theoretical_loss": 3.733388436152918,
+ "tokens_seen": 854720512
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987883650952859,
+ "loss": 1.4486,
+ "theoretical_loss": 3.733360612991725,
+ "tokens_seen": 854786048
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987723169508525,
+ "loss": 1.5065,
+ "theoretical_loss": 3.733332792560878,
+ "tokens_seen": 854851584
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987562688064193,
+ "loss": 1.3977,
+ "theoretical_loss": 3.7333049748599,
+ "tokens_seen": 854917120
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987402206619859,
+ "loss": 1.4028,
+ "theoretical_loss": 3.7332771598883134,
+ "tokens_seen": 854982656
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987241725175527,
+ "loss": 1.4019,
+ "theoretical_loss": 3.7332493476456405,
+ "tokens_seen": 855048192
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005987081243731193,
+ "loss": 1.4379,
+ "theoretical_loss": 3.7332215381314056,
+ "tokens_seen": 855113728
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986920762286861,
+ "loss": 1.4441,
+ "theoretical_loss": 3.733193731345131,
+ "tokens_seen": 855179264
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 290023,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.482955813407898,
+ "objective/train/theoretical_loss": 3.733165927286341,
+ "objective/train/tokens_used": 875704800,
+ "theoretical_loss": 3.733165927286341,
+ "tokens_seen": 855244800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986760280842527,
+ "loss": 1.3843,
+ "theoretical_loss": 3.733165927286341,
+ "tokens_seen": 855244800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986599799398195,
+ "loss": 1.3889,
+ "theoretical_loss": 3.7331381259545586,
+ "tokens_seen": 855310336
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986439317953861,
+ "loss": 1.4774,
+ "theoretical_loss": 3.7331103273493076,
+ "tokens_seen": 855375872
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986278836509529,
+ "loss": 1.4163,
+ "theoretical_loss": 3.733082531470111,
+ "tokens_seen": 855441408
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005986118355065195,
+ "loss": 1.4104,
+ "theoretical_loss": 3.7330547383164943,
+ "tokens_seen": 855506944
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985957873620863,
+ "loss": 1.3666,
+ "theoretical_loss": 3.7330269478879794,
+ "tokens_seen": 855572480
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985797392176529,
+ "loss": 1.4714,
+ "theoretical_loss": 3.732999160184092,
+ "tokens_seen": 855638016
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985636910732197,
+ "loss": 1.3387,
+ "theoretical_loss": 3.7329713752043556,
+ "tokens_seen": 855703552
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985476429287863,
+ "loss": 1.378,
+ "theoretical_loss": 3.7329435929482946,
+ "tokens_seen": 855769088
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985315947843531,
+ "loss": 1.4485,
+ "theoretical_loss": 3.732915813415434,
+ "tokens_seen": 855834624
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005985155466399197,
+ "loss": 1.32,
+ "theoretical_loss": 3.732888036605298,
+ "tokens_seen": 855900160
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984994984954865,
+ "loss": 1.4106,
+ "theoretical_loss": 3.7328602625174105,
+ "tokens_seen": 855965696
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984834503510531,
+ "loss": 1.4481,
+ "theoretical_loss": 3.7328324911512976,
+ "tokens_seen": 856031232
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984674022066199,
+ "loss": 1.2989,
+ "theoretical_loss": 3.7328047225064838,
+ "tokens_seen": 856096768
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984513540621865,
+ "loss": 1.3665,
+ "theoretical_loss": 3.7327769565824935,
+ "tokens_seen": 856162304
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984353059177533,
+ "loss": 1.2125,
+ "theoretical_loss": 3.7327491933788526,
+ "tokens_seen": 856227840
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984192577733199,
+ "loss": 1.3547,
+ "theoretical_loss": 3.7327214328950857,
+ "tokens_seen": 856293376
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005984032096288867,
+ "loss": 1.3171,
+ "theoretical_loss": 3.7326936751307196,
+ "tokens_seen": 856358912
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983871614844533,
+ "loss": 1.4804,
+ "theoretical_loss": 3.732665920085278,
+ "tokens_seen": 856424448
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983711133400201,
+ "loss": 1.3623,
+ "theoretical_loss": 3.732638167758288,
+ "tokens_seen": 856489984
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983550651955867,
+ "loss": 1.4546,
+ "theoretical_loss": 3.7326104181492745,
+ "tokens_seen": 856555520
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983390170511535,
+ "loss": 1.4384,
+ "theoretical_loss": 3.732582671257764,
+ "tokens_seen": 856621056
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983229689067201,
+ "loss": 1.3613,
+ "theoretical_loss": 3.732554927083282,
+ "tokens_seen": 856686592
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005983069207622869,
+ "loss": 1.5608,
+ "theoretical_loss": 3.7325271856253552,
+ "tokens_seen": 856752128
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982908726178535,
+ "loss": 1.4797,
+ "theoretical_loss": 3.732499446883509,
+ "tokens_seen": 856817664
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982748244734203,
+ "loss": 1.3227,
+ "theoretical_loss": 3.7324717108572703,
+ "tokens_seen": 856883200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982587763289869,
+ "loss": 1.3517,
+ "theoretical_loss": 3.7324439775461657,
+ "tokens_seen": 856948736
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982427281845537,
+ "loss": 1.4239,
+ "theoretical_loss": 3.7324162469497217,
+ "tokens_seen": 857014272
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982266800401203,
+ "loss": 1.4292,
+ "theoretical_loss": 3.732388519067465,
+ "tokens_seen": 857079808
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005982106318956871,
+ "loss": 1.4278,
+ "theoretical_loss": 3.732360793898922,
+ "tokens_seen": 857145344
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981945837512539,
+ "loss": 1.3697,
+ "theoretical_loss": 3.7323330714436205,
+ "tokens_seen": 857210880
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981785356068205,
+ "loss": 1.4008,
+ "theoretical_loss": 3.7323053517010862,
+ "tokens_seen": 857276416
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981624874623873,
+ "loss": 1.3641,
+ "theoretical_loss": 3.732277634670848,
+ "tokens_seen": 857341952
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981464393179539,
+ "loss": 1.3369,
+ "theoretical_loss": 3.732249920352432,
+ "tokens_seen": 857407488
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981303911735207,
+ "loss": 1.4052,
+ "theoretical_loss": 3.732222208745366,
+ "tokens_seen": 857473024
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005981143430290873,
+ "loss": 1.3744,
+ "theoretical_loss": 3.732194499849178,
+ "tokens_seen": 857538560
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000598098294884654,
+ "loss": 1.3696,
+ "theoretical_loss": 3.732166793663395,
+ "tokens_seen": 857604096
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980822467402207,
+ "loss": 1.3836,
+ "theoretical_loss": 3.7321390901875455,
+ "tokens_seen": 857669632
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980661985957875,
+ "loss": 1.3862,
+ "theoretical_loss": 3.7321113894211564,
+ "tokens_seen": 857735168
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980501504513541,
+ "loss": 1.4014,
+ "theoretical_loss": 3.732083691363756,
+ "tokens_seen": 857800704
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980341023069209,
+ "loss": 1.3548,
+ "theoretical_loss": 3.732055996014873,
+ "tokens_seen": 857866240
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980180541624875,
+ "loss": 1.4254,
+ "theoretical_loss": 3.732028303374035,
+ "tokens_seen": 857931776
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005980020060180543,
+ "loss": 1.4273,
+ "theoretical_loss": 3.7320006134407713,
+ "tokens_seen": 857997312
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979859578736209,
+ "loss": 1.3563,
+ "theoretical_loss": 3.7319729262146093,
+ "tokens_seen": 858062848
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979699097291877,
+ "loss": 1.3577,
+ "theoretical_loss": 3.7319452416950782,
+ "tokens_seen": 858128384
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979538615847543,
+ "loss": 1.2903,
+ "theoretical_loss": 3.731917559881707,
+ "tokens_seen": 858193920
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979378134403211,
+ "loss": 1.4991,
+ "theoretical_loss": 3.7318898807740237,
+ "tokens_seen": 858259456
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979217652958877,
+ "loss": 1.3234,
+ "theoretical_loss": 3.7318622043715584,
+ "tokens_seen": 858324992
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005979057171514545,
+ "loss": 1.4198,
+ "theoretical_loss": 3.731834530673839,
+ "tokens_seen": 858390528
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978896690070211,
+ "loss": 1.3654,
+ "theoretical_loss": 3.731806859680395,
+ "tokens_seen": 858456064
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 291206,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.360671043395996,
+ "objective/train/theoretical_loss": 3.7317791913907565,
+ "objective/train/tokens_used": 878981600,
+ "theoretical_loss": 3.7317791913907565,
+ "tokens_seen": 858521600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978736208625879,
+ "loss": 1.3988,
+ "theoretical_loss": 3.7317791913907565,
+ "tokens_seen": 858521600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978575727181545,
+ "loss": 1.4183,
+ "theoretical_loss": 3.7317515258044525,
+ "tokens_seen": 858587136
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978415245737213,
+ "loss": 1.4504,
+ "theoretical_loss": 3.731723862921012,
+ "tokens_seen": 858652672
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978254764292879,
+ "loss": 1.3,
+ "theoretical_loss": 3.7316962027399656,
+ "tokens_seen": 858718208
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005978094282848547,
+ "loss": 1.3513,
+ "theoretical_loss": 3.731668545260842,
+ "tokens_seen": 858783744
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977933801404213,
+ "loss": 1.3975,
+ "theoretical_loss": 3.731640890483172,
+ "tokens_seen": 858849280
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977773319959881,
+ "loss": 1.3566,
+ "theoretical_loss": 3.7316132384064855,
+ "tokens_seen": 858914816
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977612838515547,
+ "loss": 1.2503,
+ "theoretical_loss": 3.7315855890303125,
+ "tokens_seen": 858980352
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977452357071215,
+ "loss": 1.3104,
+ "theoretical_loss": 3.7315579423541836,
+ "tokens_seen": 859045888
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977291875626881,
+ "loss": 1.3923,
+ "theoretical_loss": 3.731530298377628,
+ "tokens_seen": 859111424
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005977131394182549,
+ "loss": 1.3938,
+ "theoretical_loss": 3.7315026571001777,
+ "tokens_seen": 859176960
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976970912738215,
+ "loss": 1.3879,
+ "theoretical_loss": 3.731475018521362,
+ "tokens_seen": 859242496
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976810431293883,
+ "loss": 1.4689,
+ "theoretical_loss": 3.731447382640713,
+ "tokens_seen": 859308032
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976649949849549,
+ "loss": 1.3008,
+ "theoretical_loss": 3.73141974945776,
+ "tokens_seen": 859373568
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976489468405216,
+ "loss": 1.399,
+ "theoretical_loss": 3.7313921189720354,
+ "tokens_seen": 859439104
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976328986960883,
+ "loss": 1.2753,
+ "theoretical_loss": 3.7313644911830695,
+ "tokens_seen": 859504640
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597616850551655,
+ "loss": 1.3152,
+ "theoretical_loss": 3.731336866090394,
+ "tokens_seen": 859570176
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005976008024072217,
+ "loss": 1.4584,
+ "theoretical_loss": 3.7313092436935396,
+ "tokens_seen": 859635712
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975847542627884,
+ "loss": 1.4569,
+ "theoretical_loss": 3.7312816239920377,
+ "tokens_seen": 859701248
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975687061183551,
+ "loss": 1.3976,
+ "theoretical_loss": 3.731254006985421,
+ "tokens_seen": 859766784
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975526579739218,
+ "loss": 1.3175,
+ "theoretical_loss": 3.7312263926732196,
+ "tokens_seen": 859832320
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975366098294885,
+ "loss": 1.3636,
+ "theoretical_loss": 3.731198781054966,
+ "tokens_seen": 859897856
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975205616850552,
+ "loss": 1.3564,
+ "theoretical_loss": 3.7311711721301926,
+ "tokens_seen": 859963392
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005975045135406218,
+ "loss": 1.348,
+ "theoretical_loss": 3.7311435658984307,
+ "tokens_seen": 860028928
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005974884653961886,
+ "loss": 1.4549,
+ "theoretical_loss": 3.7311159623592127,
+ "tokens_seen": 860094464
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005974724172517552,
+ "loss": 1.3301,
+ "theoretical_loss": 3.7310883615120707,
+ "tokens_seen": 860160000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597456369107322,
+ "loss": 1.385,
+ "theoretical_loss": 3.7310607633565374,
+ "tokens_seen": 860225536
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005974403209628886,
+ "loss": 1.3361,
+ "theoretical_loss": 3.7310331678921456,
+ "tokens_seen": 860291072
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005974242728184554,
+ "loss": 1.4569,
+ "theoretical_loss": 3.731005575118427,
+ "tokens_seen": 860356608
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597408224674022,
+ "loss": 1.4702,
+ "theoretical_loss": 3.7309779850349143,
+ "tokens_seen": 860422144
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973921765295888,
+ "loss": 1.3233,
+ "theoretical_loss": 3.730950397641141,
+ "tokens_seen": 860487680
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973761283851554,
+ "loss": 1.3819,
+ "theoretical_loss": 3.73092281293664,
+ "tokens_seen": 860553216
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973600802407222,
+ "loss": 1.3594,
+ "theoretical_loss": 3.730895230920944,
+ "tokens_seen": 860618752
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973440320962888,
+ "loss": 1.2954,
+ "theoretical_loss": 3.730867651593586,
+ "tokens_seen": 860684288
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973279839518556,
+ "loss": 1.3937,
+ "theoretical_loss": 3.7308400749541004,
+ "tokens_seen": 860749824
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005973119358074222,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7308125010020197,
+ "tokens_seen": 860815360
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597295887662989,
+ "loss": 1.4161,
+ "theoretical_loss": 3.7307849297368776,
+ "tokens_seen": 860880896
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005972798395185556,
+ "loss": 1.4161,
+ "theoretical_loss": 3.730757361158208,
+ "tokens_seen": 860946432
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005972637913741224,
+ "loss": 1.4708,
+ "theoretical_loss": 3.7307297952655443,
+ "tokens_seen": 861011968
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597247743229689,
+ "loss": 1.2609,
+ "theoretical_loss": 3.7307022320584204,
+ "tokens_seen": 861077504
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005972316950852558,
+ "loss": 1.522,
+ "theoretical_loss": 3.730674671536371,
+ "tokens_seen": 861143040
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005972156469408224,
+ "loss": 1.511,
+ "theoretical_loss": 3.7306471136989288,
+ "tokens_seen": 861208576
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971995987963892,
+ "loss": 1.3947,
+ "theoretical_loss": 3.7306195585456297,
+ "tokens_seen": 861274112
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971835506519558,
+ "loss": 1.3479,
+ "theoretical_loss": 3.730592006076007,
+ "tokens_seen": 861339648
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971675025075226,
+ "loss": 1.2669,
+ "theoretical_loss": 3.730564456289595,
+ "tokens_seen": 861405184
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971514543630892,
+ "loss": 1.394,
+ "theoretical_loss": 3.730536909185929,
+ "tokens_seen": 861470720
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597135406218656,
+ "loss": 1.4651,
+ "theoretical_loss": 3.7305093647645435,
+ "tokens_seen": 861536256
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971193580742226,
+ "loss": 1.3565,
+ "theoretical_loss": 3.730481823024973,
+ "tokens_seen": 861601792
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005971033099297894,
+ "loss": 1.4322,
+ "theoretical_loss": 3.730454283966753,
+ "tokens_seen": 861667328
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000597087261785356,
+ "loss": 1.302,
+ "theoretical_loss": 3.7304267475894184,
+ "tokens_seen": 861732864
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 292159,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5102205276489258,
+ "objective/train/theoretical_loss": 3.730399213892504,
+ "objective/train/tokens_used": 882258400,
+ "theoretical_loss": 3.730399213892504,
+ "tokens_seen": 861798400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005970712136409228,
+ "loss": 1.4731,
+ "theoretical_loss": 3.730399213892504,
+ "tokens_seen": 861798400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005970551654964894,
+ "loss": 1.4135,
+ "theoretical_loss": 3.730371682875545,
+ "tokens_seen": 861863936
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005970391173520562,
+ "loss": 1.4878,
+ "theoretical_loss": 3.730344154538077,
+ "tokens_seen": 861929472
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005970230692076228,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7303166288796357,
+ "tokens_seen": 861995008
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005970070210631896,
+ "loss": 1.5092,
+ "theoretical_loss": 3.7302891058997565,
+ "tokens_seen": 862060544
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005969909729187563,
+ "loss": 1.4024,
+ "theoretical_loss": 3.730261585597976,
+ "tokens_seen": 862126080
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596974924774323,
+ "loss": 1.399,
+ "theoretical_loss": 3.7302340679738286,
+ "tokens_seen": 862191616
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005969588766298897,
+ "loss": 1.445,
+ "theoretical_loss": 3.730206553026851,
+ "tokens_seen": 862257152
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005969428284854564,
+ "loss": 1.5203,
+ "theoretical_loss": 3.730179040756579,
+ "tokens_seen": 862322688
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005969267803410231,
+ "loss": 1.322,
+ "theoretical_loss": 3.7301515311625497,
+ "tokens_seen": 862388224
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005969107321965898,
+ "loss": 1.3981,
+ "theoretical_loss": 3.730124024244298,
+ "tokens_seen": 862453760
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005968946840521565,
+ "loss": 1.4271,
+ "theoretical_loss": 3.730096520001362,
+ "tokens_seen": 862519296
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005968786359077232,
+ "loss": 1.477,
+ "theoretical_loss": 3.730069018433277,
+ "tokens_seen": 862584832
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.00059686258776329,
+ "loss": 1.3415,
+ "theoretical_loss": 3.73004151953958,
+ "tokens_seen": 862650368
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005968465396188566,
+ "loss": 1.508,
+ "theoretical_loss": 3.730014023319808,
+ "tokens_seen": 862715904
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005968304914744233,
+ "loss": 1.3229,
+ "theoretical_loss": 3.7299865297734973,
+ "tokens_seen": 862781440
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.00059681444332999,
+ "loss": 1.4359,
+ "theoretical_loss": 3.7299590389001858,
+ "tokens_seen": 862846976
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967983951855568,
+ "loss": 1.3911,
+ "theoretical_loss": 3.72993155069941,
+ "tokens_seen": 862912512
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967823470411234,
+ "loss": 1.581,
+ "theoretical_loss": 3.7299040651707074,
+ "tokens_seen": 862978048
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967662988966902,
+ "loss": 1.3578,
+ "theoretical_loss": 3.7298765823136155,
+ "tokens_seen": 863043584
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967502507522568,
+ "loss": 1.3588,
+ "theoretical_loss": 3.7298491021276714,
+ "tokens_seen": 863109120
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967342026078236,
+ "loss": 1.3742,
+ "theoretical_loss": 3.7298216246124127,
+ "tokens_seen": 863174656
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005967181544633902,
+ "loss": 1.4532,
+ "theoretical_loss": 3.729794149767377,
+ "tokens_seen": 863240192
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596702106318957,
+ "loss": 1.32,
+ "theoretical_loss": 3.729766677592103,
+ "tokens_seen": 863305728
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005966860581745236,
+ "loss": 1.3781,
+ "theoretical_loss": 3.7297392080861282,
+ "tokens_seen": 863371264
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005966700100300904,
+ "loss": 1.3734,
+ "theoretical_loss": 3.7297117412489897,
+ "tokens_seen": 863436800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596653961885657,
+ "loss": 1.3839,
+ "theoretical_loss": 3.729684277080227,
+ "tokens_seen": 863502336
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005966379137412238,
+ "loss": 1.4134,
+ "theoretical_loss": 3.7296568155793777,
+ "tokens_seen": 863567872
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005966218655967904,
+ "loss": 1.3296,
+ "theoretical_loss": 3.7296293567459804,
+ "tokens_seen": 863633408
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005966058174523572,
+ "loss": 1.4878,
+ "theoretical_loss": 3.7296019005795733,
+ "tokens_seen": 863698944
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005965897693079238,
+ "loss": 1.4308,
+ "theoretical_loss": 3.7295744470796954,
+ "tokens_seen": 863764480
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005965737211634906,
+ "loss": 1.4164,
+ "theoretical_loss": 3.729546996245885,
+ "tokens_seen": 863830016
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005965576730190572,
+ "loss": 1.395,
+ "theoretical_loss": 3.7295195480776817,
+ "tokens_seen": 863895552
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596541624874624,
+ "loss": 1.3631,
+ "theoretical_loss": 3.729492102574624,
+ "tokens_seen": 863961088
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005965255767301906,
+ "loss": 1.4525,
+ "theoretical_loss": 3.7294646597362506,
+ "tokens_seen": 864026624
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005965095285857574,
+ "loss": 1.4787,
+ "theoretical_loss": 3.729437219562101,
+ "tokens_seen": 864092160
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596493480441324,
+ "loss": 1.4133,
+ "theoretical_loss": 3.7294097820517154,
+ "tokens_seen": 864157696
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005964774322968908,
+ "loss": 1.5079,
+ "theoretical_loss": 3.7293823472046315,
+ "tokens_seen": 864223232
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005964613841524574,
+ "loss": 1.3559,
+ "theoretical_loss": 3.729354915020391,
+ "tokens_seen": 864288768
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005964453360080242,
+ "loss": 1.5025,
+ "theoretical_loss": 3.729327485498531,
+ "tokens_seen": 864354304
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005964292878635908,
+ "loss": 1.3629,
+ "theoretical_loss": 3.7293000586385934,
+ "tokens_seen": 864419840
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005964132397191576,
+ "loss": 1.4256,
+ "theoretical_loss": 3.7292726344401173,
+ "tokens_seen": 864485376
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005963971915747242,
+ "loss": 1.5274,
+ "theoretical_loss": 3.7292452129026428,
+ "tokens_seen": 864550912
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596381143430291,
+ "loss": 1.4937,
+ "theoretical_loss": 3.7292177940257094,
+ "tokens_seen": 864616448
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005963650952858576,
+ "loss": 1.3922,
+ "theoretical_loss": 3.7291903778088584,
+ "tokens_seen": 864681984
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005963490471414244,
+ "loss": 1.4654,
+ "theoretical_loss": 3.7291629642516293,
+ "tokens_seen": 864747520
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596332998996991,
+ "loss": 1.3205,
+ "theoretical_loss": 3.7291355533535633,
+ "tokens_seen": 864813056
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005963169508525578,
+ "loss": 1.3455,
+ "theoretical_loss": 3.7291081451142003,
+ "tokens_seen": 864878592
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005963009027081244,
+ "loss": 1.5219,
+ "theoretical_loss": 3.729080739533081,
+ "tokens_seen": 864944128
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962848545636912,
+ "loss": 1.3509,
+ "theoretical_loss": 3.7290533366097467,
+ "tokens_seen": 865009664
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 293335,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5877540111541748,
+ "objective/train/theoretical_loss": 3.7290259363437377,
+ "objective/train/tokens_used": 885535200,
+ "theoretical_loss": 3.7290259363437377,
+ "tokens_seen": 865075200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962688064192578,
+ "loss": 1.4697,
+ "theoretical_loss": 3.7290259363437377,
+ "tokens_seen": 865075200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962527582748246,
+ "loss": 1.4238,
+ "theoretical_loss": 3.7289985387345963,
+ "tokens_seen": 865140736
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962367101303912,
+ "loss": 1.4721,
+ "theoretical_loss": 3.7289711437818616,
+ "tokens_seen": 865206272
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962206619859579,
+ "loss": 1.4989,
+ "theoretical_loss": 3.7289437514850765,
+ "tokens_seen": 865271808
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005962046138415246,
+ "loss": 1.3594,
+ "theoretical_loss": 3.728916361843782,
+ "tokens_seen": 865337344
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005961885656970913,
+ "loss": 1.3321,
+ "theoretical_loss": 3.728888974857519,
+ "tokens_seen": 865402880
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000596172517552658,
+ "loss": 1.3389,
+ "theoretical_loss": 3.7288615905258298,
+ "tokens_seen": 865468416
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005961564694082247,
+ "loss": 1.4009,
+ "theoretical_loss": 3.728834208848256,
+ "tokens_seen": 865533952
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005961404212637914,
+ "loss": 1.439,
+ "theoretical_loss": 3.7288068298243386,
+ "tokens_seen": 865599488
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005961243731193581,
+ "loss": 1.3532,
+ "theoretical_loss": 3.7287794534536207,
+ "tokens_seen": 865665024
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005961083249749248,
+ "loss": 1.3144,
+ "theoretical_loss": 3.728752079735644,
+ "tokens_seen": 865730560
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960922768304915,
+ "loss": 1.448,
+ "theoretical_loss": 3.728724708669951,
+ "tokens_seen": 865796096
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960762286860581,
+ "loss": 1.3527,
+ "theoretical_loss": 3.7286973402560823,
+ "tokens_seen": 865861632
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960601805416249,
+ "loss": 1.3912,
+ "theoretical_loss": 3.7286699744935827,
+ "tokens_seen": 865927168
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960441323971915,
+ "loss": 1.4209,
+ "theoretical_loss": 3.728642611381993,
+ "tokens_seen": 865992704
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960280842527583,
+ "loss": 1.3568,
+ "theoretical_loss": 3.728615250920856,
+ "tokens_seen": 866058240
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005960120361083249,
+ "loss": 1.4931,
+ "theoretical_loss": 3.728587893109715,
+ "tokens_seen": 866123776
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959959879638917,
+ "loss": 1.3623,
+ "theoretical_loss": 3.7285605379481126,
+ "tokens_seen": 866189312
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959799398194583,
+ "loss": 1.3508,
+ "theoretical_loss": 3.728533185435592,
+ "tokens_seen": 866254848
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959638916750251,
+ "loss": 1.3635,
+ "theoretical_loss": 3.7285058355716965,
+ "tokens_seen": 866320384
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959478435305917,
+ "loss": 1.4739,
+ "theoretical_loss": 3.728478488355968,
+ "tokens_seen": 866385920
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959317953861585,
+ "loss": 1.4582,
+ "theoretical_loss": 3.728451143787951,
+ "tokens_seen": 866451456
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005959157472417251,
+ "loss": 1.4172,
+ "theoretical_loss": 3.7284238018671885,
+ "tokens_seen": 866516992
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958996990972919,
+ "loss": 1.2999,
+ "theoretical_loss": 3.728396462593224,
+ "tokens_seen": 866582528
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958836509528585,
+ "loss": 1.4225,
+ "theoretical_loss": 3.7283691259656018,
+ "tokens_seen": 866648064
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958676028084253,
+ "loss": 1.3977,
+ "theoretical_loss": 3.7283417919838646,
+ "tokens_seen": 866713600
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958515546639919,
+ "loss": 1.3359,
+ "theoretical_loss": 3.7283144606475576,
+ "tokens_seen": 866779136
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958355065195587,
+ "loss": 1.5323,
+ "theoretical_loss": 3.7282871319562227,
+ "tokens_seen": 866844672
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958194583751253,
+ "loss": 1.304,
+ "theoretical_loss": 3.7282598059094063,
+ "tokens_seen": 866910208
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005958034102306921,
+ "loss": 1.3826,
+ "theoretical_loss": 3.728232482506651,
+ "tokens_seen": 866975744
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957873620862588,
+ "loss": 1.361,
+ "theoretical_loss": 3.7282051617475016,
+ "tokens_seen": 867041280
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957713139418255,
+ "loss": 1.4337,
+ "theoretical_loss": 3.728177843631503,
+ "tokens_seen": 867106816
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957552657973922,
+ "loss": 1.413,
+ "theoretical_loss": 3.7281505281581993,
+ "tokens_seen": 867172352
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957392176529589,
+ "loss": 1.4236,
+ "theoretical_loss": 3.7281232153271353,
+ "tokens_seen": 867237888
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957231695085256,
+ "loss": 1.4413,
+ "theoretical_loss": 3.728095905137856,
+ "tokens_seen": 867303424
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005957071213640923,
+ "loss": 1.4082,
+ "theoretical_loss": 3.728068597589905,
+ "tokens_seen": 867368960
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000595691073219659,
+ "loss": 1.4053,
+ "theoretical_loss": 3.7280412926828292,
+ "tokens_seen": 867434496
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005956750250752257,
+ "loss": 1.2886,
+ "theoretical_loss": 3.7280139904161724,
+ "tokens_seen": 867500032
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005956589769307924,
+ "loss": 1.3978,
+ "theoretical_loss": 3.7279866907894803,
+ "tokens_seen": 867565568
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005956429287863591,
+ "loss": 1.3115,
+ "theoretical_loss": 3.7279593938022986,
+ "tokens_seen": 867631104
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005956268806419258,
+ "loss": 1.4112,
+ "theoretical_loss": 3.7279320994541716,
+ "tokens_seen": 867696640
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005956108324974925,
+ "loss": 1.37,
+ "theoretical_loss": 3.727904807744646,
+ "tokens_seen": 867762176
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005955947843530592,
+ "loss": 1.4193,
+ "theoretical_loss": 3.7278775186732664,
+ "tokens_seen": 867827712
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005955787362086259,
+ "loss": 1.4642,
+ "theoretical_loss": 3.7278502322395797,
+ "tokens_seen": 867893248
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005955626880641926,
+ "loss": 1.4219,
+ "theoretical_loss": 3.727822948443132,
+ "tokens_seen": 867958784
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005955466399197593,
+ "loss": 1.4288,
+ "theoretical_loss": 3.727795667283468,
+ "tokens_seen": 868024320
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000595530591775326,
+ "loss": 1.4387,
+ "theoretical_loss": 3.7277683887601345,
+ "tokens_seen": 868089856
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005955145436308927,
+ "loss": 1.4614,
+ "theoretical_loss": 3.7277411128726774,
+ "tokens_seen": 868155392
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954984954864594,
+ "loss": 1.3479,
+ "theoretical_loss": 3.7277138396206437,
+ "tokens_seen": 868220928
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954824473420261,
+ "loss": 1.3631,
+ "theoretical_loss": 3.7276865690035796,
+ "tokens_seen": 868286464
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 294369,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3449276685714722,
+ "objective/train/theoretical_loss": 3.7276593010210313,
+ "objective/train/tokens_used": 888812000,
+ "theoretical_loss": 3.7276593010210313,
+ "tokens_seen": 868352000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954663991975928,
+ "loss": 1.3866,
+ "theoretical_loss": 3.7276593010210313,
+ "tokens_seen": 868352000
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954503510531595,
+ "loss": 1.4186,
+ "theoretical_loss": 3.727632035672546,
+ "tokens_seen": 868417536
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954343029087263,
+ "loss": 1.4225,
+ "theoretical_loss": 3.7276047729576707,
+ "tokens_seen": 868483072
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954182547642929,
+ "loss": 1.4115,
+ "theoretical_loss": 3.727577512875951,
+ "tokens_seen": 868548608
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005954022066198597,
+ "loss": 1.5025,
+ "theoretical_loss": 3.7275502554269355,
+ "tokens_seen": 868614144
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005953861584754263,
+ "loss": 1.2588,
+ "theoretical_loss": 3.7275230006101703,
+ "tokens_seen": 868679680
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000595370110330993,
+ "loss": 1.3555,
+ "theoretical_loss": 3.7274957484252034,
+ "tokens_seen": 868745216
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005953540621865597,
+ "loss": 1.4491,
+ "theoretical_loss": 3.7274684988715814,
+ "tokens_seen": 868810752
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005953380140421265,
+ "loss": 1.3474,
+ "theoretical_loss": 3.7274412519488522,
+ "tokens_seen": 868876288
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005953219658976931,
+ "loss": 1.4697,
+ "theoretical_loss": 3.727414007656564,
+ "tokens_seen": 868941824
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005953059177532599,
+ "loss": 1.3817,
+ "theoretical_loss": 3.7273867659942628,
+ "tokens_seen": 869007360
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952898696088265,
+ "loss": 1.3573,
+ "theoretical_loss": 3.7273595269614974,
+ "tokens_seen": 869072896
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952738214643933,
+ "loss": 1.291,
+ "theoretical_loss": 3.727332290557816,
+ "tokens_seen": 869138432
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952577733199599,
+ "loss": 1.3947,
+ "theoretical_loss": 3.7273050567827664,
+ "tokens_seen": 869203968
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952417251755267,
+ "loss": 1.3047,
+ "theoretical_loss": 3.7272778256358974,
+ "tokens_seen": 869269504
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952256770310933,
+ "loss": 1.4723,
+ "theoretical_loss": 3.7272505971167558,
+ "tokens_seen": 869335040
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005952096288866601,
+ "loss": 1.3844,
+ "theoretical_loss": 3.7272233712248903,
+ "tokens_seen": 869400576
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951935807422267,
+ "loss": 1.322,
+ "theoretical_loss": 3.7271961479598508,
+ "tokens_seen": 869466112
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951775325977935,
+ "loss": 1.3923,
+ "theoretical_loss": 3.727168927321184,
+ "tokens_seen": 869531648
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951614844533601,
+ "loss": 1.3031,
+ "theoretical_loss": 3.72714170930844,
+ "tokens_seen": 869597184
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951454363089269,
+ "loss": 1.4718,
+ "theoretical_loss": 3.7271144939211673,
+ "tokens_seen": 869662720
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951293881644935,
+ "loss": 1.4241,
+ "theoretical_loss": 3.727087281158914,
+ "tokens_seen": 869728256
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005951133400200603,
+ "loss": 1.329,
+ "theoretical_loss": 3.7270600710212305,
+ "tokens_seen": 869793792
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950972918756269,
+ "loss": 1.3704,
+ "theoretical_loss": 3.7270328635076644,
+ "tokens_seen": 869859328
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950812437311937,
+ "loss": 1.4418,
+ "theoretical_loss": 3.7270056586177662,
+ "tokens_seen": 869924864
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950651955867603,
+ "loss": 1.4354,
+ "theoretical_loss": 3.726978456351085,
+ "tokens_seen": 869990400
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950491474423271,
+ "loss": 1.5375,
+ "theoretical_loss": 3.72695125670717,
+ "tokens_seen": 870055936
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950330992978937,
+ "loss": 1.5777,
+ "theoretical_loss": 3.7269240596855706,
+ "tokens_seen": 870121472
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950170511534605,
+ "loss": 1.3397,
+ "theoretical_loss": 3.7268968652858367,
+ "tokens_seen": 870187008
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005950010030090271,
+ "loss": 1.431,
+ "theoretical_loss": 3.726869673507519,
+ "tokens_seen": 870252544
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949849548645939,
+ "loss": 1.3722,
+ "theoretical_loss": 3.726842484350166,
+ "tokens_seen": 870318080
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949689067201605,
+ "loss": 1.3484,
+ "theoretical_loss": 3.726815297813329,
+ "tokens_seen": 870383616
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949528585757273,
+ "loss": 1.4931,
+ "theoretical_loss": 3.7267881138965566,
+ "tokens_seen": 870449152
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949368104312939,
+ "loss": 1.4237,
+ "theoretical_loss": 3.726760932599401,
+ "tokens_seen": 870514688
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949207622868607,
+ "loss": 1.443,
+ "theoretical_loss": 3.726733753921411,
+ "tokens_seen": 870580224
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005949047141424273,
+ "loss": 1.3554,
+ "theoretical_loss": 3.7267065778621378,
+ "tokens_seen": 870645760
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948886659979941,
+ "loss": 1.4397,
+ "theoretical_loss": 3.726679404421131,
+ "tokens_seen": 870711296
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948726178535607,
+ "loss": 1.3659,
+ "theoretical_loss": 3.7266522335979433,
+ "tokens_seen": 870776832
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948565697091275,
+ "loss": 1.5223,
+ "theoretical_loss": 3.726625065392124,
+ "tokens_seen": 870842368
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948405215646941,
+ "loss": 1.5717,
+ "theoretical_loss": 3.726597899803224,
+ "tokens_seen": 870907904
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948244734202608,
+ "loss": 1.3426,
+ "theoretical_loss": 3.726570736830795,
+ "tokens_seen": 870973440
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005948084252758275,
+ "loss": 1.3967,
+ "theoretical_loss": 3.7265435764743877,
+ "tokens_seen": 871038976
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005947923771313942,
+ "loss": 1.3258,
+ "theoretical_loss": 3.7265164187335538,
+ "tokens_seen": 871104512
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005947763289869609,
+ "loss": 1.3116,
+ "theoretical_loss": 3.7264892636078444,
+ "tokens_seen": 871170048
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005947602808425276,
+ "loss": 1.45,
+ "theoretical_loss": 3.7264621110968106,
+ "tokens_seen": 871235584
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005947442326980943,
+ "loss": 1.4683,
+ "theoretical_loss": 3.726434961200005,
+ "tokens_seen": 871301120
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594728184553661,
+ "loss": 1.4142,
+ "theoretical_loss": 3.7264078139169783,
+ "tokens_seen": 871366656
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005947121364092277,
+ "loss": 1.3235,
+ "theoretical_loss": 3.7263806692472823,
+ "tokens_seen": 871432192
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005946960882647944,
+ "loss": 1.3342,
+ "theoretical_loss": 3.7263535271904695,
+ "tokens_seen": 871497728
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594680040120361,
+ "loss": 1.3977,
+ "theoretical_loss": 3.726326387746092,
+ "tokens_seen": 871563264
+ },
+ {
+ "epoch": 0.26,
+ "objective/train/docs_used": 295704,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3763213157653809,
+ "objective/train/theoretical_loss": 3.7262992509137014,
+ "objective/train/tokens_used": 892088800,
+ "theoretical_loss": 3.7262992509137014,
+ "tokens_seen": 871628800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005946639919759278,
+ "loss": 1.3689,
+ "theoretical_loss": 3.7262992509137014,
+ "tokens_seen": 871628800
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005946479438314944,
+ "loss": 1.4538,
+ "theoretical_loss": 3.72627211669285,
+ "tokens_seen": 871694336
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005946318956870612,
+ "loss": 1.3805,
+ "theoretical_loss": 3.7262449850830905,
+ "tokens_seen": 871759872
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005946158475426278,
+ "loss": 1.3534,
+ "theoretical_loss": 3.7262178560839754,
+ "tokens_seen": 871825408
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945997993981946,
+ "loss": 1.4327,
+ "theoretical_loss": 3.726190729695057,
+ "tokens_seen": 871890944
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945837512537613,
+ "loss": 1.4202,
+ "theoretical_loss": 3.7261636059158887,
+ "tokens_seen": 871956480
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594567703109328,
+ "loss": 1.4297,
+ "theoretical_loss": 3.726136484746022,
+ "tokens_seen": 872022016
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945516549648947,
+ "loss": 1.4908,
+ "theoretical_loss": 3.7261093661850104,
+ "tokens_seen": 872087552
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945356068204614,
+ "loss": 1.4314,
+ "theoretical_loss": 3.7260822502324076,
+ "tokens_seen": 872153088
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945195586760281,
+ "loss": 1.4478,
+ "theoretical_loss": 3.726055136887766,
+ "tokens_seen": 872218624
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005945035105315948,
+ "loss": 1.5178,
+ "theoretical_loss": 3.7260280261506393,
+ "tokens_seen": 872284160
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005944874623871615,
+ "loss": 1.3651,
+ "theoretical_loss": 3.7260009180205804,
+ "tokens_seen": 872349696
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005944714142427282,
+ "loss": 1.4939,
+ "theoretical_loss": 3.7259738124971427,
+ "tokens_seen": 872415232
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005944553660982949,
+ "loss": 1.4669,
+ "theoretical_loss": 3.7259467095798806,
+ "tokens_seen": 872480768
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005944393179538616,
+ "loss": 1.365,
+ "theoretical_loss": 3.7259196092683466,
+ "tokens_seen": 872546304
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005944232698094283,
+ "loss": 1.3216,
+ "theoretical_loss": 3.7258925115620953,
+ "tokens_seen": 872611840
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594407221664995,
+ "loss": 1.307,
+ "theoretical_loss": 3.725865416460681,
+ "tokens_seen": 872677376
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943911735205617,
+ "loss": 1.3002,
+ "theoretical_loss": 3.7258383239636563,
+ "tokens_seen": 872742912
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943751253761284,
+ "loss": 1.299,
+ "theoretical_loss": 3.7258112340705765,
+ "tokens_seen": 872808448
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943590772316951,
+ "loss": 1.3581,
+ "theoretical_loss": 3.7257841467809953,
+ "tokens_seen": 872873984
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943430290872618,
+ "loss": 1.2732,
+ "theoretical_loss": 3.725757062094467,
+ "tokens_seen": 872939520
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943269809428285,
+ "loss": 1.375,
+ "theoretical_loss": 3.7257299800105472,
+ "tokens_seen": 873005056
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005943109327983952,
+ "loss": 1.4402,
+ "theoretical_loss": 3.7257029005287894,
+ "tokens_seen": 873070592
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005942948846539619,
+ "loss": 1.3421,
+ "theoretical_loss": 3.7256758236487477,
+ "tokens_seen": 873136128
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005942788365095286,
+ "loss": 1.4131,
+ "theoretical_loss": 3.725648749369978,
+ "tokens_seen": 873201664
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005942627883650953,
+ "loss": 1.4883,
+ "theoretical_loss": 3.725621677692035,
+ "tokens_seen": 873267200
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594246740220662,
+ "loss": 1.4669,
+ "theoretical_loss": 3.725594608614473,
+ "tokens_seen": 873332736
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005942306920762287,
+ "loss": 1.4359,
+ "theoretical_loss": 3.725567542136848,
+ "tokens_seen": 873398272
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005942146439317954,
+ "loss": 1.4512,
+ "theoretical_loss": 3.725540478258715,
+ "tokens_seen": 873463808
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941985957873621,
+ "loss": 1.3902,
+ "theoretical_loss": 3.725513416979629,
+ "tokens_seen": 873529344
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941825476429288,
+ "loss": 1.4411,
+ "theoretical_loss": 3.725486358299145,
+ "tokens_seen": 873594880
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941664994984955,
+ "loss": 1.3979,
+ "theoretical_loss": 3.7254593022168203,
+ "tokens_seen": 873660416
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941504513540622,
+ "loss": 1.3819,
+ "theoretical_loss": 3.7254322487322087,
+ "tokens_seen": 873725952
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594134403209629,
+ "loss": 1.4271,
+ "theoretical_loss": 3.7254051978448666,
+ "tokens_seen": 873791488
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941183550651956,
+ "loss": 1.271,
+ "theoretical_loss": 3.7253781495543503,
+ "tokens_seen": 873857024
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005941023069207623,
+ "loss": 1.3593,
+ "theoretical_loss": 3.7253511038602154,
+ "tokens_seen": 873922560
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000594086258776329,
+ "loss": 1.4129,
+ "theoretical_loss": 3.7253240607620177,
+ "tokens_seen": 873988096
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005940702106318958,
+ "loss": 1.3292,
+ "theoretical_loss": 3.7252970202593136,
+ "tokens_seen": 874053632
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005940541624874624,
+ "loss": 1.2952,
+ "theoretical_loss": 3.7252699823516604,
+ "tokens_seen": 874119168
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005940381143430292,
+ "loss": 1.4277,
+ "theoretical_loss": 3.7252429470386135,
+ "tokens_seen": 874184704
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005940220661985958,
+ "loss": 1.3726,
+ "theoretical_loss": 3.7252159143197288,
+ "tokens_seen": 874250240
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005940060180541626,
+ "loss": 1.3579,
+ "theoretical_loss": 3.7251888841945644,
+ "tokens_seen": 874315776
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.0005939899699097292,
+ "loss": 1.4098,
+ "theoretical_loss": 3.7251618566626767,
+ "tokens_seen": 874381312
+ },
+ {
+ "epoch": 0.26,
+ "learning_rate": 0.000593973921765296,
+ "loss": 1.2776,
+ "theoretical_loss": 3.7251348317236217,
+ "tokens_seen": 874446848
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005939578736208626,
+ "loss": 1.5063,
+ "theoretical_loss": 3.725107809376957,
+ "tokens_seen": 874512384
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005939418254764294,
+ "loss": 1.4131,
+ "theoretical_loss": 3.7250807896222398,
+ "tokens_seen": 874577920
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593925777331996,
+ "loss": 1.3592,
+ "theoretical_loss": 3.725053772459027,
+ "tokens_seen": 874643456
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005939097291875628,
+ "loss": 1.4288,
+ "theoretical_loss": 3.7250267578868757,
+ "tokens_seen": 874708992
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005938936810431294,
+ "loss": 1.5116,
+ "theoretical_loss": 3.724999745905344,
+ "tokens_seen": 874774528
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005938776328986962,
+ "loss": 1.4114,
+ "theoretical_loss": 3.724972736513989,
+ "tokens_seen": 874840064
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 296615,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4356341361999512,
+ "objective/train/theoretical_loss": 3.7249457297123687,
+ "objective/train/tokens_used": 895365600,
+ "theoretical_loss": 3.7249457297123687,
+ "tokens_seen": 874905600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005938615847542628,
+ "loss": 1.3512,
+ "theoretical_loss": 3.7249457297123687,
+ "tokens_seen": 874905600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005938455366098296,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7249187255000393,
+ "tokens_seen": 874971136
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005938294884653962,
+ "loss": 1.4029,
+ "theoretical_loss": 3.724891723876561,
+ "tokens_seen": 875036672
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593813440320963,
+ "loss": 1.4275,
+ "theoretical_loss": 3.7248647248414897,
+ "tokens_seen": 875102208
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937973921765296,
+ "loss": 1.2328,
+ "theoretical_loss": 3.7248377283943848,
+ "tokens_seen": 875167744
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937813440320964,
+ "loss": 1.3946,
+ "theoretical_loss": 3.7248107345348034,
+ "tokens_seen": 875233280
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593765295887663,
+ "loss": 1.3459,
+ "theoretical_loss": 3.7247837432623054,
+ "tokens_seen": 875298816
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937492477432298,
+ "loss": 1.501,
+ "theoretical_loss": 3.724756754576447,
+ "tokens_seen": 875364352
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937331995987964,
+ "loss": 1.4329,
+ "theoretical_loss": 3.7247297684767884,
+ "tokens_seen": 875429888
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937171514543632,
+ "loss": 1.4151,
+ "theoretical_loss": 3.7247027849628873,
+ "tokens_seen": 875495424
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005937011033099298,
+ "loss": 1.4494,
+ "theoretical_loss": 3.7246758040343027,
+ "tokens_seen": 875560960
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005936850551654966,
+ "loss": 1.211,
+ "theoretical_loss": 3.724648825690594,
+ "tokens_seen": 875626496
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005936690070210632,
+ "loss": 1.489,
+ "theoretical_loss": 3.724621849931319,
+ "tokens_seen": 875692032
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00059365295887663,
+ "loss": 1.4049,
+ "theoretical_loss": 3.7245948767560373,
+ "tokens_seen": 875757568
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005936369107321966,
+ "loss": 1.3986,
+ "theoretical_loss": 3.724567906164308,
+ "tokens_seen": 875823104
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005936208625877634,
+ "loss": 1.4051,
+ "theoretical_loss": 3.72454093815569,
+ "tokens_seen": 875888640
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00059360481444333,
+ "loss": 1.4094,
+ "theoretical_loss": 3.724513972729744,
+ "tokens_seen": 875954176
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935887662988968,
+ "loss": 1.3648,
+ "theoretical_loss": 3.724487009886028,
+ "tokens_seen": 876019712
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935727181544634,
+ "loss": 1.3693,
+ "theoretical_loss": 3.724460049624102,
+ "tokens_seen": 876085248
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935566700100302,
+ "loss": 1.3855,
+ "theoretical_loss": 3.7244330919435256,
+ "tokens_seen": 876150784
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935406218655968,
+ "loss": 1.3825,
+ "theoretical_loss": 3.724406136843859,
+ "tokens_seen": 876216320
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935245737211636,
+ "loss": 1.4952,
+ "theoretical_loss": 3.7243791843246616,
+ "tokens_seen": 876281856
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005935085255767302,
+ "loss": 1.4827,
+ "theoretical_loss": 3.724352234385494,
+ "tokens_seen": 876347392
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593492477432297,
+ "loss": 1.3579,
+ "theoretical_loss": 3.7243252870259154,
+ "tokens_seen": 876412928
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005934764292878636,
+ "loss": 1.3541,
+ "theoretical_loss": 3.7242983422454867,
+ "tokens_seen": 876478464
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005934603811434304,
+ "loss": 1.4119,
+ "theoretical_loss": 3.724271400043768,
+ "tokens_seen": 876544000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593444332998997,
+ "loss": 1.3913,
+ "theoretical_loss": 3.72424446042032,
+ "tokens_seen": 876609536
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005934282848545637,
+ "loss": 1.4509,
+ "theoretical_loss": 3.724217523374703,
+ "tokens_seen": 876675072
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005934122367101304,
+ "loss": 1.4332,
+ "theoretical_loss": 3.7241905889064784,
+ "tokens_seen": 876740608
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933961885656971,
+ "loss": 1.4808,
+ "theoretical_loss": 3.7241636570152052,
+ "tokens_seen": 876806144
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933801404212638,
+ "loss": 1.4096,
+ "theoretical_loss": 3.724136727700446,
+ "tokens_seen": 876871680
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933640922768305,
+ "loss": 1.3544,
+ "theoretical_loss": 3.7241098009617604,
+ "tokens_seen": 876937216
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933480441323972,
+ "loss": 1.3773,
+ "theoretical_loss": 3.724082876798711,
+ "tokens_seen": 877002752
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933319959879639,
+ "loss": 1.4223,
+ "theoretical_loss": 3.724055955210858,
+ "tokens_seen": 877068288
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933159478435306,
+ "loss": 1.3957,
+ "theoretical_loss": 3.7240290361977624,
+ "tokens_seen": 877133824
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005933159478435306,
+ "loss": 1.3962,
+ "theoretical_loss": 3.7240021197589863,
+ "tokens_seen": 877199360
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932998996990973,
+ "loss": 1.4461,
+ "theoretical_loss": 3.723975205894091,
+ "tokens_seen": 877264896
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593283851554664,
+ "loss": 1.4704,
+ "theoretical_loss": 3.7239482946026383,
+ "tokens_seen": 877330432
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932678034102307,
+ "loss": 1.3398,
+ "theoretical_loss": 3.7239213858841893,
+ "tokens_seen": 877395968
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932517552657974,
+ "loss": 1.3765,
+ "theoretical_loss": 3.723894479738307,
+ "tokens_seen": 877461504
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932357071213641,
+ "loss": 1.4135,
+ "theoretical_loss": 3.723867576164552,
+ "tokens_seen": 877527040
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932196589769308,
+ "loss": 1.2494,
+ "theoretical_loss": 3.723840675162487,
+ "tokens_seen": 877592576
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005932036108324975,
+ "loss": 1.3955,
+ "theoretical_loss": 3.7238137767316744,
+ "tokens_seen": 877658112
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005931875626880642,
+ "loss": 1.382,
+ "theoretical_loss": 3.723786880871676,
+ "tokens_seen": 877723648
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005931715145436309,
+ "loss": 1.3312,
+ "theoretical_loss": 3.7237599875820546,
+ "tokens_seen": 877789184
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005931554663991976,
+ "loss": 1.3627,
+ "theoretical_loss": 3.723733096862372,
+ "tokens_seen": 877854720
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005931394182547643,
+ "loss": 1.308,
+ "theoretical_loss": 3.7237062087121915,
+ "tokens_seen": 877920256
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000593123370110331,
+ "loss": 1.3959,
+ "theoretical_loss": 3.7236793231310754,
+ "tokens_seen": 877985792
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005931073219658977,
+ "loss": 1.4604,
+ "theoretical_loss": 3.7236524401185864,
+ "tokens_seen": 878051328
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930912738214644,
+ "loss": 1.4636,
+ "theoretical_loss": 3.7236255596742884,
+ "tokens_seen": 878116864
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 297634,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4732657670974731,
+ "objective/train/theoretical_loss": 3.7235986817977427,
+ "objective/train/tokens_used": 898642400,
+ "theoretical_loss": 3.7235986817977427,
+ "tokens_seen": 878182400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930752256770311,
+ "loss": 1.4573,
+ "theoretical_loss": 3.7235986817977427,
+ "tokens_seen": 878182400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930591775325978,
+ "loss": 1.3969,
+ "theoretical_loss": 3.723571806488514,
+ "tokens_seen": 878247936
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930431293881645,
+ "loss": 1.2044,
+ "theoretical_loss": 3.7235449337461644,
+ "tokens_seen": 878313472
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930270812437312,
+ "loss": 1.4234,
+ "theoretical_loss": 3.723518063570258,
+ "tokens_seen": 878379008
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005930110330992979,
+ "loss": 1.3543,
+ "theoretical_loss": 3.7234911959603574,
+ "tokens_seen": 878444544
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005929949849548646,
+ "loss": 1.3475,
+ "theoretical_loss": 3.7234643309160274,
+ "tokens_seen": 878510080
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005929789368104313,
+ "loss": 1.4172,
+ "theoretical_loss": 3.7234374684368303,
+ "tokens_seen": 878575616
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000592962888665998,
+ "loss": 1.4079,
+ "theoretical_loss": 3.7234106085223306,
+ "tokens_seen": 878641152
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005929468405215647,
+ "loss": 1.3668,
+ "theoretical_loss": 3.7233837511720926,
+ "tokens_seen": 878706688
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005929307923771314,
+ "loss": 1.3957,
+ "theoretical_loss": 3.7233568963856793,
+ "tokens_seen": 878772224
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005929147442326981,
+ "loss": 1.3877,
+ "theoretical_loss": 3.7233300441626556,
+ "tokens_seen": 878837760
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928986960882648,
+ "loss": 1.4901,
+ "theoretical_loss": 3.7233031945025847,
+ "tokens_seen": 878903296
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928826479438315,
+ "loss": 1.3192,
+ "theoretical_loss": 3.723276347405032,
+ "tokens_seen": 878968832
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928665997993982,
+ "loss": 1.2972,
+ "theoretical_loss": 3.723249502869561,
+ "tokens_seen": 879034368
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928505516549649,
+ "loss": 1.3476,
+ "theoretical_loss": 3.723222660895737,
+ "tokens_seen": 879099904
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928345035105316,
+ "loss": 1.2606,
+ "theoretical_loss": 3.7231958214831242,
+ "tokens_seen": 879165440
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005928184553660983,
+ "loss": 1.4602,
+ "theoretical_loss": 3.723168984631287,
+ "tokens_seen": 879230976
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000592802407221665,
+ "loss": 1.3551,
+ "theoretical_loss": 3.7231421503397906,
+ "tokens_seen": 879296512
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927863590772317,
+ "loss": 1.3874,
+ "theoretical_loss": 3.7231153186082,
+ "tokens_seen": 879362048
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927703109327984,
+ "loss": 1.5372,
+ "theoretical_loss": 3.7230884894360807,
+ "tokens_seen": 879427584
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927542627883651,
+ "loss": 1.4817,
+ "theoretical_loss": 3.723061662822997,
+ "tokens_seen": 879493120
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927382146439318,
+ "loss": 1.4971,
+ "theoretical_loss": 3.7230348387685144,
+ "tokens_seen": 879558656
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927221664994985,
+ "loss": 1.4103,
+ "theoretical_loss": 3.723008017272198,
+ "tokens_seen": 879624192
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005927061183550653,
+ "loss": 1.2916,
+ "theoretical_loss": 3.7229811983336134,
+ "tokens_seen": 879689728
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005926900702106319,
+ "loss": 1.392,
+ "theoretical_loss": 3.7229543819523268,
+ "tokens_seen": 879755264
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005926740220661987,
+ "loss": 1.3366,
+ "theoretical_loss": 3.7229275681279033,
+ "tokens_seen": 879820800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005926579739217653,
+ "loss": 1.4538,
+ "theoretical_loss": 3.722900756859909,
+ "tokens_seen": 879886336
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000592641925777332,
+ "loss": 1.2782,
+ "theoretical_loss": 3.722873948147909,
+ "tokens_seen": 879951872
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005926258776328987,
+ "loss": 1.3477,
+ "theoretical_loss": 3.72284714199147,
+ "tokens_seen": 880017408
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005926098294884655,
+ "loss": 1.3581,
+ "theoretical_loss": 3.722820338390158,
+ "tokens_seen": 880082944
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925937813440321,
+ "loss": 1.4581,
+ "theoretical_loss": 3.7227935373435397,
+ "tokens_seen": 880148480
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925777331995989,
+ "loss": 1.251,
+ "theoretical_loss": 3.7227667388511807,
+ "tokens_seen": 880214016
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925616850551655,
+ "loss": 1.387,
+ "theoretical_loss": 3.7227399429126473,
+ "tokens_seen": 880279552
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925456369107323,
+ "loss": 1.382,
+ "theoretical_loss": 3.7227131495275065,
+ "tokens_seen": 880345088
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925295887662989,
+ "loss": 1.3781,
+ "theoretical_loss": 3.7226863586953245,
+ "tokens_seen": 880410624
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005925135406218657,
+ "loss": 1.5287,
+ "theoretical_loss": 3.7226595704156686,
+ "tokens_seen": 880476160
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924974924774323,
+ "loss": 1.3997,
+ "theoretical_loss": 3.7226327846881055,
+ "tokens_seen": 880541696
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924814443329991,
+ "loss": 1.357,
+ "theoretical_loss": 3.722606001512202,
+ "tokens_seen": 880607232
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924653961885657,
+ "loss": 1.3955,
+ "theoretical_loss": 3.722579220887525,
+ "tokens_seen": 880672768
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924493480441325,
+ "loss": 1.5141,
+ "theoretical_loss": 3.722552442813641,
+ "tokens_seen": 880738304
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924332998996991,
+ "loss": 1.3306,
+ "theoretical_loss": 3.7225256672901192,
+ "tokens_seen": 880803840
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924172517552659,
+ "loss": 1.4259,
+ "theoretical_loss": 3.7224988943165256,
+ "tokens_seen": 880869376
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005924012036108325,
+ "loss": 1.4479,
+ "theoretical_loss": 3.7224721238924277,
+ "tokens_seen": 880934912
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923851554663993,
+ "loss": 1.4694,
+ "theoretical_loss": 3.722445356017393,
+ "tokens_seen": 881000448
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923691073219659,
+ "loss": 1.3357,
+ "theoretical_loss": 3.7224185906909897,
+ "tokens_seen": 881065984
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923530591775327,
+ "loss": 1.3031,
+ "theoretical_loss": 3.7223918279127854,
+ "tokens_seen": 881131520
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923370110330993,
+ "loss": 1.4759,
+ "theoretical_loss": 3.7223650676823485,
+ "tokens_seen": 881197056
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923209628886661,
+ "loss": 1.435,
+ "theoretical_loss": 3.7223383099992455,
+ "tokens_seen": 881262592
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005923049147442327,
+ "loss": 1.4392,
+ "theoretical_loss": 3.7223115548630465,
+ "tokens_seen": 881328128
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922888665997995,
+ "loss": 1.3413,
+ "theoretical_loss": 3.722284802273318,
+ "tokens_seen": 881393664
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 298414,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.473338007926941,
+ "objective/train/theoretical_loss": 3.722258052229629,
+ "objective/train/tokens_used": 901919200,
+ "theoretical_loss": 3.722258052229629,
+ "tokens_seen": 881459200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922728184553661,
+ "loss": 1.4058,
+ "theoretical_loss": 3.722258052229629,
+ "tokens_seen": 881459200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922567703109329,
+ "loss": 1.3992,
+ "theoretical_loss": 3.7222313047315483,
+ "tokens_seen": 881524736
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922407221664995,
+ "loss": 1.5223,
+ "theoretical_loss": 3.722204559778644,
+ "tokens_seen": 881590272
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922246740220663,
+ "loss": 1.2933,
+ "theoretical_loss": 3.7221778173704845,
+ "tokens_seen": 881655808
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005922086258776329,
+ "loss": 1.4027,
+ "theoretical_loss": 3.722151077506639,
+ "tokens_seen": 881721344
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921925777331997,
+ "loss": 1.3407,
+ "theoretical_loss": 3.722124340186676,
+ "tokens_seen": 881786880
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921765295887663,
+ "loss": 1.368,
+ "theoretical_loss": 3.7220976054101653,
+ "tokens_seen": 881852416
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921604814443331,
+ "loss": 1.4898,
+ "theoretical_loss": 3.7220708731766745,
+ "tokens_seen": 881917952
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921444332998997,
+ "loss": 1.395,
+ "theoretical_loss": 3.722044143485774,
+ "tokens_seen": 881983488
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921283851554665,
+ "loss": 1.3127,
+ "theoretical_loss": 3.7220174163370325,
+ "tokens_seen": 882049024
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005921123370110331,
+ "loss": 1.4073,
+ "theoretical_loss": 3.72199069173002,
+ "tokens_seen": 882114560
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920962888665999,
+ "loss": 1.2712,
+ "theoretical_loss": 3.721963969664305,
+ "tokens_seen": 882180096
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920802407221665,
+ "loss": 1.4562,
+ "theoretical_loss": 3.721937250139457,
+ "tokens_seen": 882245632
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920641925777333,
+ "loss": 1.4463,
+ "theoretical_loss": 3.721910533155047,
+ "tokens_seen": 882311168
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920481444332999,
+ "loss": 1.4729,
+ "theoretical_loss": 3.721883818710644,
+ "tokens_seen": 882376704
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920320962888666,
+ "loss": 1.4927,
+ "theoretical_loss": 3.7218571068058175,
+ "tokens_seen": 882442240
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005920160481444333,
+ "loss": 1.2466,
+ "theoretical_loss": 3.7218303974401383,
+ "tokens_seen": 882507776
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000592,
+ "loss": 1.4052,
+ "theoretical_loss": 3.7218036906131755,
+ "tokens_seen": 882573312
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919839518555667,
+ "loss": 1.3998,
+ "theoretical_loss": 3.7217769863245005,
+ "tokens_seen": 882638848
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919679037111334,
+ "loss": 1.4173,
+ "theoretical_loss": 3.721750284573683,
+ "tokens_seen": 882704384
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919518555667001,
+ "loss": 1.3993,
+ "theoretical_loss": 3.7217235853602935,
+ "tokens_seen": 882769920
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919358074222668,
+ "loss": 1.3656,
+ "theoretical_loss": 3.721696888683902,
+ "tokens_seen": 882835456
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919197592778335,
+ "loss": 1.4411,
+ "theoretical_loss": 3.72167019454408,
+ "tokens_seen": 882900992
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005919037111334002,
+ "loss": 1.4962,
+ "theoretical_loss": 3.7216435029403976,
+ "tokens_seen": 882966528
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005918876629889669,
+ "loss": 1.4093,
+ "theoretical_loss": 3.721616813872426,
+ "tokens_seen": 883032064
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005918716148445336,
+ "loss": 1.4233,
+ "theoretical_loss": 3.721590127339736,
+ "tokens_seen": 883097600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005918555667001003,
+ "loss": 1.3883,
+ "theoretical_loss": 3.721563443341899,
+ "tokens_seen": 883163136
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591839518555667,
+ "loss": 1.4037,
+ "theoretical_loss": 3.721536761878485,
+ "tokens_seen": 883228672
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005918234704112337,
+ "loss": 1.4487,
+ "theoretical_loss": 3.7215100829490666,
+ "tokens_seen": 883294208
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005918074222668004,
+ "loss": 1.2825,
+ "theoretical_loss": 3.7214834065532147,
+ "tokens_seen": 883359744
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917913741223671,
+ "loss": 1.4454,
+ "theoretical_loss": 3.7214567326905006,
+ "tokens_seen": 883425280
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917753259779338,
+ "loss": 1.4214,
+ "theoretical_loss": 3.721430061360496,
+ "tokens_seen": 883490816
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917592778335005,
+ "loss": 1.3491,
+ "theoretical_loss": 3.721403392562772,
+ "tokens_seen": 883556352
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917432296890672,
+ "loss": 1.4518,
+ "theoretical_loss": 3.7213767262969015,
+ "tokens_seen": 883621888
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917271815446339,
+ "loss": 1.389,
+ "theoretical_loss": 3.721350062562456,
+ "tokens_seen": 883687424
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005917111334002006,
+ "loss": 1.3153,
+ "theoretical_loss": 3.7213234013590064,
+ "tokens_seen": 883752960
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005916950852557673,
+ "loss": 1.3632,
+ "theoretical_loss": 3.7212967426861265,
+ "tokens_seen": 883818496
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591679037111334,
+ "loss": 1.3569,
+ "theoretical_loss": 3.721270086543387,
+ "tokens_seen": 883884032
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005916629889669007,
+ "loss": 1.2478,
+ "theoretical_loss": 3.721243432930361,
+ "tokens_seen": 883949568
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005916469408224674,
+ "loss": 1.3319,
+ "theoretical_loss": 3.721216781846621,
+ "tokens_seen": 884015104
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005916308926780341,
+ "loss": 1.2826,
+ "theoretical_loss": 3.721190133291739,
+ "tokens_seen": 884080640
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005916148445336008,
+ "loss": 1.393,
+ "theoretical_loss": 3.7211634872652875,
+ "tokens_seen": 884146176
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915987963891675,
+ "loss": 1.4237,
+ "theoretical_loss": 3.72113684376684,
+ "tokens_seen": 884211712
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915827482447342,
+ "loss": 1.4712,
+ "theoretical_loss": 3.721110202795969,
+ "tokens_seen": 884277248
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915667001003009,
+ "loss": 1.4201,
+ "theoretical_loss": 3.721083564352247,
+ "tokens_seen": 884342784
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915506519558676,
+ "loss": 1.4385,
+ "theoretical_loss": 3.7210569284352473,
+ "tokens_seen": 884408320
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915346038114343,
+ "loss": 1.3228,
+ "theoretical_loss": 3.7210302950445433,
+ "tokens_seen": 884473856
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591518555667001,
+ "loss": 1.4452,
+ "theoretical_loss": 3.7210036641797073,
+ "tokens_seen": 884539392
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005915025075225677,
+ "loss": 1.3413,
+ "theoretical_loss": 3.7209770358403143,
+ "tokens_seen": 884604928
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005914864593781344,
+ "loss": 1.3588,
+ "theoretical_loss": 3.720950410025936,
+ "tokens_seen": 884670464
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 299625,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3763837814331055,
+ "objective/train/theoretical_loss": 3.7209237867361473,
+ "objective/train/tokens_used": 905196000,
+ "theoretical_loss": 3.7209237867361473,
+ "tokens_seen": 884736000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005914704112337011,
+ "loss": 1.3689,
+ "theoretical_loss": 3.7209237867361473,
+ "tokens_seen": 884736000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005914543630892678,
+ "loss": 1.3451,
+ "theoretical_loss": 3.720897165970521,
+ "tokens_seen": 884801536
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005914383149448345,
+ "loss": 1.2868,
+ "theoretical_loss": 3.7208705477286306,
+ "tokens_seen": 884867072
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005914222668004012,
+ "loss": 1.3519,
+ "theoretical_loss": 3.720843932010051,
+ "tokens_seen": 884932608
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591406218655968,
+ "loss": 1.3974,
+ "theoretical_loss": 3.720817318814355,
+ "tokens_seen": 884998144
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005913901705115346,
+ "loss": 1.3589,
+ "theoretical_loss": 3.720790708141118,
+ "tokens_seen": 885063680
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005913741223671013,
+ "loss": 1.3256,
+ "theoretical_loss": 3.7207640999899128,
+ "tokens_seen": 885129216
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591358074222668,
+ "loss": 1.3769,
+ "theoretical_loss": 3.720737494360314,
+ "tokens_seen": 885194752
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005913420260782348,
+ "loss": 1.3317,
+ "theoretical_loss": 3.720710891251897,
+ "tokens_seen": 885260288
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005913259779338014,
+ "loss": 1.3677,
+ "theoretical_loss": 3.7206842906642352,
+ "tokens_seen": 885325824
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005913099297893682,
+ "loss": 1.3947,
+ "theoretical_loss": 3.7206576925969035,
+ "tokens_seen": 885391360
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005912938816449348,
+ "loss": 1.4569,
+ "theoretical_loss": 3.720631097049477,
+ "tokens_seen": 885456896
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005912778335005016,
+ "loss": 1.2061,
+ "theoretical_loss": 3.720604504021529,
+ "tokens_seen": 885522432
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005912617853560682,
+ "loss": 1.4317,
+ "theoretical_loss": 3.7205779135126367,
+ "tokens_seen": 885587968
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591245737211635,
+ "loss": 1.3402,
+ "theoretical_loss": 3.720551325522373,
+ "tokens_seen": 885653504
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005912296890672016,
+ "loss": 1.3553,
+ "theoretical_loss": 3.720524740050314,
+ "tokens_seen": 885719040
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005912136409227684,
+ "loss": 1.3534,
+ "theoretical_loss": 3.720498157096035,
+ "tokens_seen": 885784576
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591197592778335,
+ "loss": 1.3756,
+ "theoretical_loss": 3.7204715766591105,
+ "tokens_seen": 885850112
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911815446339018,
+ "loss": 1.2983,
+ "theoretical_loss": 3.720444998739117,
+ "tokens_seen": 885915648
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911654964894684,
+ "loss": 1.3486,
+ "theoretical_loss": 3.720418423335629,
+ "tokens_seen": 885981184
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911494483450352,
+ "loss": 1.4364,
+ "theoretical_loss": 3.7203918504482227,
+ "tokens_seen": 886046720
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911334002006018,
+ "loss": 1.4148,
+ "theoretical_loss": 3.7203652800764737,
+ "tokens_seen": 886112256
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911173520561686,
+ "loss": 1.4583,
+ "theoretical_loss": 3.7203387122199576,
+ "tokens_seen": 886177792
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005911013039117352,
+ "loss": 1.3716,
+ "theoretical_loss": 3.7203121468782507,
+ "tokens_seen": 886243328
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591085255767302,
+ "loss": 1.3507,
+ "theoretical_loss": 3.720285584050928,
+ "tokens_seen": 886308864
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005910692076228686,
+ "loss": 1.3487,
+ "theoretical_loss": 3.7202590237375674,
+ "tokens_seen": 886374400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005910531594784354,
+ "loss": 1.4123,
+ "theoretical_loss": 3.7202324659377437,
+ "tokens_seen": 886439936
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000591037111334002,
+ "loss": 1.301,
+ "theoretical_loss": 3.7202059106510337,
+ "tokens_seen": 886505472
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005910210631895688,
+ "loss": 1.3052,
+ "theoretical_loss": 3.720179357877014,
+ "tokens_seen": 886571008
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005910050150451354,
+ "loss": 1.2118,
+ "theoretical_loss": 3.7201528076152606,
+ "tokens_seen": 886636544
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005909889669007022,
+ "loss": 1.2721,
+ "theoretical_loss": 3.7201262598653506,
+ "tokens_seen": 886702080
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005909729187562689,
+ "loss": 1.4885,
+ "theoretical_loss": 3.720099714626861,
+ "tokens_seen": 886767616
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005909568706118356,
+ "loss": 1.3028,
+ "theoretical_loss": 3.720073171899368,
+ "tokens_seen": 886833152
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005909408224674023,
+ "loss": 1.3956,
+ "theoretical_loss": 3.720046631682448,
+ "tokens_seen": 886898688
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590924774322969,
+ "loss": 1.2754,
+ "theoretical_loss": 3.7200200939756796,
+ "tokens_seen": 886964224
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005909087261785357,
+ "loss": 1.2936,
+ "theoretical_loss": 3.719993558778639,
+ "tokens_seen": 887029760
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908926780341024,
+ "loss": 1.4283,
+ "theoretical_loss": 3.7199670260909037,
+ "tokens_seen": 887095296
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908766298896691,
+ "loss": 1.3382,
+ "theoretical_loss": 3.7199404959120512,
+ "tokens_seen": 887160832
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908605817452358,
+ "loss": 1.3836,
+ "theoretical_loss": 3.7199139682416584,
+ "tokens_seen": 887226368
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908445336008025,
+ "loss": 1.4363,
+ "theoretical_loss": 3.719887443079303,
+ "tokens_seen": 887291904
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908284854563692,
+ "loss": 1.3388,
+ "theoretical_loss": 3.719860920424563,
+ "tokens_seen": 887357440
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005908124373119359,
+ "loss": 1.2925,
+ "theoretical_loss": 3.7198344002770165,
+ "tokens_seen": 887422976
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005907963891675026,
+ "loss": 1.306,
+ "theoretical_loss": 3.719807882636241,
+ "tokens_seen": 887488512
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005907803410230693,
+ "loss": 1.3648,
+ "theoretical_loss": 3.719781367501813,
+ "tokens_seen": 887554048
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590764292878636,
+ "loss": 1.3699,
+ "theoretical_loss": 3.719754854873313,
+ "tokens_seen": 887619584
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005907482447342026,
+ "loss": 1.3634,
+ "theoretical_loss": 3.719728344750318,
+ "tokens_seen": 887685120
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005907321965897694,
+ "loss": 1.4477,
+ "theoretical_loss": 3.7197018371324067,
+ "tokens_seen": 887750656
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590716148445336,
+ "loss": 1.3321,
+ "theoretical_loss": 3.7196753320191567,
+ "tokens_seen": 887816192
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005907001003009028,
+ "loss": 1.4212,
+ "theoretical_loss": 3.7196488294101466,
+ "tokens_seen": 887881728
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906840521564694,
+ "loss": 1.3911,
+ "theoretical_loss": 3.719622329304956,
+ "tokens_seen": 887947264
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 300533,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4114042520523071,
+ "objective/train/theoretical_loss": 3.7195958317031623,
+ "objective/train/tokens_used": 908472800,
+ "theoretical_loss": 3.7195958317031623,
+ "tokens_seen": 888012800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906680040120362,
+ "loss": 1.313,
+ "theoretical_loss": 3.7195958317031623,
+ "tokens_seen": 888012800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906519558676028,
+ "loss": 1.3689,
+ "theoretical_loss": 3.719569336604345,
+ "tokens_seen": 888078336
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906359077231695,
+ "loss": 1.4185,
+ "theoretical_loss": 3.7195428440080827,
+ "tokens_seen": 888143872
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906198595787362,
+ "loss": 1.2331,
+ "theoretical_loss": 3.7195163539139546,
+ "tokens_seen": 888209408
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005906038114343029,
+ "loss": 1.3944,
+ "theoretical_loss": 3.71948986632154,
+ "tokens_seen": 888274944
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005905877632898696,
+ "loss": 1.323,
+ "theoretical_loss": 3.719463381230418,
+ "tokens_seen": 888340480
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005905717151454363,
+ "loss": 1.3213,
+ "theoretical_loss": 3.7194368986401676,
+ "tokens_seen": 888406016
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590555667001003,
+ "loss": 1.4546,
+ "theoretical_loss": 3.719410418550368,
+ "tokens_seen": 888471552
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005905396188565697,
+ "loss": 1.3573,
+ "theoretical_loss": 3.7193839409605993,
+ "tokens_seen": 888537088
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005905235707121364,
+ "loss": 1.3619,
+ "theoretical_loss": 3.7193574658704405,
+ "tokens_seen": 888602624
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005905075225677031,
+ "loss": 1.3303,
+ "theoretical_loss": 3.719330993279472,
+ "tokens_seen": 888668160
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904914744232698,
+ "loss": 1.3685,
+ "theoretical_loss": 3.7193045231872732,
+ "tokens_seen": 888733696
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904754262788365,
+ "loss": 1.4181,
+ "theoretical_loss": 3.7192780555934237,
+ "tokens_seen": 888799232
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904593781344032,
+ "loss": 1.4243,
+ "theoretical_loss": 3.719251590497504,
+ "tokens_seen": 888864768
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904433299899699,
+ "loss": 1.3591,
+ "theoretical_loss": 3.7192251278990938,
+ "tokens_seen": 888930304
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904272818455366,
+ "loss": 1.3494,
+ "theoretical_loss": 3.7191986677977735,
+ "tokens_seen": 888995840
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005904112337011033,
+ "loss": 1.3963,
+ "theoretical_loss": 3.7191722101931237,
+ "tokens_seen": 889061376
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00059039518555667,
+ "loss": 1.2319,
+ "theoretical_loss": 3.7191457550847247,
+ "tokens_seen": 889126912
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005903791374122367,
+ "loss": 1.4435,
+ "theoretical_loss": 3.7191193024721567,
+ "tokens_seen": 889192448
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005903630892678034,
+ "loss": 1.2296,
+ "theoretical_loss": 3.719092852355,
+ "tokens_seen": 889257984
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005903470411233701,
+ "loss": 1.395,
+ "theoretical_loss": 3.7190664047328363,
+ "tokens_seen": 889323520
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005903309929789368,
+ "loss": 1.3008,
+ "theoretical_loss": 3.719039959605246,
+ "tokens_seen": 889389056
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005903149448345035,
+ "loss": 1.3368,
+ "theoretical_loss": 3.7190135169718097,
+ "tokens_seen": 889454592
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902988966900702,
+ "loss": 1.4134,
+ "theoretical_loss": 3.718987076832109,
+ "tokens_seen": 889520128
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902828485456369,
+ "loss": 1.2957,
+ "theoretical_loss": 3.718960639185724,
+ "tokens_seen": 889585664
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902668004012036,
+ "loss": 1.4356,
+ "theoretical_loss": 3.7189342040322373,
+ "tokens_seen": 889651200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902507522567703,
+ "loss": 1.3634,
+ "theoretical_loss": 3.718907771371229,
+ "tokens_seen": 889716736
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590234704112337,
+ "loss": 1.3358,
+ "theoretical_loss": 3.718881341202281,
+ "tokens_seen": 889782272
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902186559679037,
+ "loss": 1.2821,
+ "theoretical_loss": 3.7188549135249755,
+ "tokens_seen": 889847808
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005902026078234704,
+ "loss": 1.3268,
+ "theoretical_loss": 3.718828488338893,
+ "tokens_seen": 889913344
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901865596790371,
+ "loss": 1.4549,
+ "theoretical_loss": 3.718802065643616,
+ "tokens_seen": 889978880
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901705115346038,
+ "loss": 1.3998,
+ "theoretical_loss": 3.7187756454387255,
+ "tokens_seen": 890044416
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901544633901705,
+ "loss": 1.4264,
+ "theoretical_loss": 3.7187492277238046,
+ "tokens_seen": 890109952
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901384152457372,
+ "loss": 1.3757,
+ "theoretical_loss": 3.7187228124984344,
+ "tokens_seen": 890175488
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901223671013039,
+ "loss": 1.3704,
+ "theoretical_loss": 3.7186963997621976,
+ "tokens_seen": 890241024
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005901063189568706,
+ "loss": 1.4204,
+ "theoretical_loss": 3.7186699895146758,
+ "tokens_seen": 890306560
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005900902708124373,
+ "loss": 1.286,
+ "theoretical_loss": 3.718643581755452,
+ "tokens_seen": 890372096
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000590074222668004,
+ "loss": 1.3411,
+ "theoretical_loss": 3.7186171764841083,
+ "tokens_seen": 890437632
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005900581745235707,
+ "loss": 1.4316,
+ "theoretical_loss": 3.7185907737002273,
+ "tokens_seen": 890503168
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005900421263791374,
+ "loss": 1.3444,
+ "theoretical_loss": 3.7185643734033915,
+ "tokens_seen": 890568704
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005900260782347041,
+ "loss": 1.3862,
+ "theoretical_loss": 3.7185379755931836,
+ "tokens_seen": 890634240
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005900100300902708,
+ "loss": 1.3811,
+ "theoretical_loss": 3.718511580269187,
+ "tokens_seen": 890699776
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005899939819458375,
+ "loss": 1.4249,
+ "theoretical_loss": 3.718485187430984,
+ "tokens_seen": 890765312
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005899779338014042,
+ "loss": 1.3565,
+ "theoretical_loss": 3.7184587970781573,
+ "tokens_seen": 890830848
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005899618856569709,
+ "loss": 1.3189,
+ "theoretical_loss": 3.7184324092102914,
+ "tokens_seen": 890896384
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005899458375125377,
+ "loss": 1.4689,
+ "theoretical_loss": 3.7184060238269683,
+ "tokens_seen": 890961920
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005899297893681043,
+ "loss": 1.3887,
+ "theoretical_loss": 3.718379640927772,
+ "tokens_seen": 891027456
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000589913741223671,
+ "loss": 1.3704,
+ "theoretical_loss": 3.718353260512286,
+ "tokens_seen": 891092992
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898976930792377,
+ "loss": 1.4669,
+ "theoretical_loss": 3.718326882580093,
+ "tokens_seen": 891158528
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898816449348045,
+ "loss": 1.2718,
+ "theoretical_loss": 3.7183005071307775,
+ "tokens_seen": 891224064
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 301711,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2756049633026123,
+ "objective/train/theoretical_loss": 3.7182741341639227,
+ "objective/train/tokens_used": 911749600,
+ "theoretical_loss": 3.7182741341639227,
+ "tokens_seen": 891289600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898655967903711,
+ "loss": 1.286,
+ "theoretical_loss": 3.7182741341639227,
+ "tokens_seen": 891289600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898495486459379,
+ "loss": 1.4418,
+ "theoretical_loss": 3.7182477636791127,
+ "tokens_seen": 891355136
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898335005015045,
+ "loss": 1.331,
+ "theoretical_loss": 3.718221395675932,
+ "tokens_seen": 891420672
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898174523570713,
+ "loss": 1.3085,
+ "theoretical_loss": 3.7181950301539635,
+ "tokens_seen": 891486208
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005898014042126379,
+ "loss": 1.3143,
+ "theoretical_loss": 3.718168667112792,
+ "tokens_seen": 891551744
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897853560682047,
+ "loss": 1.2849,
+ "theoretical_loss": 3.718142306552002,
+ "tokens_seen": 891617280
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897693079237714,
+ "loss": 1.2788,
+ "theoretical_loss": 3.718115948471177,
+ "tokens_seen": 891682816
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897532597793381,
+ "loss": 1.4999,
+ "theoretical_loss": 3.718089592869902,
+ "tokens_seen": 891748352
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897372116349048,
+ "loss": 1.2384,
+ "theoretical_loss": 3.718063239747762,
+ "tokens_seen": 891813888
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897211634904715,
+ "loss": 1.4044,
+ "theoretical_loss": 3.718036889104341,
+ "tokens_seen": 891879424
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005897051153460382,
+ "loss": 1.3401,
+ "theoretical_loss": 3.7180105409392237,
+ "tokens_seen": 891944960
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005896890672016049,
+ "loss": 1.3459,
+ "theoretical_loss": 3.7179841952519945,
+ "tokens_seen": 892010496
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005896730190571716,
+ "loss": 1.283,
+ "theoretical_loss": 3.7179578520422396,
+ "tokens_seen": 892076032
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005896569709127383,
+ "loss": 1.372,
+ "theoretical_loss": 3.7179315113095437,
+ "tokens_seen": 892141568
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000589640922768305,
+ "loss": 1.4019,
+ "theoretical_loss": 3.7179051730534916,
+ "tokens_seen": 892207104
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005896248746238717,
+ "loss": 1.3742,
+ "theoretical_loss": 3.717878837273668,
+ "tokens_seen": 892272640
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005896088264794384,
+ "loss": 1.3471,
+ "theoretical_loss": 3.7178525039696595,
+ "tokens_seen": 892338176
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895927783350051,
+ "loss": 1.351,
+ "theoretical_loss": 3.7178261731410505,
+ "tokens_seen": 892403712
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895767301905718,
+ "loss": 1.3865,
+ "theoretical_loss": 3.717799844787427,
+ "tokens_seen": 892469248
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895606820461385,
+ "loss": 1.4294,
+ "theoretical_loss": 3.7177735189083743,
+ "tokens_seen": 892534784
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895446339017052,
+ "loss": 1.3942,
+ "theoretical_loss": 3.717747195503479,
+ "tokens_seen": 892600320
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895285857572719,
+ "loss": 1.4086,
+ "theoretical_loss": 3.7177208745723256,
+ "tokens_seen": 892665856
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005895125376128386,
+ "loss": 1.3606,
+ "theoretical_loss": 3.717694556114502,
+ "tokens_seen": 892731392
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894964894684053,
+ "loss": 1.2228,
+ "theoretical_loss": 3.717668240129592,
+ "tokens_seen": 892796928
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000589480441323972,
+ "loss": 1.3704,
+ "theoretical_loss": 3.717641926617183,
+ "tokens_seen": 892862464
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894643931795387,
+ "loss": 1.3442,
+ "theoretical_loss": 3.717615615576861,
+ "tokens_seen": 892928000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894483450351054,
+ "loss": 1.312,
+ "theoretical_loss": 3.717589307008213,
+ "tokens_seen": 892993536
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894322968906721,
+ "loss": 1.365,
+ "theoretical_loss": 3.717563000910824,
+ "tokens_seen": 893059072
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894162487462388,
+ "loss": 1.3723,
+ "theoretical_loss": 3.717536697284282,
+ "tokens_seen": 893124608
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005894002006018055,
+ "loss": 1.3776,
+ "theoretical_loss": 3.717510396128173,
+ "tokens_seen": 893190144
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893841524573722,
+ "loss": 1.4324,
+ "theoretical_loss": 3.7174840974420826,
+ "tokens_seen": 893255680
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893681043129389,
+ "loss": 1.4634,
+ "theoretical_loss": 3.7174578012256,
+ "tokens_seen": 893321216
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893520561685055,
+ "loss": 1.2339,
+ "theoretical_loss": 3.7174315074783104,
+ "tokens_seen": 893386752
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893360080240723,
+ "loss": 1.4133,
+ "theoretical_loss": 3.7174052161998015,
+ "tokens_seen": 893452288
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893199598796389,
+ "loss": 1.2415,
+ "theoretical_loss": 3.7173789273896602,
+ "tokens_seen": 893517824
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005893039117352057,
+ "loss": 1.465,
+ "theoretical_loss": 3.717352641047474,
+ "tokens_seen": 893583360
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892878635907723,
+ "loss": 1.3275,
+ "theoretical_loss": 3.7173263571728303,
+ "tokens_seen": 893648896
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892718154463391,
+ "loss": 1.328,
+ "theoretical_loss": 3.7173000757653156,
+ "tokens_seen": 893714432
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892557673019057,
+ "loss": 1.3894,
+ "theoretical_loss": 3.717273796824519,
+ "tokens_seen": 893779968
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892397191574725,
+ "loss": 1.4125,
+ "theoretical_loss": 3.7172475203500266,
+ "tokens_seen": 893845504
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892236710130391,
+ "loss": 1.3008,
+ "theoretical_loss": 3.717221246341427,
+ "tokens_seen": 893911040
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005892076228686058,
+ "loss": 1.4317,
+ "theoretical_loss": 3.717194974798308,
+ "tokens_seen": 893976576
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005891915747241725,
+ "loss": 1.4452,
+ "theoretical_loss": 3.7171687057202574,
+ "tokens_seen": 894042112
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005891755265797392,
+ "loss": 1.323,
+ "theoretical_loss": 3.717142439106863,
+ "tokens_seen": 894107648
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005891594784353059,
+ "loss": 1.4508,
+ "theoretical_loss": 3.717116174957713,
+ "tokens_seen": 894173184
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005891434302908726,
+ "loss": 1.42,
+ "theoretical_loss": 3.717089913272396,
+ "tokens_seen": 894238720
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005891273821464393,
+ "loss": 1.3354,
+ "theoretical_loss": 3.7170636540504995,
+ "tokens_seen": 894304256
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000589111334002006,
+ "loss": 1.4287,
+ "theoretical_loss": 3.7170373972916133,
+ "tokens_seen": 894369792
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890952858575727,
+ "loss": 1.3652,
+ "theoretical_loss": 3.7170111429953248,
+ "tokens_seen": 894435328
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890792377131394,
+ "loss": 1.3814,
+ "theoretical_loss": 3.716984891161223,
+ "tokens_seen": 894500864
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 302661,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2354652881622314,
+ "objective/train/theoretical_loss": 3.7169586417888967,
+ "objective/train/tokens_used": 915026400,
+ "theoretical_loss": 3.7169586417888967,
+ "tokens_seen": 894566400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890631895687061,
+ "loss": 1.2496,
+ "theoretical_loss": 3.7169586417888967,
+ "tokens_seen": 894566400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890471414242728,
+ "loss": 1.2781,
+ "theoretical_loss": 3.7169323948779343,
+ "tokens_seen": 894631936
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890310932798395,
+ "loss": 1.3616,
+ "theoretical_loss": 3.716906150427925,
+ "tokens_seen": 894697472
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005890150451354062,
+ "loss": 1.4008,
+ "theoretical_loss": 3.7168799084384583,
+ "tokens_seen": 894763008
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889989969909729,
+ "loss": 1.4406,
+ "theoretical_loss": 3.7168536689091223,
+ "tokens_seen": 894828544
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889829488465396,
+ "loss": 1.2974,
+ "theoretical_loss": 3.716827431839507,
+ "tokens_seen": 894894080
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889669007021063,
+ "loss": 1.4378,
+ "theoretical_loss": 3.7168011972292017,
+ "tokens_seen": 894959616
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588950852557673,
+ "loss": 1.4663,
+ "theoretical_loss": 3.7167749650777955,
+ "tokens_seen": 895025152
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889348044132397,
+ "loss": 1.3886,
+ "theoretical_loss": 3.7167487353848774,
+ "tokens_seen": 895090688
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889187562688064,
+ "loss": 1.258,
+ "theoretical_loss": 3.7167225081500384,
+ "tokens_seen": 895156224
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005889027081243731,
+ "loss": 1.302,
+ "theoretical_loss": 3.716696283372867,
+ "tokens_seen": 895221760
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888866599799398,
+ "loss": 1.3362,
+ "theoretical_loss": 3.716670061052954,
+ "tokens_seen": 895287296
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888706118355065,
+ "loss": 1.3319,
+ "theoretical_loss": 3.7166438411898883,
+ "tokens_seen": 895352832
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888545636910732,
+ "loss": 1.3314,
+ "theoretical_loss": 3.7166176237832604,
+ "tokens_seen": 895418368
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888385155466399,
+ "loss": 1.3141,
+ "theoretical_loss": 3.7165914088326604,
+ "tokens_seen": 895483904
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888224674022066,
+ "loss": 1.3686,
+ "theoretical_loss": 3.7165651963376787,
+ "tokens_seen": 895549440
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005888064192577733,
+ "loss": 1.2977,
+ "theoretical_loss": 3.716538986297905,
+ "tokens_seen": 895614976
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00058879037111334,
+ "loss": 1.4045,
+ "theoretical_loss": 3.7165127787129304,
+ "tokens_seen": 895680512
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005887743229689067,
+ "loss": 1.3782,
+ "theoretical_loss": 3.716486573582345,
+ "tokens_seen": 895746048
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005887582748244734,
+ "loss": 1.2311,
+ "theoretical_loss": 3.7164603709057387,
+ "tokens_seen": 895811584
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005887422266800401,
+ "loss": 1.5138,
+ "theoretical_loss": 3.716434170682704,
+ "tokens_seen": 895877120
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005887261785356068,
+ "loss": 1.326,
+ "theoretical_loss": 3.7164079729128296,
+ "tokens_seen": 895942656
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005887101303911735,
+ "loss": 1.4816,
+ "theoretical_loss": 3.7163817775957084,
+ "tokens_seen": 896008192
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005886940822467402,
+ "loss": 1.4378,
+ "theoretical_loss": 3.71635558473093,
+ "tokens_seen": 896073728
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588678034102307,
+ "loss": 1.3225,
+ "theoretical_loss": 3.7163293943180857,
+ "tokens_seen": 896139264
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005886619859578736,
+ "loss": 1.3436,
+ "theoretical_loss": 3.7163032063567676,
+ "tokens_seen": 896204800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005886459378134403,
+ "loss": 1.4952,
+ "theoretical_loss": 3.7162770208465656,
+ "tokens_seen": 896270336
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588629889669007,
+ "loss": 1.3227,
+ "theoretical_loss": 3.716250837787072,
+ "tokens_seen": 896335872
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005886138415245737,
+ "loss": 1.4207,
+ "theoretical_loss": 3.716224657177878,
+ "tokens_seen": 896401408
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885977933801404,
+ "loss": 1.311,
+ "theoretical_loss": 3.716198479018575,
+ "tokens_seen": 896466944
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885817452357072,
+ "loss": 1.5235,
+ "theoretical_loss": 3.7161723033087553,
+ "tokens_seen": 896532480
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885656970912739,
+ "loss": 1.451,
+ "theoretical_loss": 3.71614613004801,
+ "tokens_seen": 896598016
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885496489468406,
+ "loss": 1.3184,
+ "theoretical_loss": 3.716119959235931,
+ "tokens_seen": 896663552
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885336008024073,
+ "loss": 1.257,
+ "theoretical_loss": 3.716093790872111,
+ "tokens_seen": 896729088
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588517552657974,
+ "loss": 1.3325,
+ "theoretical_loss": 3.716067624956141,
+ "tokens_seen": 896794624
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005885015045135407,
+ "loss": 1.3376,
+ "theoretical_loss": 3.716041461487614,
+ "tokens_seen": 896860160
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884854563691074,
+ "loss": 1.4556,
+ "theoretical_loss": 3.716015300466122,
+ "tokens_seen": 896925696
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884694082246741,
+ "loss": 1.4852,
+ "theoretical_loss": 3.7159891418912574,
+ "tokens_seen": 896991232
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884533600802408,
+ "loss": 1.4474,
+ "theoretical_loss": 3.715962985762612,
+ "tokens_seen": 897056768
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884373119358075,
+ "loss": 1.3913,
+ "theoretical_loss": 3.7159368320797794,
+ "tokens_seen": 897122304
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884212637913742,
+ "loss": 1.3786,
+ "theoretical_loss": 3.7159106808423523,
+ "tokens_seen": 897187840
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005884052156469409,
+ "loss": 1.3073,
+ "theoretical_loss": 3.7158845320499223,
+ "tokens_seen": 897253376
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005883891675025076,
+ "loss": 1.3805,
+ "theoretical_loss": 3.7158583857020826,
+ "tokens_seen": 897318912
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005883731193580743,
+ "loss": 1.5188,
+ "theoretical_loss": 3.7158322417984273,
+ "tokens_seen": 897384448
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588357071213641,
+ "loss": 1.3454,
+ "theoretical_loss": 3.715806100338548,
+ "tokens_seen": 897449984
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005883410230692077,
+ "loss": 1.3985,
+ "theoretical_loss": 3.7157799613220384,
+ "tokens_seen": 897515520
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005883249749247744,
+ "loss": 1.3109,
+ "theoretical_loss": 3.715753824748492,
+ "tokens_seen": 897581056
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005883089267803411,
+ "loss": 1.3792,
+ "theoretical_loss": 3.715727690617502,
+ "tokens_seen": 897646592
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882928786359078,
+ "loss": 1.4112,
+ "theoretical_loss": 3.7157015589286617,
+ "tokens_seen": 897712128
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882768304914745,
+ "loss": 1.5598,
+ "theoretical_loss": 3.715675429681564,
+ "tokens_seen": 897777664
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 303919,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4079523086547852,
+ "objective/train/theoretical_loss": 3.7156493028758035,
+ "objective/train/tokens_used": 918303200,
+ "theoretical_loss": 3.7156493028758035,
+ "tokens_seen": 897843200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882607823470412,
+ "loss": 1.4222,
+ "theoretical_loss": 3.7156493028758035,
+ "tokens_seen": 897843200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882447342026079,
+ "loss": 1.462,
+ "theoretical_loss": 3.715623178510974,
+ "tokens_seen": 897908736
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882286860581746,
+ "loss": 1.3918,
+ "theoretical_loss": 3.7155970565866685,
+ "tokens_seen": 897974272
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005882126379137413,
+ "loss": 1.4074,
+ "theoretical_loss": 3.7155709371024814,
+ "tokens_seen": 898039808
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588196589769308,
+ "loss": 1.3259,
+ "theoretical_loss": 3.715544820058007,
+ "tokens_seen": 898105344
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881805416248747,
+ "loss": 1.387,
+ "theoretical_loss": 3.7155187054528387,
+ "tokens_seen": 898170880
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881644934804414,
+ "loss": 1.4242,
+ "theoretical_loss": 3.715492593286571,
+ "tokens_seen": 898236416
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881484453360081,
+ "loss": 1.3918,
+ "theoretical_loss": 3.715466483558799,
+ "tokens_seen": 898301952
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881323971915748,
+ "loss": 1.343,
+ "theoretical_loss": 3.7154403762691155,
+ "tokens_seen": 898367488
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881163490471415,
+ "loss": 1.3612,
+ "theoretical_loss": 3.7154142714171168,
+ "tokens_seen": 898433024
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005881003009027082,
+ "loss": 1.4501,
+ "theoretical_loss": 3.715388169002396,
+ "tokens_seen": 898498560
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005880842527582749,
+ "loss": 1.3329,
+ "theoretical_loss": 3.715362069024548,
+ "tokens_seen": 898564096
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005880682046138416,
+ "loss": 1.245,
+ "theoretical_loss": 3.7153359714831686,
+ "tokens_seen": 898629632
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005880521564694083,
+ "loss": 1.3412,
+ "theoretical_loss": 3.7153098763778516,
+ "tokens_seen": 898695168
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000588036108324975,
+ "loss": 1.3866,
+ "theoretical_loss": 3.715283783708193,
+ "tokens_seen": 898760704
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005880200601805417,
+ "loss": 1.4898,
+ "theoretical_loss": 3.715257693473787,
+ "tokens_seen": 898826240
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005880040120361084,
+ "loss": 1.3591,
+ "theoretical_loss": 3.715231605674229,
+ "tokens_seen": 898891776
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879879638916751,
+ "loss": 1.4781,
+ "theoretical_loss": 3.7152055203091146,
+ "tokens_seen": 898957312
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879719157472418,
+ "loss": 1.2317,
+ "theoretical_loss": 3.7151794373780387,
+ "tokens_seen": 899022848
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879558676028084,
+ "loss": 1.4104,
+ "theoretical_loss": 3.715153356880597,
+ "tokens_seen": 899088384
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879398194583752,
+ "loss": 1.3016,
+ "theoretical_loss": 3.715127278816386,
+ "tokens_seen": 899153920
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879237713139418,
+ "loss": 1.4785,
+ "theoretical_loss": 3.7151012031849993,
+ "tokens_seen": 899219456
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005879077231695086,
+ "loss": 1.3736,
+ "theoretical_loss": 3.715075129986034,
+ "tokens_seen": 899284992
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005878916750250752,
+ "loss": 1.3649,
+ "theoretical_loss": 3.715049059219086,
+ "tokens_seen": 899350528
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587875626880642,
+ "loss": 1.3917,
+ "theoretical_loss": 3.7150229908837513,
+ "tokens_seen": 899416064
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005878595787362086,
+ "loss": 1.3189,
+ "theoretical_loss": 3.7149969249796255,
+ "tokens_seen": 899481600
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005878435305917754,
+ "loss": 1.3525,
+ "theoretical_loss": 3.7149708615063046,
+ "tokens_seen": 899547136
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587827482447342,
+ "loss": 1.3852,
+ "theoretical_loss": 3.7149448004633854,
+ "tokens_seen": 899612672
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005878114343029087,
+ "loss": 1.3628,
+ "theoretical_loss": 3.714918741850464,
+ "tokens_seen": 899678208
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877953861584754,
+ "loss": 1.3601,
+ "theoretical_loss": 3.7148926856671367,
+ "tokens_seen": 899743744
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877793380140421,
+ "loss": 1.4436,
+ "theoretical_loss": 3.7148666319130004,
+ "tokens_seen": 899809280
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877632898696088,
+ "loss": 1.4471,
+ "theoretical_loss": 3.7148405805876514,
+ "tokens_seen": 899874816
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877472417251755,
+ "loss": 1.3154,
+ "theoretical_loss": 3.7148145316906866,
+ "tokens_seen": 899940352
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877311935807422,
+ "loss": 1.3245,
+ "theoretical_loss": 3.7147884852217024,
+ "tokens_seen": 900005888
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005877151454363089,
+ "loss": 1.4425,
+ "theoretical_loss": 3.714762441180296,
+ "tokens_seen": 900071424
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876990972918756,
+ "loss": 1.38,
+ "theoretical_loss": 3.714736399566065,
+ "tokens_seen": 900136960
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876830491474423,
+ "loss": 1.1755,
+ "theoretical_loss": 3.7147103603786054,
+ "tokens_seen": 900202496
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587667001003009,
+ "loss": 1.2811,
+ "theoretical_loss": 3.7146843236175155,
+ "tokens_seen": 900268032
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876509528585757,
+ "loss": 1.4444,
+ "theoretical_loss": 3.714658289282392,
+ "tokens_seen": 900333568
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876349047141424,
+ "loss": 1.4442,
+ "theoretical_loss": 3.7146322573728323,
+ "tokens_seen": 900399104
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876188565697091,
+ "loss": 1.4458,
+ "theoretical_loss": 3.7146062278884338,
+ "tokens_seen": 900464640
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005876028084252758,
+ "loss": 1.304,
+ "theoretical_loss": 3.7145802008287943,
+ "tokens_seen": 900530176
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005875867602808425,
+ "loss": 1.3643,
+ "theoretical_loss": 3.7145541761935115,
+ "tokens_seen": 900595712
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005875707121364092,
+ "loss": 1.4021,
+ "theoretical_loss": 3.714528153982183,
+ "tokens_seen": 900661248
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005875546639919759,
+ "loss": 1.4293,
+ "theoretical_loss": 3.714502134194407,
+ "tokens_seen": 900726784
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005875386158475426,
+ "loss": 1.396,
+ "theoretical_loss": 3.7144761168297817,
+ "tokens_seen": 900792320
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005875225677031093,
+ "loss": 1.3798,
+ "theoretical_loss": 3.714450101887904,
+ "tokens_seen": 900857856
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587506519558676,
+ "loss": 1.3311,
+ "theoretical_loss": 3.7144240893683733,
+ "tokens_seen": 900923392
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874904714142427,
+ "loss": 1.3374,
+ "theoretical_loss": 3.7143980792707874,
+ "tokens_seen": 900988928
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874744232698094,
+ "loss": 1.4061,
+ "theoretical_loss": 3.7143720715947444,
+ "tokens_seen": 901054464
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 305176,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4514453411102295,
+ "objective/train/theoretical_loss": 3.7143460663398433,
+ "objective/train/tokens_used": 921580000,
+ "theoretical_loss": 3.7143460663398433,
+ "tokens_seen": 901120000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874583751253761,
+ "loss": 1.4118,
+ "theoretical_loss": 3.7143460663398433,
+ "tokens_seen": 901120000
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874423269809428,
+ "loss": 1.4363,
+ "theoretical_loss": 3.714320063505682,
+ "tokens_seen": 901185536
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874262788365095,
+ "loss": 1.2861,
+ "theoretical_loss": 3.7142940630918595,
+ "tokens_seen": 901251072
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005874102306920762,
+ "loss": 1.5371,
+ "theoretical_loss": 3.7142680650979747,
+ "tokens_seen": 901316608
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005873941825476429,
+ "loss": 1.3465,
+ "theoretical_loss": 3.7142420695236265,
+ "tokens_seen": 901382144
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005873781344032096,
+ "loss": 1.3457,
+ "theoretical_loss": 3.7142160763684133,
+ "tokens_seen": 901447680
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005873620862587764,
+ "loss": 1.3535,
+ "theoretical_loss": 3.7141900856319343,
+ "tokens_seen": 901513216
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587346038114343,
+ "loss": 1.4932,
+ "theoretical_loss": 3.714164097313789,
+ "tokens_seen": 901578752
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005873299899699098,
+ "loss": 1.343,
+ "theoretical_loss": 3.7141381114135763,
+ "tokens_seen": 901644288
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005873139418254764,
+ "loss": 1.3572,
+ "theoretical_loss": 3.7141121279308957,
+ "tokens_seen": 901709824
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872978936810432,
+ "loss": 1.3743,
+ "theoretical_loss": 3.7140861468653465,
+ "tokens_seen": 901775360
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872818455366098,
+ "loss": 1.4433,
+ "theoretical_loss": 3.7140601682165277,
+ "tokens_seen": 901840896
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872657973921766,
+ "loss": 1.393,
+ "theoretical_loss": 3.71403419198404,
+ "tokens_seen": 901906432
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872497492477432,
+ "loss": 1.2557,
+ "theoretical_loss": 3.714008218167482,
+ "tokens_seen": 901971968
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00058723370110331,
+ "loss": 1.4866,
+ "theoretical_loss": 3.7139822467664545,
+ "tokens_seen": 902037504
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872176529588767,
+ "loss": 1.3731,
+ "theoretical_loss": 3.713956277780557,
+ "tokens_seen": 902103040
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005872016048144434,
+ "loss": 1.4039,
+ "theoretical_loss": 3.7139303112093884,
+ "tokens_seen": 902168576
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.00058718555667001,
+ "loss": 1.2867,
+ "theoretical_loss": 3.7139043470525506,
+ "tokens_seen": 902234112
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005871695085255768,
+ "loss": 1.3927,
+ "theoretical_loss": 3.713878385309643,
+ "tokens_seen": 902299648
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005871534603811435,
+ "loss": 1.4256,
+ "theoretical_loss": 3.7138524259802654,
+ "tokens_seen": 902365184
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005871374122367102,
+ "loss": 1.4969,
+ "theoretical_loss": 3.7138264690640184,
+ "tokens_seen": 902430720
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005871213640922769,
+ "loss": 1.3782,
+ "theoretical_loss": 3.7138005145605026,
+ "tokens_seen": 902496256
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005871053159478436,
+ "loss": 1.3181,
+ "theoretical_loss": 3.713774562469319,
+ "tokens_seen": 902561792
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005870892678034103,
+ "loss": 1.4391,
+ "theoretical_loss": 3.7137486127900674,
+ "tokens_seen": 902627328
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000587073219658977,
+ "loss": 1.4097,
+ "theoretical_loss": 3.713722665522349,
+ "tokens_seen": 902692864
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005870571715145437,
+ "loss": 1.4201,
+ "theoretical_loss": 3.713696720665765,
+ "tokens_seen": 902758400
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005870411233701104,
+ "loss": 1.349,
+ "theoretical_loss": 3.7136707782199156,
+ "tokens_seen": 902823936
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005870250752256771,
+ "loss": 1.3211,
+ "theoretical_loss": 3.7136448381844023,
+ "tokens_seen": 902889472
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005870090270812438,
+ "loss": 1.4117,
+ "theoretical_loss": 3.713618900558826,
+ "tokens_seen": 902955008
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005869929789368105,
+ "loss": 1.4085,
+ "theoretical_loss": 3.7135929653427877,
+ "tokens_seen": 903020544
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005869769307923772,
+ "loss": 1.3658,
+ "theoretical_loss": 3.7135670325358894,
+ "tokens_seen": 903086080
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005869608826479439,
+ "loss": 1.5581,
+ "theoretical_loss": 3.7135411021377323,
+ "tokens_seen": 903151616
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005869448345035106,
+ "loss": 1.4429,
+ "theoretical_loss": 3.713515174147917,
+ "tokens_seen": 903217152
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005869287863590773,
+ "loss": 1.3243,
+ "theoretical_loss": 3.7134892485660465,
+ "tokens_seen": 903282688
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000586912738214644,
+ "loss": 1.4235,
+ "theoretical_loss": 3.7134633253917215,
+ "tokens_seen": 903348224
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868966900702107,
+ "loss": 1.4358,
+ "theoretical_loss": 3.713437404624544,
+ "tokens_seen": 903413760
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868806419257774,
+ "loss": 1.4272,
+ "theoretical_loss": 3.7134114862641163,
+ "tokens_seen": 903479296
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868645937813441,
+ "loss": 1.3875,
+ "theoretical_loss": 3.71338557031004,
+ "tokens_seen": 903544832
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868485456369108,
+ "loss": 1.3641,
+ "theoretical_loss": 3.713359656761917,
+ "tokens_seen": 903610368
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868324974924775,
+ "loss": 1.4771,
+ "theoretical_loss": 3.71333374561935,
+ "tokens_seen": 903675904
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868164493480442,
+ "loss": 1.3326,
+ "theoretical_loss": 3.713307836881941,
+ "tokens_seen": 903741440
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005868004012036109,
+ "loss": 1.4815,
+ "theoretical_loss": 3.713281930549292,
+ "tokens_seen": 903806976
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005867843530591776,
+ "loss": 1.2739,
+ "theoretical_loss": 3.7132560266210057,
+ "tokens_seen": 903872512
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005867683049147443,
+ "loss": 1.3471,
+ "theoretical_loss": 3.713230125096685,
+ "tokens_seen": 903938048
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000586752256770311,
+ "loss": 1.4101,
+ "theoretical_loss": 3.7132042259759324,
+ "tokens_seen": 904003584
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005867362086258777,
+ "loss": 1.4148,
+ "theoretical_loss": 3.71317832925835,
+ "tokens_seen": 904069120
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005867201604814444,
+ "loss": 1.4313,
+ "theoretical_loss": 3.7131524349435416,
+ "tokens_seen": 904134656
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005867041123370111,
+ "loss": 1.398,
+ "theoretical_loss": 3.7131265430311093,
+ "tokens_seen": 904200192
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866880641925778,
+ "loss": 1.3387,
+ "theoretical_loss": 3.713100653520657,
+ "tokens_seen": 904265728
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866720160481445,
+ "loss": 1.521,
+ "theoretical_loss": 3.7130747664117862,
+ "tokens_seen": 904331264
+ },
+ {
+ "epoch": 0.27,
+ "objective/train/docs_used": 306066,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.375870943069458,
+ "objective/train/theoretical_loss": 3.7130488817041023,
+ "objective/train/tokens_used": 924856800,
+ "theoretical_loss": 3.7130488817041023,
+ "tokens_seen": 904396800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866559679037112,
+ "loss": 1.4468,
+ "theoretical_loss": 3.7130488817041023,
+ "tokens_seen": 904396800
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866399197592779,
+ "loss": 1.3345,
+ "theoretical_loss": 3.7130229993972073,
+ "tokens_seen": 904462336
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866238716148446,
+ "loss": 1.3374,
+ "theoretical_loss": 3.712997119490704,
+ "tokens_seen": 904527872
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005866078234704113,
+ "loss": 1.4349,
+ "theoretical_loss": 3.7129712419841976,
+ "tokens_seen": 904593408
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000586591775325978,
+ "loss": 1.3844,
+ "theoretical_loss": 3.7129453668772903,
+ "tokens_seen": 904658944
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005865757271815447,
+ "loss": 1.3579,
+ "theoretical_loss": 3.7129194941695864,
+ "tokens_seen": 904724480
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005865596790371113,
+ "loss": 1.3183,
+ "theoretical_loss": 3.7128936238606896,
+ "tokens_seen": 904790016
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005865436308926781,
+ "loss": 1.3435,
+ "theoretical_loss": 3.712867755950203,
+ "tokens_seen": 904855552
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005865275827482447,
+ "loss": 1.4187,
+ "theoretical_loss": 3.7128418904377316,
+ "tokens_seen": 904921088
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005865115346038115,
+ "loss": 1.4611,
+ "theoretical_loss": 3.712816027322879,
+ "tokens_seen": 904986624
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864954864593781,
+ "loss": 1.2744,
+ "theoretical_loss": 3.71279016660525,
+ "tokens_seen": 905052160
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864794383149449,
+ "loss": 1.3814,
+ "theoretical_loss": 3.712764308284448,
+ "tokens_seen": 905117696
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864633901705115,
+ "loss": 1.4546,
+ "theoretical_loss": 3.7127384523600773,
+ "tokens_seen": 905183232
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864473420260783,
+ "loss": 1.2955,
+ "theoretical_loss": 3.712712598831743,
+ "tokens_seen": 905248768
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864312938816449,
+ "loss": 1.3816,
+ "theoretical_loss": 3.712686747699049,
+ "tokens_seen": 905314304
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005864152457372116,
+ "loss": 1.3562,
+ "theoretical_loss": 3.7126608989616,
+ "tokens_seen": 905379840
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863991975927783,
+ "loss": 1.3533,
+ "theoretical_loss": 3.7126350526190013,
+ "tokens_seen": 905445376
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000586383149448345,
+ "loss": 1.4271,
+ "theoretical_loss": 3.712609208670857,
+ "tokens_seen": 905510912
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863671013039117,
+ "loss": 1.3787,
+ "theoretical_loss": 3.712583367116773,
+ "tokens_seen": 905576448
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863510531594784,
+ "loss": 1.3765,
+ "theoretical_loss": 3.7125575279563527,
+ "tokens_seen": 905641984
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863350050150451,
+ "loss": 1.5112,
+ "theoretical_loss": 3.7125316911892026,
+ "tokens_seen": 905707520
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863189568706118,
+ "loss": 1.3779,
+ "theoretical_loss": 3.7125058568149267,
+ "tokens_seen": 905773056
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005863029087261785,
+ "loss": 1.3798,
+ "theoretical_loss": 3.7124800248331318,
+ "tokens_seen": 905838592
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005862868605817452,
+ "loss": 1.4332,
+ "theoretical_loss": 3.712454195243422,
+ "tokens_seen": 905904128
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005862708124373119,
+ "loss": 1.3842,
+ "theoretical_loss": 3.712428368045403,
+ "tokens_seen": 905969664
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005862547642928786,
+ "loss": 1.3734,
+ "theoretical_loss": 3.7124025432386807,
+ "tokens_seen": 906035200
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005862387161484453,
+ "loss": 1.358,
+ "theoretical_loss": 3.712376720822861,
+ "tokens_seen": 906100736
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000586222668004012,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7123509007975484,
+ "tokens_seen": 906166272
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005862066198595787,
+ "loss": 1.4996,
+ "theoretical_loss": 3.7123250831623498,
+ "tokens_seen": 906231808
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861905717151454,
+ "loss": 1.3434,
+ "theoretical_loss": 3.712299267916871,
+ "tokens_seen": 906297344
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861745235707121,
+ "loss": 1.4045,
+ "theoretical_loss": 3.7122734550607177,
+ "tokens_seen": 906362880
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861584754262789,
+ "loss": 1.4185,
+ "theoretical_loss": 3.712247644593497,
+ "tokens_seen": 906428416
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861424272818455,
+ "loss": 1.3847,
+ "theoretical_loss": 3.7122218365148134,
+ "tokens_seen": 906493952
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861263791374123,
+ "loss": 1.3732,
+ "theoretical_loss": 3.7121960308242743,
+ "tokens_seen": 906559488
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005861103309929789,
+ "loss": 1.2714,
+ "theoretical_loss": 3.7121702275214865,
+ "tokens_seen": 906625024
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860942828485457,
+ "loss": 1.3374,
+ "theoretical_loss": 3.7121444266060553,
+ "tokens_seen": 906690560
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860782347041123,
+ "loss": 1.3808,
+ "theoretical_loss": 3.712118628077588,
+ "tokens_seen": 906756096
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860621865596791,
+ "loss": 1.4362,
+ "theoretical_loss": 3.712092831935691,
+ "tokens_seen": 906821632
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860461384152457,
+ "loss": 1.3248,
+ "theoretical_loss": 3.712067038179972,
+ "tokens_seen": 906887168
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860300902708125,
+ "loss": 1.485,
+ "theoretical_loss": 3.7120412468100366,
+ "tokens_seen": 906952704
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005860140421263791,
+ "loss": 1.3724,
+ "theoretical_loss": 3.7120154578254922,
+ "tokens_seen": 907018240
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859979939819459,
+ "loss": 1.4268,
+ "theoretical_loss": 3.711989671225946,
+ "tokens_seen": 907083776
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859819458375125,
+ "loss": 1.4211,
+ "theoretical_loss": 3.711963887011005,
+ "tokens_seen": 907149312
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859658976930793,
+ "loss": 1.4342,
+ "theoretical_loss": 3.7119381051802764,
+ "tokens_seen": 907214848
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.000585949849548646,
+ "loss": 1.3711,
+ "theoretical_loss": 3.7119123257333677,
+ "tokens_seen": 907280384
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859338014042127,
+ "loss": 1.3671,
+ "theoretical_loss": 3.711886548669886,
+ "tokens_seen": 907345920
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859177532597793,
+ "loss": 1.3794,
+ "theoretical_loss": 3.7118607739894394,
+ "tokens_seen": 907411456
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 0.0005859017051153461,
+ "loss": 1.3348,
+ "theoretical_loss": 3.711835001691635,
+ "tokens_seen": 907476992
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858856569709127,
+ "loss": 1.4623,
+ "theoretical_loss": 3.7118092317760807,
+ "tokens_seen": 907542528
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858696088264795,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7117834642423846,
+ "tokens_seen": 907608064
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 307328,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.252403736114502,
+ "objective/train/theoretical_loss": 3.7117576990901537,
+ "objective/train/tokens_used": 928133600,
+ "theoretical_loss": 3.7117576990901537,
+ "tokens_seen": 907673600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858535606820462,
+ "loss": 1.3462,
+ "theoretical_loss": 3.7117576990901537,
+ "tokens_seen": 907673600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858375125376129,
+ "loss": 1.3259,
+ "theoretical_loss": 3.711731936318997,
+ "tokens_seen": 907739136
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858214643931796,
+ "loss": 1.3657,
+ "theoretical_loss": 3.711706175928522,
+ "tokens_seen": 907804672
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005858054162487463,
+ "loss": 1.538,
+ "theoretical_loss": 3.7116804179183367,
+ "tokens_seen": 907870208
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000585789368104313,
+ "loss": 1.3548,
+ "theoretical_loss": 3.7116546622880495,
+ "tokens_seen": 907935744
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005857733199598797,
+ "loss": 1.3871,
+ "theoretical_loss": 3.711628909037269,
+ "tokens_seen": 908001280
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005857572718154464,
+ "loss": 1.3435,
+ "theoretical_loss": 3.7116031581656035,
+ "tokens_seen": 908066816
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005857412236710131,
+ "loss": 1.3695,
+ "theoretical_loss": 3.7115774096726617,
+ "tokens_seen": 908132352
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005857251755265798,
+ "loss": 1.3902,
+ "theoretical_loss": 3.711551663558052,
+ "tokens_seen": 908197888
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005857091273821465,
+ "loss": 1.2973,
+ "theoretical_loss": 3.7115259198213835,
+ "tokens_seen": 908263424
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005856930792377132,
+ "loss": 1.5021,
+ "theoretical_loss": 3.7115001784622637,
+ "tokens_seen": 908328960
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005856770310932799,
+ "loss": 1.2715,
+ "theoretical_loss": 3.7114744394803036,
+ "tokens_seen": 908394496
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005856609829488466,
+ "loss": 1.3926,
+ "theoretical_loss": 3.711448702875111,
+ "tokens_seen": 908460032
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005856449348044133,
+ "loss": 1.3528,
+ "theoretical_loss": 3.7114229686462945,
+ "tokens_seen": 908525568
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.00058562888665998,
+ "loss": 1.3323,
+ "theoretical_loss": 3.711397236793464,
+ "tokens_seen": 908591104
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005856128385155467,
+ "loss": 1.3253,
+ "theoretical_loss": 3.711371507316229,
+ "tokens_seen": 908656640
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855967903711134,
+ "loss": 1.2965,
+ "theoretical_loss": 3.7113457802141983,
+ "tokens_seen": 908722176
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855807422266801,
+ "loss": 1.3901,
+ "theoretical_loss": 3.7113200554869814,
+ "tokens_seen": 908787712
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855646940822468,
+ "loss": 1.2374,
+ "theoretical_loss": 3.7112943331341883,
+ "tokens_seen": 908853248
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855486459378135,
+ "loss": 1.3409,
+ "theoretical_loss": 3.711268613155428,
+ "tokens_seen": 908918784
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855325977933802,
+ "loss": 1.4335,
+ "theoretical_loss": 3.7112428955503116,
+ "tokens_seen": 908984320
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855165496489469,
+ "loss": 1.4171,
+ "theoretical_loss": 3.711217180318447,
+ "tokens_seen": 909049856
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005855005015045136,
+ "loss": 1.3504,
+ "theoretical_loss": 3.7111914674594457,
+ "tokens_seen": 909115392
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005854844533600803,
+ "loss": 1.3422,
+ "theoretical_loss": 3.7111657569729166,
+ "tokens_seen": 909180928
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000585468405215647,
+ "loss": 1.3639,
+ "theoretical_loss": 3.7111400488584705,
+ "tokens_seen": 909246464
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005854523570712137,
+ "loss": 1.3825,
+ "theoretical_loss": 3.7111143431157174,
+ "tokens_seen": 909312000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005854363089267804,
+ "loss": 1.4059,
+ "theoretical_loss": 3.7110886397442675,
+ "tokens_seen": 909377536
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005854202607823471,
+ "loss": 1.3736,
+ "theoretical_loss": 3.7110629387437317,
+ "tokens_seen": 909443072
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005854042126379138,
+ "loss": 1.3197,
+ "theoretical_loss": 3.7110372401137193,
+ "tokens_seen": 909508608
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005853881644934805,
+ "loss": 1.4951,
+ "theoretical_loss": 3.711011543853842,
+ "tokens_seen": 909574144
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005853721163490472,
+ "loss": 1.3901,
+ "theoretical_loss": 3.71098584996371,
+ "tokens_seen": 909639680
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005853560682046139,
+ "loss": 1.4033,
+ "theoretical_loss": 3.7109601584429344,
+ "tokens_seen": 909705216
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005853400200601806,
+ "loss": 1.3312,
+ "theoretical_loss": 3.7109344692911255,
+ "tokens_seen": 909770752
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005853239719157473,
+ "loss": 1.4203,
+ "theoretical_loss": 3.710908782507895,
+ "tokens_seen": 909836288
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000585307923771314,
+ "loss": 1.4514,
+ "theoretical_loss": 3.710883098092853,
+ "tokens_seen": 909901824
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852918756268807,
+ "loss": 1.4127,
+ "theoretical_loss": 3.7108574160456107,
+ "tokens_seen": 909967360
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852758274824474,
+ "loss": 1.4031,
+ "theoretical_loss": 3.7108317363657797,
+ "tokens_seen": 910032896
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852597793380141,
+ "loss": 1.407,
+ "theoretical_loss": 3.710806059052972,
+ "tokens_seen": 910098432
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852437311935808,
+ "loss": 1.3883,
+ "theoretical_loss": 3.7107803841067977,
+ "tokens_seen": 910163968
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852276830491475,
+ "loss": 1.3155,
+ "theoretical_loss": 3.710754711526869,
+ "tokens_seen": 910229504
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005852116349047142,
+ "loss": 1.343,
+ "theoretical_loss": 3.7107290413127973,
+ "tokens_seen": 910295040
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005851955867602809,
+ "loss": 1.3359,
+ "theoretical_loss": 3.7107033734641943,
+ "tokens_seen": 910360576
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005851795386158476,
+ "loss": 1.356,
+ "theoretical_loss": 3.710677707980672,
+ "tokens_seen": 910426112
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005851634904714142,
+ "loss": 1.3019,
+ "theoretical_loss": 3.710652044861842,
+ "tokens_seen": 910491648
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000585147442326981,
+ "loss": 1.3407,
+ "theoretical_loss": 3.7106263841073153,
+ "tokens_seen": 910557184
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005851313941825476,
+ "loss": 1.4251,
+ "theoretical_loss": 3.7106007257167057,
+ "tokens_seen": 910622720
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005851153460381144,
+ "loss": 1.4142,
+ "theoretical_loss": 3.7105750696896247,
+ "tokens_seen": 910688256
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000585099297893681,
+ "loss": 1.4038,
+ "theoretical_loss": 3.710549416025684,
+ "tokens_seen": 910753792
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850832497492478,
+ "loss": 1.4423,
+ "theoretical_loss": 3.7105237647244964,
+ "tokens_seen": 910819328
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850672016048144,
+ "loss": 1.4541,
+ "theoretical_loss": 3.710498115785674,
+ "tokens_seen": 910884864
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 308239,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.30680251121521,
+ "objective/train/theoretical_loss": 3.7104724692088293,
+ "objective/train/tokens_used": 931410400,
+ "theoretical_loss": 3.7104724692088293,
+ "tokens_seen": 910950400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850511534603812,
+ "loss": 1.386,
+ "theoretical_loss": 3.7104724692088293,
+ "tokens_seen": 910950400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850351053159478,
+ "loss": 1.4251,
+ "theoretical_loss": 3.7104468249935754,
+ "tokens_seen": 911015936
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850190571715145,
+ "loss": 1.4013,
+ "theoretical_loss": 3.710421183139524,
+ "tokens_seen": 911081472
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005850030090270812,
+ "loss": 1.2597,
+ "theoretical_loss": 3.710395543646289,
+ "tokens_seen": 911147008
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005849869608826479,
+ "loss": 1.4452,
+ "theoretical_loss": 3.710369906513483,
+ "tokens_seen": 911212544
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005849709127382146,
+ "loss": 1.4181,
+ "theoretical_loss": 3.710344271740718,
+ "tokens_seen": 911278080
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005849548645937814,
+ "loss": 1.4182,
+ "theoretical_loss": 3.710318639327608,
+ "tokens_seen": 911343616
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584938816449348,
+ "loss": 1.3231,
+ "theoretical_loss": 3.7102930092737663,
+ "tokens_seen": 911409152
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005849227683049148,
+ "loss": 1.3125,
+ "theoretical_loss": 3.710267381578805,
+ "tokens_seen": 911474688
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005849067201604814,
+ "loss": 1.3279,
+ "theoretical_loss": 3.710241756242339,
+ "tokens_seen": 911540224
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005848906720160482,
+ "loss": 1.3827,
+ "theoretical_loss": 3.7102161332639794,
+ "tokens_seen": 911605760
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005848746238716148,
+ "loss": 1.2973,
+ "theoretical_loss": 3.7101905126433423,
+ "tokens_seen": 911671296
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005848585757271816,
+ "loss": 1.4018,
+ "theoretical_loss": 3.7101648943800396,
+ "tokens_seen": 911736832
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005848425275827482,
+ "loss": 1.4474,
+ "theoretical_loss": 3.7101392784736857,
+ "tokens_seen": 911802368
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584826479438315,
+ "loss": 1.3856,
+ "theoretical_loss": 3.7101136649238944,
+ "tokens_seen": 911867904
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005848104312938816,
+ "loss": 1.435,
+ "theoretical_loss": 3.7100880537302787,
+ "tokens_seen": 911933440
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005847943831494484,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7100624448924533,
+ "tokens_seen": 911998976
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584778335005015,
+ "loss": 1.378,
+ "theoretical_loss": 3.710036838410032,
+ "tokens_seen": 912064512
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005847622868605818,
+ "loss": 1.3083,
+ "theoretical_loss": 3.7100112342826295,
+ "tokens_seen": 912130048
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005847462387161484,
+ "loss": 1.4158,
+ "theoretical_loss": 3.7099856325098592,
+ "tokens_seen": 912195584
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005847301905717152,
+ "loss": 1.3268,
+ "theoretical_loss": 3.7099600330913356,
+ "tokens_seen": 912261120
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005847141424272818,
+ "loss": 1.412,
+ "theoretical_loss": 3.709934436026674,
+ "tokens_seen": 912326656
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005846980942828486,
+ "loss": 1.3536,
+ "theoretical_loss": 3.709908841315488,
+ "tokens_seen": 912392192
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005846820461384152,
+ "loss": 1.4512,
+ "theoretical_loss": 3.709883248957392,
+ "tokens_seen": 912457728
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584665997993982,
+ "loss": 1.3793,
+ "theoretical_loss": 3.709857658952001,
+ "tokens_seen": 912523264
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005846499498495486,
+ "loss": 1.4338,
+ "theoretical_loss": 3.7098320712989303,
+ "tokens_seen": 912588800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005846339017051154,
+ "loss": 1.3351,
+ "theoretical_loss": 3.7098064859977944,
+ "tokens_seen": 912654336
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584617853560682,
+ "loss": 1.379,
+ "theoretical_loss": 3.7097809030482076,
+ "tokens_seen": 912719872
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005846018054162488,
+ "loss": 1.3192,
+ "theoretical_loss": 3.709755322449786,
+ "tokens_seen": 912785408
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005845857572718154,
+ "loss": 1.3225,
+ "theoretical_loss": 3.7097297442021437,
+ "tokens_seen": 912850944
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005845697091273822,
+ "loss": 1.3692,
+ "theoretical_loss": 3.7097041683048966,
+ "tokens_seen": 912916480
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005845536609829488,
+ "loss": 1.3323,
+ "theoretical_loss": 3.7096785947576603,
+ "tokens_seen": 912982016
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005845376128385156,
+ "loss": 1.4171,
+ "theoretical_loss": 3.7096530235600493,
+ "tokens_seen": 913047552
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005845215646940822,
+ "loss": 1.4956,
+ "theoretical_loss": 3.70962745471168,
+ "tokens_seen": 913113088
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584505516549649,
+ "loss": 1.3194,
+ "theoretical_loss": 3.709601888212167,
+ "tokens_seen": 913178624
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005844894684052156,
+ "loss": 1.3993,
+ "theoretical_loss": 3.7095763240611275,
+ "tokens_seen": 913244160
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005844734202607824,
+ "loss": 1.4061,
+ "theoretical_loss": 3.7095507622581754,
+ "tokens_seen": 913309696
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584457372116349,
+ "loss": 1.3649,
+ "theoretical_loss": 3.7095252028029275,
+ "tokens_seen": 913375232
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005844413239719158,
+ "loss": 1.3078,
+ "theoretical_loss": 3.7094996456950002,
+ "tokens_seen": 913440768
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005844252758274825,
+ "loss": 1.419,
+ "theoretical_loss": 3.709474090934009,
+ "tokens_seen": 913506304
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005844092276830492,
+ "loss": 1.4194,
+ "theoretical_loss": 3.70944853851957,
+ "tokens_seen": 913571840
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005843931795386159,
+ "loss": 1.3092,
+ "theoretical_loss": 3.7094229884512995,
+ "tokens_seen": 913637376
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005843771313941826,
+ "loss": 1.36,
+ "theoretical_loss": 3.7093974407288135,
+ "tokens_seen": 913702912
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005843610832497493,
+ "loss": 1.4087,
+ "theoretical_loss": 3.7093718953517287,
+ "tokens_seen": 913768448
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584345035105316,
+ "loss": 1.3081,
+ "theoretical_loss": 3.7093463523196624,
+ "tokens_seen": 913833984
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005843289869608827,
+ "loss": 1.4695,
+ "theoretical_loss": 3.70932081163223,
+ "tokens_seen": 913899520
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005843129388164494,
+ "loss": 1.3757,
+ "theoretical_loss": 3.709295273289048,
+ "tokens_seen": 913965056
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842968906720161,
+ "loss": 1.3217,
+ "theoretical_loss": 3.709269737289734,
+ "tokens_seen": 914030592
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842808425275828,
+ "loss": 1.4032,
+ "theoretical_loss": 3.7092442036339044,
+ "tokens_seen": 914096128
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842647943831495,
+ "loss": 1.3886,
+ "theoretical_loss": 3.7092186723211773,
+ "tokens_seen": 914161664
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 309446,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3680208921432495,
+ "objective/train/theoretical_loss": 3.7091931433511673,
+ "objective/train/tokens_used": 934687200,
+ "theoretical_loss": 3.7091931433511673,
+ "tokens_seen": 914227200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842487462387162,
+ "loss": 1.3914,
+ "theoretical_loss": 3.7091931433511673,
+ "tokens_seen": 914227200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842326980942829,
+ "loss": 1.4397,
+ "theoretical_loss": 3.7091676167234944,
+ "tokens_seen": 914292736
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842166499498496,
+ "loss": 1.4257,
+ "theoretical_loss": 3.7091420924377734,
+ "tokens_seen": 914358272
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005842006018054163,
+ "loss": 1.4036,
+ "theoretical_loss": 3.709116570493623,
+ "tokens_seen": 914423808
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000584184553660983,
+ "loss": 1.3334,
+ "theoretical_loss": 3.70909105089066,
+ "tokens_seen": 914489344
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005841685055165497,
+ "loss": 1.3621,
+ "theoretical_loss": 3.709065533628502,
+ "tokens_seen": 914554880
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005841524573721164,
+ "loss": 1.3287,
+ "theoretical_loss": 3.7090400187067667,
+ "tokens_seen": 914620416
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005841364092276831,
+ "loss": 1.3724,
+ "theoretical_loss": 3.709014506125072,
+ "tokens_seen": 914685952
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005841203610832498,
+ "loss": 1.3539,
+ "theoretical_loss": 3.708988995883035,
+ "tokens_seen": 914751488
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005841043129388165,
+ "loss": 1.4639,
+ "theoretical_loss": 3.7089634879802738,
+ "tokens_seen": 914817024
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005840882647943832,
+ "loss": 1.4849,
+ "theoretical_loss": 3.7089379824164066,
+ "tokens_seen": 914882560
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005840722166499499,
+ "loss": 1.3807,
+ "theoretical_loss": 3.708912479191051,
+ "tokens_seen": 914948096
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005840561685055166,
+ "loss": 1.4203,
+ "theoretical_loss": 3.708886978303825,
+ "tokens_seen": 915013632
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005840401203610833,
+ "loss": 1.447,
+ "theoretical_loss": 3.708861479754348,
+ "tokens_seen": 915079168
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.00058402407221665,
+ "loss": 1.3096,
+ "theoretical_loss": 3.708835983542237,
+ "tokens_seen": 915144704
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005840080240722167,
+ "loss": 1.375,
+ "theoretical_loss": 3.708810489667111,
+ "tokens_seen": 915210240
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839919759277834,
+ "loss": 1.3331,
+ "theoretical_loss": 3.7087849981285883,
+ "tokens_seen": 915275776
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839759277833501,
+ "loss": 1.307,
+ "theoretical_loss": 3.7087595089262875,
+ "tokens_seen": 915341312
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839598796389168,
+ "loss": 1.395,
+ "theoretical_loss": 3.7087340220598266,
+ "tokens_seen": 915406848
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839438314944835,
+ "loss": 1.3897,
+ "theoretical_loss": 3.7087085375288256,
+ "tokens_seen": 915472384
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839277833500502,
+ "loss": 1.4486,
+ "theoretical_loss": 3.708683055332902,
+ "tokens_seen": 915537920
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005839117352056169,
+ "loss": 1.4165,
+ "theoretical_loss": 3.7086575754716757,
+ "tokens_seen": 915603456
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005838956870611836,
+ "loss": 1.4194,
+ "theoretical_loss": 3.7086320979447653,
+ "tokens_seen": 915668992
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005838796389167503,
+ "loss": 1.4557,
+ "theoretical_loss": 3.70860662275179,
+ "tokens_seen": 915734528
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000583863590772317,
+ "loss": 1.3824,
+ "theoretical_loss": 3.7085811498923693,
+ "tokens_seen": 915800064
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005838475426278837,
+ "loss": 1.4194,
+ "theoretical_loss": 3.7085556793661216,
+ "tokens_seen": 915865600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005838314944834504,
+ "loss": 1.4242,
+ "theoretical_loss": 3.708530211172667,
+ "tokens_seen": 915931136
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005838154463390171,
+ "loss": 1.3427,
+ "theoretical_loss": 3.7085047453116244,
+ "tokens_seen": 915996672
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837993981945838,
+ "loss": 1.3287,
+ "theoretical_loss": 3.7084792817826147,
+ "tokens_seen": 916062208
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837833500501505,
+ "loss": 1.4027,
+ "theoretical_loss": 3.7084538205852553,
+ "tokens_seen": 916127744
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837673019057172,
+ "loss": 1.3169,
+ "theoretical_loss": 3.708428361719168,
+ "tokens_seen": 916193280
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837512537612839,
+ "loss": 1.3977,
+ "theoretical_loss": 3.7084029051839713,
+ "tokens_seen": 916258816
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837352056168505,
+ "loss": 1.4781,
+ "theoretical_loss": 3.7083774509792855,
+ "tokens_seen": 916324352
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837191574724173,
+ "loss": 1.4445,
+ "theoretical_loss": 3.708351999104731,
+ "tokens_seen": 916389888
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005837031093279839,
+ "loss": 1.4419,
+ "theoretical_loss": 3.7083265495599274,
+ "tokens_seen": 916455424
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836870611835507,
+ "loss": 1.4046,
+ "theoretical_loss": 3.708301102344495,
+ "tokens_seen": 916520960
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836710130391173,
+ "loss": 1.383,
+ "theoretical_loss": 3.7082756574580538,
+ "tokens_seen": 916586496
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836549648946841,
+ "loss": 1.3516,
+ "theoretical_loss": 3.7082502149002243,
+ "tokens_seen": 916652032
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836389167502507,
+ "loss": 1.2977,
+ "theoretical_loss": 3.7082247746706276,
+ "tokens_seen": 916717568
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836228686058175,
+ "loss": 1.3344,
+ "theoretical_loss": 3.7081993367688835,
+ "tokens_seen": 916783104
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005836068204613841,
+ "loss": 1.4308,
+ "theoretical_loss": 3.708173901194612,
+ "tokens_seen": 916848640
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835907723169509,
+ "loss": 1.4316,
+ "theoretical_loss": 3.708148467947435,
+ "tokens_seen": 916914176
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835747241725175,
+ "loss": 1.3856,
+ "theoretical_loss": 3.708123037026973,
+ "tokens_seen": 916979712
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835586760280843,
+ "loss": 1.3514,
+ "theoretical_loss": 3.7080976084328467,
+ "tokens_seen": 917045248
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835426278836509,
+ "loss": 1.2815,
+ "theoretical_loss": 3.7080721821646767,
+ "tokens_seen": 917110784
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835265797392177,
+ "loss": 1.3831,
+ "theoretical_loss": 3.7080467582220855,
+ "tokens_seen": 917176320
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005835105315947843,
+ "loss": 1.4292,
+ "theoretical_loss": 3.708021336604692,
+ "tokens_seen": 917241856
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834944834503511,
+ "loss": 1.4708,
+ "theoretical_loss": 3.707995917312119,
+ "tokens_seen": 917307392
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834784353059177,
+ "loss": 1.4295,
+ "theoretical_loss": 3.7079705003439876,
+ "tokens_seen": 917372928
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834623871614845,
+ "loss": 1.3694,
+ "theoretical_loss": 3.7079450856999188,
+ "tokens_seen": 917438464
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 310499,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3221814632415771,
+ "objective/train/theoretical_loss": 3.7079196733795348,
+ "objective/train/tokens_used": 937964000,
+ "theoretical_loss": 3.7079196733795348,
+ "tokens_seen": 917504000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834463390170511,
+ "loss": 1.3723,
+ "theoretical_loss": 3.7079196733795348,
+ "tokens_seen": 917504000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834302908726179,
+ "loss": 1.3693,
+ "theoretical_loss": 3.7078942633824563,
+ "tokens_seen": 917569536
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005834142427281845,
+ "loss": 1.3665,
+ "theoretical_loss": 3.7078688557083055,
+ "tokens_seen": 917635072
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833981945837513,
+ "loss": 1.4111,
+ "theoretical_loss": 3.7078434503567044,
+ "tokens_seen": 917700608
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833821464393179,
+ "loss": 1.415,
+ "theoretical_loss": 3.707818047327274,
+ "tokens_seen": 917766144
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833660982948847,
+ "loss": 1.4638,
+ "theoretical_loss": 3.7077926466196374,
+ "tokens_seen": 917831680
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833500501504513,
+ "loss": 1.3468,
+ "theoretical_loss": 3.7077672482334156,
+ "tokens_seen": 917897216
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833340020060181,
+ "loss": 1.2463,
+ "theoretical_loss": 3.7077418521682315,
+ "tokens_seen": 917962752
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833179538615847,
+ "loss": 1.2948,
+ "theoretical_loss": 3.7077164584237066,
+ "tokens_seen": 918028288
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005833019057171515,
+ "loss": 1.4244,
+ "theoretical_loss": 3.7076910669994634,
+ "tokens_seen": 918093824
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005832858575727181,
+ "loss": 1.4251,
+ "theoretical_loss": 3.7076656778951245,
+ "tokens_seen": 918159360
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005832698094282849,
+ "loss": 1.468,
+ "theoretical_loss": 3.7076402911103123,
+ "tokens_seen": 918224896
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005832537612838515,
+ "loss": 1.4648,
+ "theoretical_loss": 3.7076149066446495,
+ "tokens_seen": 918290432
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005832377131394183,
+ "loss": 1.3711,
+ "theoretical_loss": 3.707589524497759,
+ "tokens_seen": 918355968
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000583221664994985,
+ "loss": 1.3111,
+ "theoretical_loss": 3.707564144669263,
+ "tokens_seen": 918421504
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005832056168505517,
+ "loss": 1.3687,
+ "theoretical_loss": 3.707538767158784,
+ "tokens_seen": 918487040
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831895687061183,
+ "loss": 1.3913,
+ "theoretical_loss": 3.7075133919659455,
+ "tokens_seen": 918552576
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831735205616851,
+ "loss": 1.3898,
+ "theoretical_loss": 3.7074880190903707,
+ "tokens_seen": 918618112
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831574724172517,
+ "loss": 1.3975,
+ "theoretical_loss": 3.707462648531682,
+ "tokens_seen": 918683648
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831414242728185,
+ "loss": 1.4459,
+ "theoretical_loss": 3.7074372802895033,
+ "tokens_seen": 918749184
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831253761283851,
+ "loss": 1.4212,
+ "theoretical_loss": 3.7074119143634574,
+ "tokens_seen": 918814720
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005831093279839519,
+ "loss": 1.3909,
+ "theoretical_loss": 3.707386550753168,
+ "tokens_seen": 918880256
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005830932798395186,
+ "loss": 1.3345,
+ "theoretical_loss": 3.7073611894582585,
+ "tokens_seen": 918945792
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005830772316950853,
+ "loss": 1.3417,
+ "theoretical_loss": 3.707335830478352,
+ "tokens_seen": 919011328
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000583061183550652,
+ "loss": 1.398,
+ "theoretical_loss": 3.707310473813072,
+ "tokens_seen": 919076864
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005830451354062187,
+ "loss": 1.3413,
+ "theoretical_loss": 3.707285119462043,
+ "tokens_seen": 919142400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005830290872617854,
+ "loss": 1.4258,
+ "theoretical_loss": 3.7072597674248886,
+ "tokens_seen": 919207936
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005830130391173521,
+ "loss": 1.3826,
+ "theoretical_loss": 3.707234417701232,
+ "tokens_seen": 919273472
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829969909729188,
+ "loss": 1.4574,
+ "theoretical_loss": 3.707209070290698,
+ "tokens_seen": 919339008
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829809428284855,
+ "loss": 1.4397,
+ "theoretical_loss": 3.70718372519291,
+ "tokens_seen": 919404544
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829648946840522,
+ "loss": 1.3371,
+ "theoretical_loss": 3.707158382407493,
+ "tokens_seen": 919470080
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829488465396189,
+ "loss": 1.4102,
+ "theoretical_loss": 3.7071330419340702,
+ "tokens_seen": 919535616
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829327983951856,
+ "loss": 1.4334,
+ "theoretical_loss": 3.707107703772267,
+ "tokens_seen": 919601152
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005829167502507523,
+ "loss": 1.4841,
+ "theoretical_loss": 3.7070823679217066,
+ "tokens_seen": 919666688
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582900702106319,
+ "loss": 1.439,
+ "theoretical_loss": 3.7070570343820144,
+ "tokens_seen": 919732224
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828846539618857,
+ "loss": 1.3562,
+ "theoretical_loss": 3.707031703152815,
+ "tokens_seen": 919797760
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828686058174524,
+ "loss": 1.4295,
+ "theoretical_loss": 3.7070063742337322,
+ "tokens_seen": 919863296
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828525576730191,
+ "loss": 1.3745,
+ "theoretical_loss": 3.706981047624392,
+ "tokens_seen": 919928832
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828365095285858,
+ "loss": 1.3858,
+ "theoretical_loss": 3.706955723324418,
+ "tokens_seen": 919994368
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828204613841525,
+ "loss": 1.3942,
+ "theoretical_loss": 3.706930401333436,
+ "tokens_seen": 920059904
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005828044132397192,
+ "loss": 1.3518,
+ "theoretical_loss": 3.7069050816510707,
+ "tokens_seen": 920125440
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005827883650952859,
+ "loss": 1.4327,
+ "theoretical_loss": 3.7068797642769478,
+ "tokens_seen": 920190976
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005827723169508526,
+ "loss": 1.4311,
+ "theoretical_loss": 3.7068544492106916,
+ "tokens_seen": 920256512
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005827562688064193,
+ "loss": 1.4935,
+ "theoretical_loss": 3.706829136451928,
+ "tokens_seen": 920322048
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582740220661986,
+ "loss": 1.3996,
+ "theoretical_loss": 3.7068038260002814,
+ "tokens_seen": 920387584
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005827241725175527,
+ "loss": 1.3055,
+ "theoretical_loss": 3.706778517855379,
+ "tokens_seen": 920453120
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005827081243731194,
+ "loss": 1.3009,
+ "theoretical_loss": 3.706753212016845,
+ "tokens_seen": 920518656
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826920762286861,
+ "loss": 1.3855,
+ "theoretical_loss": 3.7067279084843054,
+ "tokens_seen": 920584192
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826760280842528,
+ "loss": 1.4394,
+ "theoretical_loss": 3.7067026072573857,
+ "tokens_seen": 920649728
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826599799398195,
+ "loss": 1.3881,
+ "theoretical_loss": 3.7066773083357125,
+ "tokens_seen": 920715264
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 311765,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3808051347732544,
+ "objective/train/theoretical_loss": 3.7066520117189103,
+ "objective/train/tokens_used": 941240800,
+ "theoretical_loss": 3.7066520117189103,
+ "tokens_seen": 920780800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826439317953862,
+ "loss": 1.3491,
+ "theoretical_loss": 3.7066520117189103,
+ "tokens_seen": 920780800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826278836509529,
+ "loss": 1.421,
+ "theoretical_loss": 3.706626717406607,
+ "tokens_seen": 920846336
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005826118355065196,
+ "loss": 1.2673,
+ "theoretical_loss": 3.7066014253984267,
+ "tokens_seen": 920911872
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825957873620863,
+ "loss": 1.4163,
+ "theoretical_loss": 3.7065761356939966,
+ "tokens_seen": 920977408
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825797392176531,
+ "loss": 1.415,
+ "theoretical_loss": 3.7065508482929435,
+ "tokens_seen": 921042944
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825636910732197,
+ "loss": 1.3242,
+ "theoretical_loss": 3.7065255631948926,
+ "tokens_seen": 921108480
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825476429287865,
+ "loss": 1.3071,
+ "theoretical_loss": 3.706500280399471,
+ "tokens_seen": 921174016
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825315947843531,
+ "loss": 1.3735,
+ "theoretical_loss": 3.7064749999063045,
+ "tokens_seen": 921239552
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005825155466399199,
+ "loss": 1.3342,
+ "theoretical_loss": 3.7064497217150207,
+ "tokens_seen": 921305088
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824994984954865,
+ "loss": 1.3412,
+ "theoretical_loss": 3.7064244458252453,
+ "tokens_seen": 921370624
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824834503510532,
+ "loss": 1.4254,
+ "theoretical_loss": 3.706399172236606,
+ "tokens_seen": 921436160
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824674022066199,
+ "loss": 1.4699,
+ "theoretical_loss": 3.706373900948729,
+ "tokens_seen": 921501696
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824513540621866,
+ "loss": 1.3505,
+ "theoretical_loss": 3.7063486319612418,
+ "tokens_seen": 921567232
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824353059177533,
+ "loss": 1.451,
+ "theoretical_loss": 3.7063233652737706,
+ "tokens_seen": 921632768
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.00058241925777332,
+ "loss": 1.3264,
+ "theoretical_loss": 3.7062981008859435,
+ "tokens_seen": 921698304
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005824032096288867,
+ "loss": 1.3487,
+ "theoretical_loss": 3.7062728387973873,
+ "tokens_seen": 921763840
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823871614844534,
+ "loss": 1.3578,
+ "theoretical_loss": 3.7062475790077283,
+ "tokens_seen": 921829376
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823711133400201,
+ "loss": 1.3563,
+ "theoretical_loss": 3.706222321516596,
+ "tokens_seen": 921894912
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823550651955868,
+ "loss": 1.4439,
+ "theoretical_loss": 3.7061970663236155,
+ "tokens_seen": 921960448
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823390170511534,
+ "loss": 1.3973,
+ "theoretical_loss": 3.7061718134284165,
+ "tokens_seen": 922025984
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823229689067202,
+ "loss": 1.2689,
+ "theoretical_loss": 3.706146562830625,
+ "tokens_seen": 922091520
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005823069207622868,
+ "loss": 1.3997,
+ "theoretical_loss": 3.70612131452987,
+ "tokens_seen": 922157056
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005822908726178536,
+ "loss": 1.4652,
+ "theoretical_loss": 3.7060960685257784,
+ "tokens_seen": 922222592
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005822748244734202,
+ "loss": 1.339,
+ "theoretical_loss": 3.7060708248179783,
+ "tokens_seen": 922288128
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582258776328987,
+ "loss": 1.4247,
+ "theoretical_loss": 3.7060455834060977,
+ "tokens_seen": 922353664
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005822427281845536,
+ "loss": 1.4533,
+ "theoretical_loss": 3.7060203442897652,
+ "tokens_seen": 922419200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005822266800401204,
+ "loss": 1.3602,
+ "theoretical_loss": 3.7059951074686084,
+ "tokens_seen": 922484736
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582210631895687,
+ "loss": 1.4916,
+ "theoretical_loss": 3.705969872942256,
+ "tokens_seen": 922550272
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821945837512538,
+ "loss": 1.4019,
+ "theoretical_loss": 3.705944640710335,
+ "tokens_seen": 922615808
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821785356068204,
+ "loss": 1.4527,
+ "theoretical_loss": 3.7059194107724758,
+ "tokens_seen": 922681344
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821624874623872,
+ "loss": 1.3166,
+ "theoretical_loss": 3.7058941831283057,
+ "tokens_seen": 922746880
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821464393179538,
+ "loss": 1.3701,
+ "theoretical_loss": 3.7058689577774535,
+ "tokens_seen": 922812416
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821303911735206,
+ "loss": 1.4912,
+ "theoretical_loss": 3.705843734719548,
+ "tokens_seen": 922877952
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005821143430290872,
+ "loss": 1.3856,
+ "theoretical_loss": 3.7058185139542177,
+ "tokens_seen": 922943488
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582098294884654,
+ "loss": 1.3559,
+ "theoretical_loss": 3.705793295481092,
+ "tokens_seen": 923009024
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005820822467402206,
+ "loss": 1.2921,
+ "theoretical_loss": 3.7057680792997996,
+ "tokens_seen": 923074560
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005820661985957874,
+ "loss": 1.4317,
+ "theoretical_loss": 3.7057428654099693,
+ "tokens_seen": 923140096
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000582050150451354,
+ "loss": 1.3484,
+ "theoretical_loss": 3.70571765381123,
+ "tokens_seen": 923205632
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005820341023069208,
+ "loss": 1.4788,
+ "theoretical_loss": 3.705692444503211,
+ "tokens_seen": 923271168
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005820180541624874,
+ "loss": 1.3357,
+ "theoretical_loss": 3.7056672374855424,
+ "tokens_seen": 923336704
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005820020060180542,
+ "loss": 1.2888,
+ "theoretical_loss": 3.7056420327578534,
+ "tokens_seen": 923402240
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005819859578736208,
+ "loss": 1.355,
+ "theoretical_loss": 3.705616830319773,
+ "tokens_seen": 923467776
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005819699097291876,
+ "loss": 1.3329,
+ "theoretical_loss": 3.7055916301709297,
+ "tokens_seen": 923533312
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005819538615847542,
+ "loss": 1.3257,
+ "theoretical_loss": 3.705566432310955,
+ "tokens_seen": 923598848
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581937813440321,
+ "loss": 1.3043,
+ "theoretical_loss": 3.705541236739478,
+ "tokens_seen": 923664384
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005819217652958876,
+ "loss": 1.4197,
+ "theoretical_loss": 3.7055160434561283,
+ "tokens_seen": 923729920
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005819057171514544,
+ "loss": 1.372,
+ "theoretical_loss": 3.7054908524605357,
+ "tokens_seen": 923795456
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581889669007021,
+ "loss": 1.3131,
+ "theoretical_loss": 3.70546566375233,
+ "tokens_seen": 923860992
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005818736208625878,
+ "loss": 1.4444,
+ "theoretical_loss": 3.705440477331142,
+ "tokens_seen": 923926528
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005818575727181544,
+ "loss": 1.3753,
+ "theoretical_loss": 3.7054152931966016,
+ "tokens_seen": 923992064
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 312607,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3203315734863281,
+ "objective/train/theoretical_loss": 3.705390111348339,
+ "objective/train/tokens_used": 944517600,
+ "theoretical_loss": 3.705390111348339,
+ "tokens_seen": 924057600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005818415245737212,
+ "loss": 1.3776,
+ "theoretical_loss": 3.705390111348339,
+ "tokens_seen": 924057600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005818254764292878,
+ "loss": 1.4551,
+ "theoretical_loss": 3.705364931785984,
+ "tokens_seen": 924123136
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005818094282848546,
+ "loss": 1.1539,
+ "theoretical_loss": 3.7053397545091675,
+ "tokens_seen": 924188672
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005817933801404212,
+ "loss": 1.3143,
+ "theoretical_loss": 3.7053145795175197,
+ "tokens_seen": 924254208
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581777331995988,
+ "loss": 1.2291,
+ "theoretical_loss": 3.705289406810672,
+ "tokens_seen": 924319744
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005817612838515546,
+ "loss": 1.372,
+ "theoretical_loss": 3.7052642363882535,
+ "tokens_seen": 924385280
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005817452357071214,
+ "loss": 1.2999,
+ "theoretical_loss": 3.7052390682498966,
+ "tokens_seen": 924450816
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581729187562688,
+ "loss": 1.386,
+ "theoretical_loss": 3.705213902395231,
+ "tokens_seen": 924516352
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005817131394182548,
+ "loss": 1.441,
+ "theoretical_loss": 3.705188738823889,
+ "tokens_seen": 924581888
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816970912738215,
+ "loss": 1.3598,
+ "theoretical_loss": 3.7051635775354996,
+ "tokens_seen": 924647424
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816810431293882,
+ "loss": 1.2233,
+ "theoretical_loss": 3.705138418529695,
+ "tokens_seen": 924712960
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816649949849549,
+ "loss": 1.2805,
+ "theoretical_loss": 3.7051132618061073,
+ "tokens_seen": 924778496
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816489468405216,
+ "loss": 1.4946,
+ "theoretical_loss": 3.705088107364366,
+ "tokens_seen": 924844032
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816328986960883,
+ "loss": 1.3883,
+ "theoretical_loss": 3.7050629552041032,
+ "tokens_seen": 924909568
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581616850551655,
+ "loss": 1.3847,
+ "theoretical_loss": 3.7050378053249506,
+ "tokens_seen": 924975104
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005816008024072217,
+ "loss": 1.2696,
+ "theoretical_loss": 3.70501265772654,
+ "tokens_seen": 925040640
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815847542627884,
+ "loss": 1.3819,
+ "theoretical_loss": 3.7049875124085014,
+ "tokens_seen": 925106176
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815687061183551,
+ "loss": 1.4409,
+ "theoretical_loss": 3.7049623693704685,
+ "tokens_seen": 925171712
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815526579739218,
+ "loss": 1.4033,
+ "theoretical_loss": 3.7049372286120716,
+ "tokens_seen": 925237248
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815366098294885,
+ "loss": 1.3403,
+ "theoretical_loss": 3.704912090132944,
+ "tokens_seen": 925302784
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815205616850552,
+ "loss": 1.3072,
+ "theoretical_loss": 3.704886953932716,
+ "tokens_seen": 925368320
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005815045135406219,
+ "loss": 1.3655,
+ "theoretical_loss": 3.704861820011021,
+ "tokens_seen": 925433856
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005814884653961886,
+ "loss": 1.3922,
+ "theoretical_loss": 3.70483668836749,
+ "tokens_seen": 925499392
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005814724172517553,
+ "loss": 1.3912,
+ "theoretical_loss": 3.7048115590017563,
+ "tokens_seen": 925564928
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581456369107322,
+ "loss": 1.397,
+ "theoretical_loss": 3.704786431913451,
+ "tokens_seen": 925630464
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005814403209628887,
+ "loss": 1.3192,
+ "theoretical_loss": 3.7047613071022076,
+ "tokens_seen": 925696000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005814242728184554,
+ "loss": 1.4069,
+ "theoretical_loss": 3.7047361845676585,
+ "tokens_seen": 925761536
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005814082246740221,
+ "loss": 1.4717,
+ "theoretical_loss": 3.704711064309435,
+ "tokens_seen": 925827072
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005813921765295888,
+ "loss": 1.3609,
+ "theoretical_loss": 3.704685946327171,
+ "tokens_seen": 925892608
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005813761283851556,
+ "loss": 1.3113,
+ "theoretical_loss": 3.704660830620499,
+ "tokens_seen": 925958144
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005813600802407222,
+ "loss": 1.4137,
+ "theoretical_loss": 3.704635717189051,
+ "tokens_seen": 926023680
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581344032096289,
+ "loss": 1.3567,
+ "theoretical_loss": 3.7046106060324613,
+ "tokens_seen": 926089216
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005813279839518556,
+ "loss": 1.4285,
+ "theoretical_loss": 3.7045854971503616,
+ "tokens_seen": 926154752
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005813119358074224,
+ "loss": 1.3764,
+ "theoretical_loss": 3.704560390542385,
+ "tokens_seen": 926220288
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581295887662989,
+ "loss": 1.2782,
+ "theoretical_loss": 3.7045352862081655,
+ "tokens_seen": 926285824
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005812798395185558,
+ "loss": 1.3771,
+ "theoretical_loss": 3.7045101841473365,
+ "tokens_seen": 926351360
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005812637913741224,
+ "loss": 1.293,
+ "theoretical_loss": 3.70448508435953,
+ "tokens_seen": 926416896
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005812477432296892,
+ "loss": 1.4121,
+ "theoretical_loss": 3.7044599868443804,
+ "tokens_seen": 926482432
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005812316950852558,
+ "loss": 1.4473,
+ "theoretical_loss": 3.7044348916015206,
+ "tokens_seen": 926547968
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005812156469408226,
+ "loss": 1.2784,
+ "theoretical_loss": 3.704409798630585,
+ "tokens_seen": 926613504
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005811995987963892,
+ "loss": 1.5596,
+ "theoretical_loss": 3.7043847079312058,
+ "tokens_seen": 926679040
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581183550651956,
+ "loss": 1.51,
+ "theoretical_loss": 3.704359619503018,
+ "tokens_seen": 926744576
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005811675025075226,
+ "loss": 1.4248,
+ "theoretical_loss": 3.704334533345656,
+ "tokens_seen": 926810112
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005811514543630894,
+ "loss": 1.3905,
+ "theoretical_loss": 3.704309449458752,
+ "tokens_seen": 926875648
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000581135406218656,
+ "loss": 1.3438,
+ "theoretical_loss": 3.7042843678419404,
+ "tokens_seen": 926941184
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005811193580742228,
+ "loss": 1.406,
+ "theoretical_loss": 3.7042592884948564,
+ "tokens_seen": 927006720
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005811033099297894,
+ "loss": 1.3171,
+ "theoretical_loss": 3.704234211417133,
+ "tokens_seen": 927072256
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810872617853561,
+ "loss": 1.3694,
+ "theoretical_loss": 3.704209136608405,
+ "tokens_seen": 927137792
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810712136409228,
+ "loss": 1.317,
+ "theoretical_loss": 3.704184064068307,
+ "tokens_seen": 927203328
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810551654964895,
+ "loss": 1.3885,
+ "theoretical_loss": 3.704158993796473,
+ "tokens_seen": 927268864
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 313936,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.348827600479126,
+ "objective/train/theoretical_loss": 3.7041339257925374,
+ "objective/train/tokens_used": 947794400,
+ "theoretical_loss": 3.7041339257925374,
+ "tokens_seen": 927334400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810391173520562,
+ "loss": 1.4109,
+ "theoretical_loss": 3.7041339257925374,
+ "tokens_seen": 927334400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810230692076229,
+ "loss": 1.3908,
+ "theoretical_loss": 3.7041088600561345,
+ "tokens_seen": 927399936
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005810070210631896,
+ "loss": 1.3938,
+ "theoretical_loss": 3.7040837965869002,
+ "tokens_seen": 927465472
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005809909729187563,
+ "loss": 1.3591,
+ "theoretical_loss": 3.704058735384468,
+ "tokens_seen": 927531008
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000580974924774323,
+ "loss": 1.3256,
+ "theoretical_loss": 3.704033676448473,
+ "tokens_seen": 927596544
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005809588766298897,
+ "loss": 1.4747,
+ "theoretical_loss": 3.7040086197785502,
+ "tokens_seen": 927662080
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005809428284854563,
+ "loss": 1.3971,
+ "theoretical_loss": 3.7039835653743354,
+ "tokens_seen": 927727616
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005809267803410231,
+ "loss": 1.3978,
+ "theoretical_loss": 3.703958513235463,
+ "tokens_seen": 927793152
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005809107321965897,
+ "loss": 1.405,
+ "theoretical_loss": 3.703933463361568,
+ "tokens_seen": 927858688
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808946840521565,
+ "loss": 1.5449,
+ "theoretical_loss": 3.7039084157522857,
+ "tokens_seen": 927924224
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808786359077231,
+ "loss": 1.4407,
+ "theoretical_loss": 3.703883370407252,
+ "tokens_seen": 927989760
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808625877632899,
+ "loss": 1.2564,
+ "theoretical_loss": 3.7038583273261017,
+ "tokens_seen": 928055296
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808465396188565,
+ "loss": 1.3291,
+ "theoretical_loss": 3.7038332865084707,
+ "tokens_seen": 928120832
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808304914744233,
+ "loss": 1.3966,
+ "theoretical_loss": 3.703808247953994,
+ "tokens_seen": 928186368
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005808144433299899,
+ "loss": 1.452,
+ "theoretical_loss": 3.7037832116623086,
+ "tokens_seen": 928251904
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807983951855567,
+ "loss": 1.4109,
+ "theoretical_loss": 3.7037581776330493,
+ "tokens_seen": 928317440
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807823470411233,
+ "loss": 1.2942,
+ "theoretical_loss": 3.7037331458658516,
+ "tokens_seen": 928382976
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807662988966901,
+ "loss": 1.3621,
+ "theoretical_loss": 3.7037081163603522,
+ "tokens_seen": 928448512
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807502507522567,
+ "loss": 1.4988,
+ "theoretical_loss": 3.703683089116187,
+ "tokens_seen": 928514048
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807342026078235,
+ "loss": 1.5123,
+ "theoretical_loss": 3.7036580641329917,
+ "tokens_seen": 928579584
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807181544633901,
+ "loss": 1.3268,
+ "theoretical_loss": 3.703633041410403,
+ "tokens_seen": 928645120
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005807021063189569,
+ "loss": 1.3235,
+ "theoretical_loss": 3.7036080209480566,
+ "tokens_seen": 928710656
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806860581745235,
+ "loss": 1.361,
+ "theoretical_loss": 3.7035830027455896,
+ "tokens_seen": 928776192
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806700100300903,
+ "loss": 1.3305,
+ "theoretical_loss": 3.7035579868026383,
+ "tokens_seen": 928841728
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806539618856569,
+ "loss": 1.3372,
+ "theoretical_loss": 3.703532973118839,
+ "tokens_seen": 928907264
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806379137412237,
+ "loss": 1.3979,
+ "theoretical_loss": 3.703507961693828,
+ "tokens_seen": 928972800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806218655967903,
+ "loss": 1.3843,
+ "theoretical_loss": 3.703482952527242,
+ "tokens_seen": 929038336
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005806058174523571,
+ "loss": 1.3754,
+ "theoretical_loss": 3.7034579456187187,
+ "tokens_seen": 929103872
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805897693079237,
+ "loss": 1.3553,
+ "theoretical_loss": 3.7034329409678937,
+ "tokens_seen": 929169408
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805737211634905,
+ "loss": 1.4483,
+ "theoretical_loss": 3.703407938574405,
+ "tokens_seen": 929234944
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805576730190571,
+ "loss": 1.3585,
+ "theoretical_loss": 3.703382938437889,
+ "tokens_seen": 929300480
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805416248746239,
+ "loss": 1.4125,
+ "theoretical_loss": 3.703357940557983,
+ "tokens_seen": 929366016
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805255767301905,
+ "loss": 1.3327,
+ "theoretical_loss": 3.7033329449343246,
+ "tokens_seen": 929431552
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005805095285857573,
+ "loss": 1.3395,
+ "theoretical_loss": 3.7033079515665506,
+ "tokens_seen": 929497088
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804934804413239,
+ "loss": 1.5735,
+ "theoretical_loss": 3.703282960454299,
+ "tokens_seen": 929562624
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804774322968907,
+ "loss": 1.3129,
+ "theoretical_loss": 3.703257971597206,
+ "tokens_seen": 929628160
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804613841524573,
+ "loss": 1.3756,
+ "theoretical_loss": 3.70323298499491,
+ "tokens_seen": 929693696
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804453360080241,
+ "loss": 1.3092,
+ "theoretical_loss": 3.7032080006470487,
+ "tokens_seen": 929759232
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804292878635907,
+ "loss": 1.3523,
+ "theoretical_loss": 3.7031830185532595,
+ "tokens_seen": 929824768
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005804132397191575,
+ "loss": 1.3675,
+ "theoretical_loss": 3.7031580387131804,
+ "tokens_seen": 929890304
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803971915747241,
+ "loss": 1.3851,
+ "theoretical_loss": 3.7031330611264495,
+ "tokens_seen": 929955840
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803811434302909,
+ "loss": 1.2445,
+ "theoretical_loss": 3.703108085792704,
+ "tokens_seen": 930021376
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803650952858576,
+ "loss": 1.3681,
+ "theoretical_loss": 3.7030831127115826,
+ "tokens_seen": 930086912
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803490471414243,
+ "loss": 1.3383,
+ "theoretical_loss": 3.7030581418827233,
+ "tokens_seen": 930152448
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000580332998996991,
+ "loss": 1.3631,
+ "theoretical_loss": 3.703033173305764,
+ "tokens_seen": 930217984
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803169508525577,
+ "loss": 1.3712,
+ "theoretical_loss": 3.703008206980343,
+ "tokens_seen": 930283520
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005803009027081244,
+ "loss": 1.328,
+ "theoretical_loss": 3.7029832429060994,
+ "tokens_seen": 930349056
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802848545636911,
+ "loss": 1.3491,
+ "theoretical_loss": 3.7029582810826707,
+ "tokens_seen": 930414592
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802688064192578,
+ "loss": 1.4885,
+ "theoretical_loss": 3.702933321509696,
+ "tokens_seen": 930480128
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802527582748245,
+ "loss": 1.3232,
+ "theoretical_loss": 3.702908364186814,
+ "tokens_seen": 930545664
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 314827,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3260948657989502,
+ "objective/train/theoretical_loss": 3.702883409113663,
+ "objective/train/tokens_used": 951071200,
+ "theoretical_loss": 3.702883409113663,
+ "tokens_seen": 930611200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802367101303912,
+ "loss": 1.3089,
+ "theoretical_loss": 3.702883409113663,
+ "tokens_seen": 930611200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802206619859579,
+ "loss": 1.3488,
+ "theoretical_loss": 3.7028584562898823,
+ "tokens_seen": 930676736
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005802046138415246,
+ "loss": 1.4493,
+ "theoretical_loss": 3.7028335057151103,
+ "tokens_seen": 930742272
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801885656970913,
+ "loss": 1.4082,
+ "theoretical_loss": 3.7028085573889857,
+ "tokens_seen": 930807808
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801725175526581,
+ "loss": 1.4522,
+ "theoretical_loss": 3.7027836113111485,
+ "tokens_seen": 930873344
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801564694082247,
+ "loss": 1.4337,
+ "theoretical_loss": 3.7027586674812376,
+ "tokens_seen": 930938880
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801404212637915,
+ "loss": 1.3837,
+ "theoretical_loss": 3.7027337258988915,
+ "tokens_seen": 931004416
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801243731193581,
+ "loss": 1.3573,
+ "theoretical_loss": 3.7027087865637496,
+ "tokens_seen": 931069952
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005801083249749249,
+ "loss": 1.3928,
+ "theoretical_loss": 3.702683849475452,
+ "tokens_seen": 931135488
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800922768304915,
+ "loss": 1.49,
+ "theoretical_loss": 3.7026589146336377,
+ "tokens_seen": 931201024
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800762286860583,
+ "loss": 1.4633,
+ "theoretical_loss": 3.702633982037946,
+ "tokens_seen": 931266560
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800601805416249,
+ "loss": 1.417,
+ "theoretical_loss": 3.7026090516880172,
+ "tokens_seen": 931332096
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800441323971917,
+ "loss": 1.3736,
+ "theoretical_loss": 3.7025841235834904,
+ "tokens_seen": 931397632
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800280842527583,
+ "loss": 1.4305,
+ "theoretical_loss": 3.702559197724006,
+ "tokens_seen": 931463168
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005800120361083251,
+ "loss": 1.3007,
+ "theoretical_loss": 3.702534274109203,
+ "tokens_seen": 931528704
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799959879638917,
+ "loss": 1.2282,
+ "theoretical_loss": 3.702509352738722,
+ "tokens_seen": 931594240
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799799398194585,
+ "loss": 1.3787,
+ "theoretical_loss": 3.7024844336122027,
+ "tokens_seen": 931659776
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799638916750251,
+ "loss": 1.4832,
+ "theoretical_loss": 3.7024595167292853,
+ "tokens_seen": 931725312
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799478435305919,
+ "loss": 1.3689,
+ "theoretical_loss": 3.7024346020896104,
+ "tokens_seen": 931790848
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799317953861585,
+ "loss": 1.3569,
+ "theoretical_loss": 3.7024096896928183,
+ "tokens_seen": 931856384
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005799157472417253,
+ "loss": 1.4301,
+ "theoretical_loss": 3.7023847795385487,
+ "tokens_seen": 931921920
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798996990972919,
+ "loss": 1.3889,
+ "theoretical_loss": 3.7023598716264425,
+ "tokens_seen": 931987456
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798836509528587,
+ "loss": 1.3287,
+ "theoretical_loss": 3.7023349659561395,
+ "tokens_seen": 932052992
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798676028084253,
+ "loss": 1.409,
+ "theoretical_loss": 3.702310062527282,
+ "tokens_seen": 932118528
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798515546639921,
+ "loss": 1.4233,
+ "theoretical_loss": 3.7022851613395096,
+ "tokens_seen": 932184064
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798355065195587,
+ "loss": 1.3944,
+ "theoretical_loss": 3.7022602623924623,
+ "tokens_seen": 932249600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798194583751255,
+ "loss": 1.3632,
+ "theoretical_loss": 3.702235365685783,
+ "tokens_seen": 932315136
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005798034102306921,
+ "loss": 1.3414,
+ "theoretical_loss": 3.702210471219111,
+ "tokens_seen": 932380672
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797873620862589,
+ "loss": 1.4716,
+ "theoretical_loss": 3.7021855789920872,
+ "tokens_seen": 932446208
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797713139418255,
+ "loss": 1.4241,
+ "theoretical_loss": 3.702160689004354,
+ "tokens_seen": 932511744
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797552657973923,
+ "loss": 1.4795,
+ "theoretical_loss": 3.702135801255552,
+ "tokens_seen": 932577280
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797392176529589,
+ "loss": 1.3867,
+ "theoretical_loss": 3.7021109157453225,
+ "tokens_seen": 932642816
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797231695085257,
+ "loss": 1.2557,
+ "theoretical_loss": 3.7020860324733063,
+ "tokens_seen": 932708352
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005797071213640923,
+ "loss": 1.2278,
+ "theoretical_loss": 3.7020611514391457,
+ "tokens_seen": 932773888
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579691073219659,
+ "loss": 1.4045,
+ "theoretical_loss": 3.7020362726424816,
+ "tokens_seen": 932839424
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005796750250752257,
+ "loss": 1.3367,
+ "theoretical_loss": 3.7020113960829564,
+ "tokens_seen": 932904960
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005796589769307924,
+ "loss": 1.4045,
+ "theoretical_loss": 3.701986521760211,
+ "tokens_seen": 932970496
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005796429287863591,
+ "loss": 1.2608,
+ "theoretical_loss": 3.701961649673888,
+ "tokens_seen": 933036032
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005796268806419258,
+ "loss": 1.3224,
+ "theoretical_loss": 3.701936779823628,
+ "tokens_seen": 933101568
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005796108324974925,
+ "loss": 1.4404,
+ "theoretical_loss": 3.7019119122090736,
+ "tokens_seen": 933167104
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005795947843530592,
+ "loss": 1.4872,
+ "theoretical_loss": 3.7018870468298672,
+ "tokens_seen": 933232640
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005795787362086259,
+ "loss": 1.3217,
+ "theoretical_loss": 3.701862183685651,
+ "tokens_seen": 933298176
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005795626880641926,
+ "loss": 1.2918,
+ "theoretical_loss": 3.701837322776066,
+ "tokens_seen": 933363712
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005795466399197592,
+ "loss": 1.3156,
+ "theoretical_loss": 3.7018124641007564,
+ "tokens_seen": 933429248
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579530591775326,
+ "loss": 1.3608,
+ "theoretical_loss": 3.7017876076593623,
+ "tokens_seen": 933494784
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005795145436308926,
+ "loss": 1.3975,
+ "theoretical_loss": 3.701762753451528,
+ "tokens_seen": 933560320
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794984954864594,
+ "loss": 1.3239,
+ "theoretical_loss": 3.701737901476895,
+ "tokens_seen": 933625856
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579482447342026,
+ "loss": 1.3837,
+ "theoretical_loss": 3.7017130517351067,
+ "tokens_seen": 933691392
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794663991975928,
+ "loss": 1.2296,
+ "theoretical_loss": 3.701688204225804,
+ "tokens_seen": 933756928
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794503510531594,
+ "loss": 1.4696,
+ "theoretical_loss": 3.7016633589486325,
+ "tokens_seen": 933822464
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 316104,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.325061559677124,
+ "objective/train/theoretical_loss": 3.7016385159032326,
+ "objective/train/tokens_used": 954348000,
+ "theoretical_loss": 3.7016385159032326,
+ "tokens_seen": 933888000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794343029087262,
+ "loss": 1.3622,
+ "theoretical_loss": 3.7016385159032326,
+ "tokens_seen": 933888000
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794182547642928,
+ "loss": 1.3294,
+ "theoretical_loss": 3.7016136750892485,
+ "tokens_seen": 933953536
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005794022066198596,
+ "loss": 1.3868,
+ "theoretical_loss": 3.701588836506323,
+ "tokens_seen": 934019072
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005793861584754262,
+ "loss": 1.3594,
+ "theoretical_loss": 3.7015640001540984,
+ "tokens_seen": 934084608
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579370110330993,
+ "loss": 1.3129,
+ "theoretical_loss": 3.701539166032219,
+ "tokens_seen": 934150144
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005793540621865596,
+ "loss": 1.4787,
+ "theoretical_loss": 3.7015143341403274,
+ "tokens_seen": 934215680
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005793380140421264,
+ "loss": 1.3054,
+ "theoretical_loss": 3.7014895044780673,
+ "tokens_seen": 934281216
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579321965897693,
+ "loss": 1.5103,
+ "theoretical_loss": 3.701464677045082,
+ "tokens_seen": 934346752
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005793059177532598,
+ "loss": 1.2755,
+ "theoretical_loss": 3.7014398518410148,
+ "tokens_seen": 934412288
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005792898696088264,
+ "loss": 1.3949,
+ "theoretical_loss": 3.7014150288655094,
+ "tokens_seen": 934477824
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005792738214643932,
+ "loss": 1.3603,
+ "theoretical_loss": 3.7013902081182097,
+ "tokens_seen": 934543360
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005792577733199598,
+ "loss": 1.2299,
+ "theoretical_loss": 3.7013653895987595,
+ "tokens_seen": 934608896
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005792417251755266,
+ "loss": 1.3886,
+ "theoretical_loss": 3.701340573306802,
+ "tokens_seen": 934674432
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005792256770310932,
+ "loss": 1.3569,
+ "theoretical_loss": 3.7013157592419814,
+ "tokens_seen": 934739968
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.00057920962888666,
+ "loss": 1.2818,
+ "theoretical_loss": 3.701290947403942,
+ "tokens_seen": 934805504
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005791935807422266,
+ "loss": 1.2883,
+ "theoretical_loss": 3.701266137792328,
+ "tokens_seen": 934871040
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005791775325977934,
+ "loss": 1.3721,
+ "theoretical_loss": 3.7012413304067833,
+ "tokens_seen": 934936576
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.00057916148445336,
+ "loss": 1.4051,
+ "theoretical_loss": 3.701216525246952,
+ "tokens_seen": 935002112
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005791454363089268,
+ "loss": 1.4096,
+ "theoretical_loss": 3.701191722312478,
+ "tokens_seen": 935067648
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005791293881644934,
+ "loss": 1.4638,
+ "theoretical_loss": 3.701166921603007,
+ "tokens_seen": 935133184
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005791133400200602,
+ "loss": 1.3949,
+ "theoretical_loss": 3.7011421231181822,
+ "tokens_seen": 935198720
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005790972918756268,
+ "loss": 1.3449,
+ "theoretical_loss": 3.7011173268576494,
+ "tokens_seen": 935264256
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005790812437311936,
+ "loss": 1.3907,
+ "theoretical_loss": 3.701092532821052,
+ "tokens_seen": 935329792
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005790651955867602,
+ "loss": 1.409,
+ "theoretical_loss": 3.7010677410080355,
+ "tokens_seen": 935395328
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579049147442327,
+ "loss": 1.4154,
+ "theoretical_loss": 3.7010429514182444,
+ "tokens_seen": 935460864
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005790330992978936,
+ "loss": 1.2992,
+ "theoretical_loss": 3.701018164051324,
+ "tokens_seen": 935526400
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005790170511534604,
+ "loss": 1.4031,
+ "theoretical_loss": 3.7009933789069187,
+ "tokens_seen": 935591936
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000579001003009027,
+ "loss": 1.4881,
+ "theoretical_loss": 3.7009685959846736,
+ "tokens_seen": 935657472
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005789849548645938,
+ "loss": 1.3425,
+ "theoretical_loss": 3.7009438152842344,
+ "tokens_seen": 935723008
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005789689067201606,
+ "loss": 1.3699,
+ "theoretical_loss": 3.7009190368052463,
+ "tokens_seen": 935788544
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005789528585757272,
+ "loss": 1.3374,
+ "theoretical_loss": 3.7008942605473543,
+ "tokens_seen": 935854080
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578936810431294,
+ "loss": 1.4309,
+ "theoretical_loss": 3.7008694865102036,
+ "tokens_seen": 935919616
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005789207622868606,
+ "loss": 1.4752,
+ "theoretical_loss": 3.7008447146934396,
+ "tokens_seen": 935985152
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005789047141424274,
+ "loss": 1.3088,
+ "theoretical_loss": 3.7008199450967085,
+ "tokens_seen": 936050688
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578888665997994,
+ "loss": 1.3702,
+ "theoretical_loss": 3.7007951777196553,
+ "tokens_seen": 936116224
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005788726178535608,
+ "loss": 1.3588,
+ "theoretical_loss": 3.7007704125619263,
+ "tokens_seen": 936181760
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005788565697091274,
+ "loss": 1.354,
+ "theoretical_loss": 3.7007456496231668,
+ "tokens_seen": 936247296
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005788405215646942,
+ "loss": 1.3702,
+ "theoretical_loss": 3.7007208889030228,
+ "tokens_seen": 936312832
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005788244734202608,
+ "loss": 1.4917,
+ "theoretical_loss": 3.7006961304011403,
+ "tokens_seen": 936378368
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005788084252758276,
+ "loss": 1.4233,
+ "theoretical_loss": 3.7006713741171655,
+ "tokens_seen": 936443904
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005787923771313942,
+ "loss": 1.5235,
+ "theoretical_loss": 3.700646620050744,
+ "tokens_seen": 936509440
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578776328986961,
+ "loss": 1.3346,
+ "theoretical_loss": 3.700621868201523,
+ "tokens_seen": 936574976
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005787602808425276,
+ "loss": 1.3589,
+ "theoretical_loss": 3.7005971185691475,
+ "tokens_seen": 936640512
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005787442326980944,
+ "loss": 1.3806,
+ "theoretical_loss": 3.700572371153265,
+ "tokens_seen": 936706048
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578728184553661,
+ "loss": 1.3391,
+ "theoretical_loss": 3.7005476259535213,
+ "tokens_seen": 936771584
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005787121364092278,
+ "loss": 1.3121,
+ "theoretical_loss": 3.700522882969563,
+ "tokens_seen": 936837120
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005786960882647944,
+ "loss": 1.3904,
+ "theoretical_loss": 3.7004981422010372,
+ "tokens_seen": 936902656
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005786800401203612,
+ "loss": 1.3613,
+ "theoretical_loss": 3.7004734036475897,
+ "tokens_seen": 936968192
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005786639919759278,
+ "loss": 1.3834,
+ "theoretical_loss": 3.7004486673088675,
+ "tokens_seen": 937033728
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005786479438314946,
+ "loss": 1.381,
+ "theoretical_loss": 3.7004239331845183,
+ "tokens_seen": 937099264
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 317140,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3840632438659668,
+ "objective/train/theoretical_loss": 3.700399201274188,
+ "objective/train/tokens_used": 957624800,
+ "theoretical_loss": 3.700399201274188,
+ "tokens_seen": 937164800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005786318956870612,
+ "loss": 1.3405,
+ "theoretical_loss": 3.700399201274188,
+ "tokens_seen": 937164800
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578615847542628,
+ "loss": 1.3924,
+ "theoretical_loss": 3.700374471577524,
+ "tokens_seen": 937230336
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785997993981946,
+ "loss": 1.431,
+ "theoretical_loss": 3.700349744094174,
+ "tokens_seen": 937295872
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785837512537614,
+ "loss": 1.3422,
+ "theoretical_loss": 3.7003250188237837,
+ "tokens_seen": 937361408
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578567703109328,
+ "loss": 1.3686,
+ "theoretical_loss": 3.700300295766002,
+ "tokens_seen": 937426944
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785516549648948,
+ "loss": 1.3968,
+ "theoretical_loss": 3.700275574920475,
+ "tokens_seen": 937492480
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785356068204614,
+ "loss": 1.3153,
+ "theoretical_loss": 3.700250856286851,
+ "tokens_seen": 937558016
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785195586760282,
+ "loss": 1.4538,
+ "theoretical_loss": 3.7002261398647764,
+ "tokens_seen": 937623552
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005785035105315948,
+ "loss": 1.4646,
+ "theoretical_loss": 3.7002014256539004,
+ "tokens_seen": 937689088
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005784874623871616,
+ "loss": 1.4331,
+ "theoretical_loss": 3.7001767136538692,
+ "tokens_seen": 937754624
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005784714142427282,
+ "loss": 1.3257,
+ "theoretical_loss": 3.700152003864331,
+ "tokens_seen": 937820160
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578455366098295,
+ "loss": 1.3505,
+ "theoretical_loss": 3.700127296284934,
+ "tokens_seen": 937885696
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005784393179538616,
+ "loss": 1.3649,
+ "theoretical_loss": 3.700102590915326,
+ "tokens_seen": 937951232
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005784232698094284,
+ "loss": 1.4132,
+ "theoretical_loss": 3.700077887755154,
+ "tokens_seen": 938016768
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578407221664995,
+ "loss": 1.3479,
+ "theoretical_loss": 3.7000531868040674,
+ "tokens_seen": 938082304
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783911735205618,
+ "loss": 1.4686,
+ "theoretical_loss": 3.7000284880617134,
+ "tokens_seen": 938147840
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783751253761284,
+ "loss": 1.4947,
+ "theoretical_loss": 3.700003791527741,
+ "tokens_seen": 938213376
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783590772316952,
+ "loss": 1.354,
+ "theoretical_loss": 3.699979097201798,
+ "tokens_seen": 938278912
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783430290872618,
+ "loss": 1.329,
+ "theoretical_loss": 3.699954405083533,
+ "tokens_seen": 938344448
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783269809428286,
+ "loss": 1.3881,
+ "theoretical_loss": 3.699929715172594,
+ "tokens_seen": 938409984
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005783109327983952,
+ "loss": 1.4842,
+ "theoretical_loss": 3.69990502746863,
+ "tokens_seen": 938475520
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578294884653962,
+ "loss": 1.2677,
+ "theoretical_loss": 3.69988034197129,
+ "tokens_seen": 938541056
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005782788365095286,
+ "loss": 1.3845,
+ "theoretical_loss": 3.6998556586802214,
+ "tokens_seen": 938606592
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005782627883650953,
+ "loss": 1.2799,
+ "theoretical_loss": 3.6998309775950746,
+ "tokens_seen": 938672128
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.000578246740220662,
+ "loss": 1.4618,
+ "theoretical_loss": 3.699806298715497,
+ "tokens_seen": 938737664
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005782306920762287,
+ "loss": 1.4908,
+ "theoretical_loss": 3.6997816220411384,
+ "tokens_seen": 938803200
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005782146439317954,
+ "loss": 1.2352,
+ "theoretical_loss": 3.6997569475716476,
+ "tokens_seen": 938868736
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781985957873621,
+ "loss": 1.3963,
+ "theoretical_loss": 3.6997322753066744,
+ "tokens_seen": 938934272
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781825476429288,
+ "loss": 1.3798,
+ "theoretical_loss": 3.699707605245867,
+ "tokens_seen": 938999808
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781664994984955,
+ "loss": 1.3078,
+ "theoretical_loss": 3.699682937388875,
+ "tokens_seen": 939065344
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781504513540621,
+ "loss": 1.4004,
+ "theoretical_loss": 3.6996582717353474,
+ "tokens_seen": 939130880
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781344032096289,
+ "loss": 1.4052,
+ "theoretical_loss": 3.699633608284935,
+ "tokens_seen": 939196416
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781183550651955,
+ "loss": 1.4474,
+ "theoretical_loss": 3.699608947037285,
+ "tokens_seen": 939261952
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005781023069207623,
+ "loss": 1.4475,
+ "theoretical_loss": 3.699584287992049,
+ "tokens_seen": 939327488
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780862587763289,
+ "loss": 1.3526,
+ "theoretical_loss": 3.699559631148876,
+ "tokens_seen": 939393024
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780702106318957,
+ "loss": 1.3568,
+ "theoretical_loss": 3.6995349765074157,
+ "tokens_seen": 939458560
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780541624874623,
+ "loss": 1.3594,
+ "theoretical_loss": 3.699510324067318,
+ "tokens_seen": 939524096
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780381143430291,
+ "loss": 1.2978,
+ "theoretical_loss": 3.6994856738282325,
+ "tokens_seen": 939589632
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780220661985957,
+ "loss": 1.4097,
+ "theoretical_loss": 3.6994610257898097,
+ "tokens_seen": 939655168
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005780060180541625,
+ "loss": 1.3657,
+ "theoretical_loss": 3.6994363799516994,
+ "tokens_seen": 939720704
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779899699097291,
+ "loss": 1.5101,
+ "theoretical_loss": 3.699411736313552,
+ "tokens_seen": 939786240
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779739217652959,
+ "loss": 1.488,
+ "theoretical_loss": 3.699387094875017,
+ "tokens_seen": 939851776
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779578736208625,
+ "loss": 1.266,
+ "theoretical_loss": 3.6993624556357454,
+ "tokens_seen": 939917312
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779418254764293,
+ "loss": 1.2609,
+ "theoretical_loss": 3.6993378185953874,
+ "tokens_seen": 939982848
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779257773319959,
+ "loss": 1.3096,
+ "theoretical_loss": 3.6993131837535937,
+ "tokens_seen": 940048384
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005779097291875627,
+ "loss": 1.4811,
+ "theoretical_loss": 3.6992885511100146,
+ "tokens_seen": 940113920
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005778936810431293,
+ "loss": 1.3487,
+ "theoretical_loss": 3.6992639206643005,
+ "tokens_seen": 940179456
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005778776328986961,
+ "loss": 1.4474,
+ "theoretical_loss": 3.6992392924161024,
+ "tokens_seen": 940244992
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005778615847542627,
+ "loss": 1.4966,
+ "theoretical_loss": 3.699214666365071,
+ "tokens_seen": 940310528
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005778455366098295,
+ "loss": 1.381,
+ "theoretical_loss": 3.6991900425108577,
+ "tokens_seen": 940376064
+ },
+ {
+ "epoch": 0.28,
+ "objective/train/docs_used": 318390,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4625651836395264,
+ "objective/train/theoretical_loss": 3.6991654208531126,
+ "objective/train/tokens_used": 960901600,
+ "theoretical_loss": 3.6991654208531126,
+ "tokens_seen": 940441600
+ },
+ {
+ "epoch": 0.28,
+ "learning_rate": 0.0005778294884653961,
+ "loss": 1.3777,
+ "theoretical_loss": 3.6991654208531126,
+ "tokens_seen": 940441600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005778134403209629,
+ "loss": 1.3918,
+ "theoretical_loss": 3.6991408013914873,
+ "tokens_seen": 940507136
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777973921765295,
+ "loss": 1.4778,
+ "theoretical_loss": 3.6991161841256326,
+ "tokens_seen": 940572672
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777813440320963,
+ "loss": 1.3639,
+ "theoretical_loss": 3.6990915690551995,
+ "tokens_seen": 940638208
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000577765295887663,
+ "loss": 1.3475,
+ "theoretical_loss": 3.6990669561798404,
+ "tokens_seen": 940703744
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777492477432297,
+ "loss": 1.3163,
+ "theoretical_loss": 3.699042345499205,
+ "tokens_seen": 940769280
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777331995987965,
+ "loss": 1.3902,
+ "theoretical_loss": 3.6990177370129462,
+ "tokens_seen": 940834816
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777171514543631,
+ "loss": 1.4047,
+ "theoretical_loss": 3.698993130720714,
+ "tokens_seen": 940900352
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005777011033099299,
+ "loss": 1.2753,
+ "theoretical_loss": 3.6989685266221617,
+ "tokens_seen": 940965888
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776850551654965,
+ "loss": 1.4524,
+ "theoretical_loss": 3.69894392471694,
+ "tokens_seen": 941031424
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776690070210633,
+ "loss": 1.3627,
+ "theoretical_loss": 3.6989193250047006,
+ "tokens_seen": 941096960
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776529588766299,
+ "loss": 1.3037,
+ "theoretical_loss": 3.6988947274850954,
+ "tokens_seen": 941162496
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776369107321967,
+ "loss": 1.4261,
+ "theoretical_loss": 3.6988701321577766,
+ "tokens_seen": 941228032
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776208625877633,
+ "loss": 1.298,
+ "theoretical_loss": 3.698845539022396,
+ "tokens_seen": 941293568
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005776048144433301,
+ "loss": 1.5898,
+ "theoretical_loss": 3.6988209480786054,
+ "tokens_seen": 941359104
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775887662988967,
+ "loss": 1.4488,
+ "theoretical_loss": 3.6987963593260567,
+ "tokens_seen": 941424640
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775727181544635,
+ "loss": 1.4654,
+ "theoretical_loss": 3.6987717727644034,
+ "tokens_seen": 941490176
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775566700100301,
+ "loss": 1.4613,
+ "theoretical_loss": 3.698747188393297,
+ "tokens_seen": 941555712
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775406218655969,
+ "loss": 1.284,
+ "theoretical_loss": 3.6987226062123897,
+ "tokens_seen": 941621248
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775245737211635,
+ "loss": 1.4079,
+ "theoretical_loss": 3.698698026221334,
+ "tokens_seen": 941686784
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005775085255767303,
+ "loss": 1.4136,
+ "theoretical_loss": 3.698673448419783,
+ "tokens_seen": 941752320
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774924774322969,
+ "loss": 1.504,
+ "theoretical_loss": 3.6986488728073885,
+ "tokens_seen": 941817856
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774764292878637,
+ "loss": 1.3827,
+ "theoretical_loss": 3.6986242993838037,
+ "tokens_seen": 941883392
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774603811434303,
+ "loss": 1.4615,
+ "theoretical_loss": 3.6985997281486815,
+ "tokens_seen": 941948928
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774443329989971,
+ "loss": 1.3283,
+ "theoretical_loss": 3.6985751591016744,
+ "tokens_seen": 942014464
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774282848545637,
+ "loss": 1.2975,
+ "theoretical_loss": 3.6985505922424355,
+ "tokens_seen": 942080000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005774122367101305,
+ "loss": 1.4891,
+ "theoretical_loss": 3.698526027570618,
+ "tokens_seen": 942145536
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773961885656971,
+ "loss": 1.311,
+ "theoretical_loss": 3.698501465085875,
+ "tokens_seen": 942211072
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773801404212639,
+ "loss": 1.4593,
+ "theoretical_loss": 3.6984769047878587,
+ "tokens_seen": 942276608
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773640922768305,
+ "loss": 1.4576,
+ "theoretical_loss": 3.6984523466762234,
+ "tokens_seen": 942342144
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773480441323973,
+ "loss": 1.3528,
+ "theoretical_loss": 3.698427790750623,
+ "tokens_seen": 942407680
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773319959879639,
+ "loss": 1.3787,
+ "theoretical_loss": 3.698403237010709,
+ "tokens_seen": 942473216
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005773159478435307,
+ "loss": 1.406,
+ "theoretical_loss": 3.6983786854561362,
+ "tokens_seen": 942538752
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772998996990973,
+ "loss": 1.3402,
+ "theoretical_loss": 3.6983541360865586,
+ "tokens_seen": 942604288
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772838515546641,
+ "loss": 1.265,
+ "theoretical_loss": 3.6983295889016286,
+ "tokens_seen": 942669824
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772678034102307,
+ "loss": 1.3356,
+ "theoretical_loss": 3.6983050439010006,
+ "tokens_seen": 942735360
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772517552657975,
+ "loss": 1.2517,
+ "theoretical_loss": 3.6982805010843283,
+ "tokens_seen": 942800896
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772357071213641,
+ "loss": 1.3905,
+ "theoretical_loss": 3.6982559604512657,
+ "tokens_seen": 942866432
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772196589769309,
+ "loss": 1.3475,
+ "theoretical_loss": 3.6982314220014665,
+ "tokens_seen": 942931968
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005772036108324975,
+ "loss": 1.3674,
+ "theoretical_loss": 3.698206885734585,
+ "tokens_seen": 942997504
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771875626880643,
+ "loss": 1.3298,
+ "theoretical_loss": 3.6981823516502756,
+ "tokens_seen": 943063040
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771715145436309,
+ "loss": 1.3344,
+ "theoretical_loss": 3.698157819748192,
+ "tokens_seen": 943128576
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771554663991977,
+ "loss": 1.3038,
+ "theoretical_loss": 3.6981332900279886,
+ "tokens_seen": 943194112
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771394182547643,
+ "loss": 1.4594,
+ "theoretical_loss": 3.698108762489319,
+ "tokens_seen": 943259648
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771233701103311,
+ "loss": 1.537,
+ "theoretical_loss": 3.6980842371318396,
+ "tokens_seen": 943325184
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005771073219658977,
+ "loss": 1.3607,
+ "theoretical_loss": 3.698059713955203,
+ "tokens_seen": 943390720
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770912738214645,
+ "loss": 1.3189,
+ "theoretical_loss": 3.6980351929590647,
+ "tokens_seen": 943456256
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770752256770311,
+ "loss": 1.3077,
+ "theoretical_loss": 3.6980106741430796,
+ "tokens_seen": 943521792
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770591775325979,
+ "loss": 1.3808,
+ "theoretical_loss": 3.697986157506902,
+ "tokens_seen": 943587328
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770431293881645,
+ "loss": 1.3744,
+ "theoretical_loss": 3.697961643050186,
+ "tokens_seen": 943652864
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 319507,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.49009370803833,
+ "objective/train/theoretical_loss": 3.697937130772588,
+ "objective/train/tokens_used": 964178400,
+ "theoretical_loss": 3.697937130772588,
+ "tokens_seen": 943718400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770270812437313,
+ "loss": 1.4312,
+ "theoretical_loss": 3.697937130772588,
+ "tokens_seen": 943718400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005770110330992979,
+ "loss": 1.2883,
+ "theoretical_loss": 3.697912620673762,
+ "tokens_seen": 943783936
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769949849548647,
+ "loss": 1.4021,
+ "theoretical_loss": 3.6978881127533634,
+ "tokens_seen": 943849472
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769789368104313,
+ "loss": 1.428,
+ "theoretical_loss": 3.6978636070110475,
+ "tokens_seen": 943915008
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769628886659981,
+ "loss": 1.2986,
+ "theoretical_loss": 3.6978391034464693,
+ "tokens_seen": 943980544
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769468405215647,
+ "loss": 1.3775,
+ "theoretical_loss": 3.6978146020592835,
+ "tokens_seen": 944046080
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769307923771315,
+ "loss": 1.3471,
+ "theoretical_loss": 3.697790102849147,
+ "tokens_seen": 944111616
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005769147442326981,
+ "loss": 1.3715,
+ "theoretical_loss": 3.6977656058157136,
+ "tokens_seen": 944177152
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768986960882649,
+ "loss": 1.4267,
+ "theoretical_loss": 3.69774111095864,
+ "tokens_seen": 944242688
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768826479438315,
+ "loss": 1.2568,
+ "theoretical_loss": 3.697716618277582,
+ "tokens_seen": 944308224
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768665997993982,
+ "loss": 1.3657,
+ "theoretical_loss": 3.6976921277721937,
+ "tokens_seen": 944373760
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768505516549649,
+ "loss": 1.3924,
+ "theoretical_loss": 3.697667639442132,
+ "tokens_seen": 944439296
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768345035105316,
+ "loss": 1.4021,
+ "theoretical_loss": 3.6976431532870535,
+ "tokens_seen": 944504832
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005768184553660983,
+ "loss": 1.3073,
+ "theoretical_loss": 3.697618669306612,
+ "tokens_seen": 944570368
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576802407221665,
+ "loss": 1.3843,
+ "theoretical_loss": 3.697594187500466,
+ "tokens_seen": 944635904
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767863590772317,
+ "loss": 1.4449,
+ "theoretical_loss": 3.6975697078682694,
+ "tokens_seen": 944701440
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767703109327984,
+ "loss": 1.3983,
+ "theoretical_loss": 3.69754523040968,
+ "tokens_seen": 944766976
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767542627883651,
+ "loss": 1.3271,
+ "theoretical_loss": 3.697520755124353,
+ "tokens_seen": 944832512
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767382146439318,
+ "loss": 1.3503,
+ "theoretical_loss": 3.6974962820119455,
+ "tokens_seen": 944898048
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767221664994984,
+ "loss": 1.2448,
+ "theoretical_loss": 3.697471811072113,
+ "tokens_seen": 944963584
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005767061183550652,
+ "loss": 1.3467,
+ "theoretical_loss": 3.697447342304513,
+ "tokens_seen": 945029120
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005766900702106318,
+ "loss": 1.2963,
+ "theoretical_loss": 3.697422875708801,
+ "tokens_seen": 945094656
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005766740220661986,
+ "loss": 1.3276,
+ "theoretical_loss": 3.6973984112846345,
+ "tokens_seen": 945160192
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005766579739217652,
+ "loss": 1.3833,
+ "theoretical_loss": 3.69737394903167,
+ "tokens_seen": 945225728
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576641925777332,
+ "loss": 1.365,
+ "theoretical_loss": 3.697349488949564,
+ "tokens_seen": 945291264
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005766258776328986,
+ "loss": 1.3639,
+ "theoretical_loss": 3.6973250310379733,
+ "tokens_seen": 945356800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005766098294884654,
+ "loss": 1.3873,
+ "theoretical_loss": 3.697300575296555,
+ "tokens_seen": 945422336
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576593781344032,
+ "loss": 1.4619,
+ "theoretical_loss": 3.6972761217249666,
+ "tokens_seen": 945487872
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005765777331995988,
+ "loss": 1.4046,
+ "theoretical_loss": 3.6972516703228644,
+ "tokens_seen": 945553408
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005765616850551655,
+ "loss": 1.4257,
+ "theoretical_loss": 3.697227221089906,
+ "tokens_seen": 945618944
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005765456369107322,
+ "loss": 1.4034,
+ "theoretical_loss": 3.697202774025749,
+ "tokens_seen": 945684480
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576529588766299,
+ "loss": 1.3286,
+ "theoretical_loss": 3.6971783291300504,
+ "tokens_seen": 945750016
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005765135406218656,
+ "loss": 1.3103,
+ "theoretical_loss": 3.6971538864024676,
+ "tokens_seen": 945815552
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764974924774323,
+ "loss": 1.3545,
+ "theoretical_loss": 3.697129445842658,
+ "tokens_seen": 945881088
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576481444332999,
+ "loss": 1.3565,
+ "theoretical_loss": 3.697105007450279,
+ "tokens_seen": 945946624
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764653961885658,
+ "loss": 1.3578,
+ "theoretical_loss": 3.697080571224989,
+ "tokens_seen": 946012160
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764493480441324,
+ "loss": 1.4201,
+ "theoretical_loss": 3.6970561371664443,
+ "tokens_seen": 946077696
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764332998996992,
+ "loss": 1.4193,
+ "theoretical_loss": 3.6970317052743042,
+ "tokens_seen": 946143232
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764172517552658,
+ "loss": 1.4432,
+ "theoretical_loss": 3.697007275548226,
+ "tokens_seen": 946208768
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005764012036108326,
+ "loss": 1.4115,
+ "theoretical_loss": 3.6969828479878677,
+ "tokens_seen": 946274304
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005763851554663992,
+ "loss": 1.4443,
+ "theoretical_loss": 3.6969584225928873,
+ "tokens_seen": 946339840
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576369107321966,
+ "loss": 1.3909,
+ "theoretical_loss": 3.696933999362943,
+ "tokens_seen": 946405376
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005763530591775326,
+ "loss": 1.3736,
+ "theoretical_loss": 3.696909578297692,
+ "tokens_seen": 946470912
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005763370110330994,
+ "loss": 1.5252,
+ "theoretical_loss": 3.6968851593967944,
+ "tokens_seen": 946536448
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576320962888666,
+ "loss": 1.4132,
+ "theoretical_loss": 3.696860742659908,
+ "tokens_seen": 946601984
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005763049147442328,
+ "loss": 1.3822,
+ "theoretical_loss": 3.6968363280866896,
+ "tokens_seen": 946667520
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005762888665997994,
+ "loss": 1.4366,
+ "theoretical_loss": 3.6968119156768,
+ "tokens_seen": 946733056
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005762728184553662,
+ "loss": 1.3858,
+ "theoretical_loss": 3.696787505429896,
+ "tokens_seen": 946798592
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005762567703109328,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6967630973456376,
+ "tokens_seen": 946864128
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005762407221664996,
+ "loss": 1.387,
+ "theoretical_loss": 3.6967386914236826,
+ "tokens_seen": 946929664
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 320452,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4507883787155151,
+ "objective/train/theoretical_loss": 3.6967142876636903,
+ "objective/train/tokens_used": 967455200,
+ "theoretical_loss": 3.6967142876636903,
+ "tokens_seen": 946995200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005762246740220662,
+ "loss": 1.4276,
+ "theoretical_loss": 3.6967142876636903,
+ "tokens_seen": 946995200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576208625877633,
+ "loss": 1.3087,
+ "theoretical_loss": 3.696689886065319,
+ "tokens_seen": 947060736
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005761925777331996,
+ "loss": 1.3931,
+ "theoretical_loss": 3.6966654866282282,
+ "tokens_seen": 947126272
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005761765295887664,
+ "loss": 1.4385,
+ "theoretical_loss": 3.6966410893520774,
+ "tokens_seen": 947191808
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576160481444333,
+ "loss": 1.3713,
+ "theoretical_loss": 3.696616694236524,
+ "tokens_seen": 947257344
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005761444332998998,
+ "loss": 1.3719,
+ "theoretical_loss": 3.696592301281229,
+ "tokens_seen": 947322880
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005761283851554664,
+ "loss": 1.2812,
+ "theoretical_loss": 3.6965679104858515,
+ "tokens_seen": 947388416
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005761123370110332,
+ "loss": 1.4943,
+ "theoretical_loss": 3.6965435218500495,
+ "tokens_seen": 947453952
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760962888665998,
+ "loss": 1.4148,
+ "theoretical_loss": 3.6965191353734843,
+ "tokens_seen": 947519488
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760802407221666,
+ "loss": 1.2766,
+ "theoretical_loss": 3.6964947510558135,
+ "tokens_seen": 947585024
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760641925777332,
+ "loss": 1.4846,
+ "theoretical_loss": 3.6964703688966978,
+ "tokens_seen": 947650560
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760481444333,
+ "loss": 1.4069,
+ "theoretical_loss": 3.696445988895797,
+ "tokens_seen": 947716096
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760320962888666,
+ "loss": 1.3377,
+ "theoretical_loss": 3.69642161105277,
+ "tokens_seen": 947781632
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005760160481444334,
+ "loss": 1.289,
+ "theoretical_loss": 3.6963972353672774,
+ "tokens_seen": 947847168
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000576,
+ "loss": 1.3997,
+ "theoretical_loss": 3.696372861838979,
+ "tokens_seen": 947912704
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759839518555668,
+ "loss": 1.448,
+ "theoretical_loss": 3.696348490467534,
+ "tokens_seen": 947978240
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759679037111334,
+ "loss": 1.4665,
+ "theoretical_loss": 3.6963241212526032,
+ "tokens_seen": 948043776
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759518555667002,
+ "loss": 1.2979,
+ "theoretical_loss": 3.6962997541938467,
+ "tokens_seen": 948109312
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759358074222668,
+ "loss": 1.3509,
+ "theoretical_loss": 3.696275389290925,
+ "tokens_seen": 948174848
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759197592778336,
+ "loss": 1.3388,
+ "theoretical_loss": 3.6962510265434974,
+ "tokens_seen": 948240384
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005759037111334002,
+ "loss": 1.3306,
+ "theoretical_loss": 3.6962266659512246,
+ "tokens_seen": 948305920
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575887662988967,
+ "loss": 1.4484,
+ "theoretical_loss": 3.6962023075137673,
+ "tokens_seen": 948371456
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005758716148445336,
+ "loss": 1.4334,
+ "theoretical_loss": 3.696177951230786,
+ "tokens_seen": 948436992
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005758555667001004,
+ "loss": 1.3966,
+ "theoretical_loss": 3.6961535971019415,
+ "tokens_seen": 948502528
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575839518555667,
+ "loss": 1.2923,
+ "theoretical_loss": 3.696129245126894,
+ "tokens_seen": 948568064
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005758234704112338,
+ "loss": 1.4143,
+ "theoretical_loss": 3.6961048953053046,
+ "tokens_seen": 948633600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005758074222668004,
+ "loss": 1.3425,
+ "theoretical_loss": 3.6960805476368335,
+ "tokens_seen": 948699136
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005757913741223672,
+ "loss": 1.4563,
+ "theoretical_loss": 3.696056202121142,
+ "tokens_seen": 948764672
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005757753259779338,
+ "loss": 1.3792,
+ "theoretical_loss": 3.696031858757891,
+ "tokens_seen": 948830208
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005757592778335006,
+ "loss": 1.3231,
+ "theoretical_loss": 3.696007517546742,
+ "tokens_seen": 948895744
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005757432296890672,
+ "loss": 1.3163,
+ "theoretical_loss": 3.6959831784873565,
+ "tokens_seen": 948961280
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575727181544634,
+ "loss": 1.4687,
+ "theoretical_loss": 3.695958841579394,
+ "tokens_seen": 949026816
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005757111334002006,
+ "loss": 1.3181,
+ "theoretical_loss": 3.6959345068225167,
+ "tokens_seen": 949092352
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005756950852557674,
+ "loss": 1.3718,
+ "theoretical_loss": 3.6959101742163867,
+ "tokens_seen": 949157888
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575679037111334,
+ "loss": 1.3565,
+ "theoretical_loss": 3.695885843760664,
+ "tokens_seen": 949223424
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005756629889669008,
+ "loss": 1.4187,
+ "theoretical_loss": 3.695861515455011,
+ "tokens_seen": 949288960
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005756469408224674,
+ "loss": 1.372,
+ "theoretical_loss": 3.6958371892990893,
+ "tokens_seen": 949354496
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005756308926780342,
+ "loss": 1.3937,
+ "theoretical_loss": 3.695812865292561,
+ "tokens_seen": 949420032
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005756148445336008,
+ "loss": 1.4296,
+ "theoretical_loss": 3.695788543435086,
+ "tokens_seen": 949485568
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005755987963891676,
+ "loss": 1.3169,
+ "theoretical_loss": 3.695764223726328,
+ "tokens_seen": 949551104
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005755827482447342,
+ "loss": 1.3959,
+ "theoretical_loss": 3.695739906165948,
+ "tokens_seen": 949616640
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575566700100301,
+ "loss": 1.3078,
+ "theoretical_loss": 3.6957155907536086,
+ "tokens_seen": 949682176
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005755506519558676,
+ "loss": 1.4033,
+ "theoretical_loss": 3.6956912774889714,
+ "tokens_seen": 949747712
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005755346038114344,
+ "loss": 1.3494,
+ "theoretical_loss": 3.6956669663716983,
+ "tokens_seen": 949813248
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575518555667001,
+ "loss": 1.3703,
+ "theoretical_loss": 3.695642657401452,
+ "tokens_seen": 949878784
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005755025075225678,
+ "loss": 1.4269,
+ "theoretical_loss": 3.6956183505778943,
+ "tokens_seen": 949944320
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754864593781344,
+ "loss": 1.4431,
+ "theoretical_loss": 3.6955940459006884,
+ "tokens_seen": 950009856
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754704112337011,
+ "loss": 1.3288,
+ "theoretical_loss": 3.6955697433694956,
+ "tokens_seen": 950075392
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754543630892678,
+ "loss": 1.3045,
+ "theoretical_loss": 3.6955454429839785,
+ "tokens_seen": 950140928
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754383149448345,
+ "loss": 1.3202,
+ "theoretical_loss": 3.695521144743801,
+ "tokens_seen": 950206464
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 321619,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4422682523727417,
+ "objective/train/theoretical_loss": 3.695496848648624,
+ "objective/train/tokens_used": 970732000,
+ "theoretical_loss": 3.695496848648624,
+ "tokens_seen": 950272000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754222668004012,
+ "loss": 1.5001,
+ "theoretical_loss": 3.695496848648624,
+ "tokens_seen": 950272000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005754062186559679,
+ "loss": 1.3796,
+ "theoretical_loss": 3.695472554698112,
+ "tokens_seen": 950337536
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005753901705115346,
+ "loss": 1.3869,
+ "theoretical_loss": 3.695448262891926,
+ "tokens_seen": 950403072
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005753741223671013,
+ "loss": 1.4017,
+ "theoretical_loss": 3.69542397322973,
+ "tokens_seen": 950468608
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575358074222668,
+ "loss": 1.4283,
+ "theoretical_loss": 3.6953996857111875,
+ "tokens_seen": 950534144
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005753420260782347,
+ "loss": 1.3539,
+ "theoretical_loss": 3.69537540033596,
+ "tokens_seen": 950599680
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005753259779338014,
+ "loss": 1.3158,
+ "theoretical_loss": 3.6953511171037112,
+ "tokens_seen": 950665216
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005753099297893681,
+ "loss": 1.3968,
+ "theoretical_loss": 3.6953268360141047,
+ "tokens_seen": 950730752
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752938816449348,
+ "loss": 1.3244,
+ "theoretical_loss": 3.695302557066804,
+ "tokens_seen": 950796288
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752778335005015,
+ "loss": 1.4501,
+ "theoretical_loss": 3.695278280261472,
+ "tokens_seen": 950861824
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752617853560682,
+ "loss": 1.3713,
+ "theoretical_loss": 3.6952540055977714,
+ "tokens_seen": 950927360
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752457372116349,
+ "loss": 1.3846,
+ "theoretical_loss": 3.695229733075367,
+ "tokens_seen": 950992896
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752296890672016,
+ "loss": 1.3798,
+ "theoretical_loss": 3.695205462693922,
+ "tokens_seen": 951058432
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005752136409227683,
+ "loss": 1.3508,
+ "theoretical_loss": 3.695181194453099,
+ "tokens_seen": 951123968
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575197592778335,
+ "loss": 1.4068,
+ "theoretical_loss": 3.6951569283525627,
+ "tokens_seen": 951189504
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751815446339017,
+ "loss": 1.3751,
+ "theoretical_loss": 3.695132664391977,
+ "tokens_seen": 951255040
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751654964894684,
+ "loss": 1.4757,
+ "theoretical_loss": 3.6951084025710053,
+ "tokens_seen": 951320576
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751494483450351,
+ "loss": 1.3342,
+ "theoretical_loss": 3.695084142889312,
+ "tokens_seen": 951386112
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751334002006018,
+ "loss": 1.452,
+ "theoretical_loss": 3.695059885346561,
+ "tokens_seen": 951451648
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751173520561685,
+ "loss": 1.3871,
+ "theoretical_loss": 3.6950356299424163,
+ "tokens_seen": 951517184
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005751013039117353,
+ "loss": 1.499,
+ "theoretical_loss": 3.6950113766765416,
+ "tokens_seen": 951582720
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005750852557673019,
+ "loss": 1.3029,
+ "theoretical_loss": 3.694987125548602,
+ "tokens_seen": 951648256
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005750692076228687,
+ "loss": 1.3716,
+ "theoretical_loss": 3.694962876558261,
+ "tokens_seen": 951713792
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005750531594784353,
+ "loss": 1.3362,
+ "theoretical_loss": 3.6949386297051836,
+ "tokens_seen": 951779328
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000575037111334002,
+ "loss": 1.4624,
+ "theoretical_loss": 3.6949143849890342,
+ "tokens_seen": 951844864
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005750210631895687,
+ "loss": 1.3921,
+ "theoretical_loss": 3.6948901424094776,
+ "tokens_seen": 951910400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005750050150451355,
+ "loss": 1.3522,
+ "theoretical_loss": 3.6948659019661783,
+ "tokens_seen": 951975936
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749889669007021,
+ "loss": 1.4732,
+ "theoretical_loss": 3.6948416636588,
+ "tokens_seen": 952041472
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749729187562689,
+ "loss": 1.4277,
+ "theoretical_loss": 3.694817427487009,
+ "tokens_seen": 952107008
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749568706118355,
+ "loss": 1.2995,
+ "theoretical_loss": 3.6947931934504687,
+ "tokens_seen": 952172544
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749408224674023,
+ "loss": 1.4295,
+ "theoretical_loss": 3.6947689615488457,
+ "tokens_seen": 952238080
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749247743229689,
+ "loss": 1.4033,
+ "theoretical_loss": 3.6947447317818036,
+ "tokens_seen": 952303616
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005749087261785357,
+ "loss": 1.3872,
+ "theoretical_loss": 3.694720504149008,
+ "tokens_seen": 952369152
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748926780341023,
+ "loss": 1.4375,
+ "theoretical_loss": 3.694696278650124,
+ "tokens_seen": 952434688
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748766298896691,
+ "loss": 1.3641,
+ "theoretical_loss": 3.6946720552848173,
+ "tokens_seen": 952500224
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748605817452357,
+ "loss": 1.379,
+ "theoretical_loss": 3.694647834052753,
+ "tokens_seen": 952565760
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748445336008025,
+ "loss": 1.2778,
+ "theoretical_loss": 3.6946236149535956,
+ "tokens_seen": 952631296
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748284854563691,
+ "loss": 1.3413,
+ "theoretical_loss": 3.6945993979870115,
+ "tokens_seen": 952696832
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005748124373119359,
+ "loss": 1.4252,
+ "theoretical_loss": 3.694575183152666,
+ "tokens_seen": 952762368
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747963891675025,
+ "loss": 1.3981,
+ "theoretical_loss": 3.6945509704502246,
+ "tokens_seen": 952827904
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747803410230693,
+ "loss": 1.3961,
+ "theoretical_loss": 3.6945267598793534,
+ "tokens_seen": 952893440
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747642928786359,
+ "loss": 1.3123,
+ "theoretical_loss": 3.694502551439718,
+ "tokens_seen": 952958976
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747482447342027,
+ "loss": 1.396,
+ "theoretical_loss": 3.6944783451309835,
+ "tokens_seen": 953024512
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747321965897693,
+ "loss": 1.4178,
+ "theoretical_loss": 3.694454140952817,
+ "tokens_seen": 953090048
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747161484453361,
+ "loss": 1.3638,
+ "theoretical_loss": 3.694429938904884,
+ "tokens_seen": 953155584
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005747001003009027,
+ "loss": 1.5011,
+ "theoretical_loss": 3.69440573898685,
+ "tokens_seen": 953221120
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746840521564695,
+ "loss": 1.3855,
+ "theoretical_loss": 3.694381541198382,
+ "tokens_seen": 953286656
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746680040120361,
+ "loss": 1.3978,
+ "theoretical_loss": 3.694357345539146,
+ "tokens_seen": 953352192
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746519558676029,
+ "loss": 1.4071,
+ "theoretical_loss": 3.6943331520088076,
+ "tokens_seen": 953417728
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746359077231695,
+ "loss": 1.4986,
+ "theoretical_loss": 3.694308960607034,
+ "tokens_seen": 953483264
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 322577,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3857512474060059,
+ "objective/train/theoretical_loss": 3.694284771333491,
+ "objective/train/tokens_used": 974008800,
+ "theoretical_loss": 3.694284771333491,
+ "tokens_seen": 953548800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746198595787363,
+ "loss": 1.3747,
+ "theoretical_loss": 3.694284771333491,
+ "tokens_seen": 953548800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005746038114343029,
+ "loss": 1.4324,
+ "theoretical_loss": 3.6942605841878464,
+ "tokens_seen": 953614336
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745877632898697,
+ "loss": 1.35,
+ "theoretical_loss": 3.694236399169765,
+ "tokens_seen": 953679872
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745717151454363,
+ "loss": 1.3603,
+ "theoretical_loss": 3.6942122162789146,
+ "tokens_seen": 953745408
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745556670010031,
+ "loss": 1.444,
+ "theoretical_loss": 3.694188035514962,
+ "tokens_seen": 953810944
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745396188565697,
+ "loss": 1.3485,
+ "theoretical_loss": 3.694163856877573,
+ "tokens_seen": 953876480
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745235707121365,
+ "loss": 1.2706,
+ "theoretical_loss": 3.6941396803664155,
+ "tokens_seen": 953942016
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005745075225677031,
+ "loss": 1.4315,
+ "theoretical_loss": 3.6941155059811566,
+ "tokens_seen": 954007552
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744914744232699,
+ "loss": 1.406,
+ "theoretical_loss": 3.6940913337214627,
+ "tokens_seen": 954073088
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744754262788365,
+ "loss": 1.3743,
+ "theoretical_loss": 3.6940671635870013,
+ "tokens_seen": 954138624
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744593781344033,
+ "loss": 1.3819,
+ "theoretical_loss": 3.6940429955774388,
+ "tokens_seen": 954204160
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744433299899699,
+ "loss": 1.4316,
+ "theoretical_loss": 3.6940188296924434,
+ "tokens_seen": 954269696
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744272818455367,
+ "loss": 1.4471,
+ "theoretical_loss": 3.6939946659316822,
+ "tokens_seen": 954335232
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005744112337011033,
+ "loss": 1.3403,
+ "theoretical_loss": 3.693970504294823,
+ "tokens_seen": 954400768
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743951855566701,
+ "loss": 1.4192,
+ "theoretical_loss": 3.6939463447815317,
+ "tokens_seen": 954466304
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743791374122367,
+ "loss": 1.4291,
+ "theoretical_loss": 3.693922187391478,
+ "tokens_seen": 954531840
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743630892678035,
+ "loss": 1.3475,
+ "theoretical_loss": 3.6938980321243284,
+ "tokens_seen": 954597376
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743470411233701,
+ "loss": 1.3311,
+ "theoretical_loss": 3.6938738789797507,
+ "tokens_seen": 954662912
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743309929789369,
+ "loss": 1.3657,
+ "theoretical_loss": 3.6938497279574127,
+ "tokens_seen": 954728448
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005743149448345035,
+ "loss": 1.2317,
+ "theoretical_loss": 3.6938255790569823,
+ "tokens_seen": 954793984
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742988966900703,
+ "loss": 1.476,
+ "theoretical_loss": 3.6938014322781276,
+ "tokens_seen": 954859520
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742828485456369,
+ "loss": 1.2893,
+ "theoretical_loss": 3.6937772876205166,
+ "tokens_seen": 954925056
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742668004012037,
+ "loss": 1.3469,
+ "theoretical_loss": 3.6937531450838175,
+ "tokens_seen": 954990592
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742507522567703,
+ "loss": 1.3121,
+ "theoretical_loss": 3.6937290046676976,
+ "tokens_seen": 955056128
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742347041123371,
+ "loss": 1.4125,
+ "theoretical_loss": 3.693704866371826,
+ "tokens_seen": 955121664
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742186559679037,
+ "loss": 1.3416,
+ "theoretical_loss": 3.6936807301958714,
+ "tokens_seen": 955187200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005742026078234705,
+ "loss": 1.4748,
+ "theoretical_loss": 3.693656596139501,
+ "tokens_seen": 955252736
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741865596790371,
+ "loss": 1.3533,
+ "theoretical_loss": 3.693632464202384,
+ "tokens_seen": 955318272
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741705115346039,
+ "loss": 1.417,
+ "theoretical_loss": 3.693608334384189,
+ "tokens_seen": 955383808
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741544633901705,
+ "loss": 1.3945,
+ "theoretical_loss": 3.6935842066845845,
+ "tokens_seen": 955449344
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741384152457373,
+ "loss": 1.3676,
+ "theoretical_loss": 3.6935600811032394,
+ "tokens_seen": 955514880
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741223671013039,
+ "loss": 1.4023,
+ "theoretical_loss": 3.6935359576398215,
+ "tokens_seen": 955580416
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005741063189568707,
+ "loss": 1.3785,
+ "theoretical_loss": 3.6935118362940007,
+ "tokens_seen": 955645952
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005740902708124373,
+ "loss": 1.5164,
+ "theoretical_loss": 3.6934877170654454,
+ "tokens_seen": 955711488
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000574074222668004,
+ "loss": 1.3221,
+ "theoretical_loss": 3.693463599953825,
+ "tokens_seen": 955777024
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005740581745235707,
+ "loss": 1.4135,
+ "theoretical_loss": 3.693439484958808,
+ "tokens_seen": 955842560
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005740421263791374,
+ "loss": 1.512,
+ "theoretical_loss": 3.693415372080064,
+ "tokens_seen": 955908096
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005740260782347041,
+ "loss": 1.3623,
+ "theoretical_loss": 3.693391261317262,
+ "tokens_seen": 955973632
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005740100300902708,
+ "loss": 1.4184,
+ "theoretical_loss": 3.693367152670071,
+ "tokens_seen": 956039168
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005739939819458375,
+ "loss": 1.486,
+ "theoretical_loss": 3.6933430461381613,
+ "tokens_seen": 956104704
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005739779338014042,
+ "loss": 1.3866,
+ "theoretical_loss": 3.693318941721201,
+ "tokens_seen": 956170240
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005739618856569709,
+ "loss": 1.4363,
+ "theoretical_loss": 3.693294839418861,
+ "tokens_seen": 956235776
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005739458375125376,
+ "loss": 1.4961,
+ "theoretical_loss": 3.69327073923081,
+ "tokens_seen": 956301312
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005739297893681043,
+ "loss": 1.2741,
+ "theoretical_loss": 3.6932466411567173,
+ "tokens_seen": 956366848
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573913741223671,
+ "loss": 1.3535,
+ "theoretical_loss": 3.693222545196254,
+ "tokens_seen": 956432384
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738976930792377,
+ "loss": 1.3106,
+ "theoretical_loss": 3.693198451349089,
+ "tokens_seen": 956497920
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738816449348044,
+ "loss": 1.3628,
+ "theoretical_loss": 3.6931743596148916,
+ "tokens_seen": 956563456
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738655967903711,
+ "loss": 1.408,
+ "theoretical_loss": 3.6931502699933336,
+ "tokens_seen": 956628992
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738495486459378,
+ "loss": 1.2204,
+ "theoretical_loss": 3.693126182484083,
+ "tokens_seen": 956694528
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738335005015045,
+ "loss": 1.3629,
+ "theoretical_loss": 3.693102097086811,
+ "tokens_seen": 956760064
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 323899,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3709452152252197,
+ "objective/train/theoretical_loss": 3.693078013801187,
+ "objective/train/tokens_used": 977285600,
+ "theoretical_loss": 3.693078013801187,
+ "tokens_seen": 956825600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005738174523570712,
+ "loss": 1.3689,
+ "theoretical_loss": 3.693078013801187,
+ "tokens_seen": 956825600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573801404212638,
+ "loss": 1.3385,
+ "theoretical_loss": 3.6930539326268828,
+ "tokens_seen": 956891136
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005737853560682046,
+ "loss": 1.407,
+ "theoretical_loss": 3.6930298535635666,
+ "tokens_seen": 956956672
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005737693079237713,
+ "loss": 1.3102,
+ "theoretical_loss": 3.693005776610911,
+ "tokens_seen": 957022208
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573753259779338,
+ "loss": 1.3474,
+ "theoretical_loss": 3.692981701768585,
+ "tokens_seen": 957087744
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005737372116349047,
+ "loss": 1.3031,
+ "theoretical_loss": 3.6929576290362593,
+ "tokens_seen": 957153280
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005737211634904714,
+ "loss": 1.3257,
+ "theoretical_loss": 3.6929335584136047,
+ "tokens_seen": 957218816
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005737051153460382,
+ "loss": 1.41,
+ "theoretical_loss": 3.692909489900292,
+ "tokens_seen": 957284352
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005736890672016048,
+ "loss": 1.4056,
+ "theoretical_loss": 3.6928854234959925,
+ "tokens_seen": 957349888
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005736730190571716,
+ "loss": 1.2758,
+ "theoretical_loss": 3.692861359200376,
+ "tokens_seen": 957415424
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005736569709127382,
+ "loss": 1.3864,
+ "theoretical_loss": 3.6928372970131136,
+ "tokens_seen": 957480960
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573640922768305,
+ "loss": 1.3342,
+ "theoretical_loss": 3.692813236933877,
+ "tokens_seen": 957546496
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005736248746238716,
+ "loss": 1.3311,
+ "theoretical_loss": 3.692789178962337,
+ "tokens_seen": 957612032
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005736088264794384,
+ "loss": 1.3913,
+ "theoretical_loss": 3.692765123098164,
+ "tokens_seen": 957677568
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573592778335005,
+ "loss": 1.3976,
+ "theoretical_loss": 3.69274106934103,
+ "tokens_seen": 957743104
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005735767301905718,
+ "loss": 1.5343,
+ "theoretical_loss": 3.6927170176906063,
+ "tokens_seen": 957808640
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005735606820461384,
+ "loss": 1.3732,
+ "theoretical_loss": 3.692692968146564,
+ "tokens_seen": 957874176
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005735446339017052,
+ "loss": 1.3071,
+ "theoretical_loss": 3.692668920708574,
+ "tokens_seen": 957939712
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005735285857572718,
+ "loss": 1.3795,
+ "theoretical_loss": 3.692644875376309,
+ "tokens_seen": 958005248
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005735125376128386,
+ "loss": 1.4814,
+ "theoretical_loss": 3.6926208321494394,
+ "tokens_seen": 958070784
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005734964894684052,
+ "loss": 1.4087,
+ "theoretical_loss": 3.692596791027638,
+ "tokens_seen": 958136320
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573480441323972,
+ "loss": 1.3891,
+ "theoretical_loss": 3.6925727520105753,
+ "tokens_seen": 958201856
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005734643931795386,
+ "loss": 1.2506,
+ "theoretical_loss": 3.692548715097924,
+ "tokens_seen": 958267392
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005734483450351054,
+ "loss": 1.3148,
+ "theoretical_loss": 3.6925246802893557,
+ "tokens_seen": 958332928
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573432296890672,
+ "loss": 1.4035,
+ "theoretical_loss": 3.6925006475845423,
+ "tokens_seen": 958398464
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005734162487462388,
+ "loss": 1.226,
+ "theoretical_loss": 3.692476616983156,
+ "tokens_seen": 958464000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005734002006018054,
+ "loss": 1.4648,
+ "theoretical_loss": 3.6924525884848687,
+ "tokens_seen": 958529536
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005733841524573722,
+ "loss": 1.4966,
+ "theoretical_loss": 3.6924285620893524,
+ "tokens_seen": 958595072
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005733681043129388,
+ "loss": 1.3732,
+ "theoretical_loss": 3.69240453779628,
+ "tokens_seen": 958660608
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005733520561685056,
+ "loss": 1.4946,
+ "theoretical_loss": 3.692380515605323,
+ "tokens_seen": 958726144
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005733360080240722,
+ "loss": 1.3418,
+ "theoretical_loss": 3.6923564955161545,
+ "tokens_seen": 958791680
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573319959879639,
+ "loss": 1.3368,
+ "theoretical_loss": 3.6923324775284465,
+ "tokens_seen": 958857216
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005733039117352056,
+ "loss": 1.5341,
+ "theoretical_loss": 3.692308461641872,
+ "tokens_seen": 958922752
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005732878635907724,
+ "loss": 1.3288,
+ "theoretical_loss": 3.6922844478561028,
+ "tokens_seen": 958988288
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573271815446339,
+ "loss": 1.3637,
+ "theoretical_loss": 3.6922604361708125,
+ "tokens_seen": 959053824
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005732557673019058,
+ "loss": 1.4385,
+ "theoretical_loss": 3.692236426585673,
+ "tokens_seen": 959119360
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005732397191574724,
+ "loss": 1.2887,
+ "theoretical_loss": 3.6922124191003576,
+ "tokens_seen": 959184896
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005732236710130392,
+ "loss": 1.3529,
+ "theoretical_loss": 3.6921884137145393,
+ "tokens_seen": 959250432
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005732076228686058,
+ "loss": 1.394,
+ "theoretical_loss": 3.692164410427891,
+ "tokens_seen": 959315968
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005731915747241726,
+ "loss": 1.3602,
+ "theoretical_loss": 3.6921404092400856,
+ "tokens_seen": 959381504
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005731755265797392,
+ "loss": 1.407,
+ "theoretical_loss": 3.6921164101507964,
+ "tokens_seen": 959447040
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573159478435306,
+ "loss": 1.4307,
+ "theoretical_loss": 3.6920924131596964,
+ "tokens_seen": 959512576
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005731434302908726,
+ "loss": 1.4011,
+ "theoretical_loss": 3.692068418266459,
+ "tokens_seen": 959578112
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005731273821464394,
+ "loss": 1.4794,
+ "theoretical_loss": 3.6920444254707574,
+ "tokens_seen": 959643648
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000573111334002006,
+ "loss": 1.4011,
+ "theoretical_loss": 3.6920204347722656,
+ "tokens_seen": 959709184
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730952858575728,
+ "loss": 1.3967,
+ "theoretical_loss": 3.6919964461706565,
+ "tokens_seen": 959774720
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730792377131394,
+ "loss": 1.4577,
+ "theoretical_loss": 3.691972459665603,
+ "tokens_seen": 959840256
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730631895687062,
+ "loss": 1.4282,
+ "theoretical_loss": 3.6919484752567806,
+ "tokens_seen": 959905792
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730471414242728,
+ "loss": 1.2874,
+ "theoretical_loss": 3.691924492943861,
+ "tokens_seen": 959971328
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730310932798396,
+ "loss": 1.3386,
+ "theoretical_loss": 3.6919005127265194,
+ "tokens_seen": 960036864
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 325104,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.465695858001709,
+ "objective/train/theoretical_loss": 3.6918765346044298,
+ "objective/train/tokens_used": 980562400,
+ "theoretical_loss": 3.6918765346044298,
+ "tokens_seen": 960102400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005730150451354062,
+ "loss": 1.4385,
+ "theoretical_loss": 3.6918765346044298,
+ "tokens_seen": 960102400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000572998996990973,
+ "loss": 1.4227,
+ "theoretical_loss": 3.6918525585772644,
+ "tokens_seen": 960167936
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729829488465396,
+ "loss": 1.429,
+ "theoretical_loss": 3.691828584644699,
+ "tokens_seen": 960233472
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729669007021064,
+ "loss": 1.3622,
+ "theoretical_loss": 3.6918046128064064,
+ "tokens_seen": 960299008
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729508525576731,
+ "loss": 1.5271,
+ "theoretical_loss": 3.691780643062062,
+ "tokens_seen": 960364544
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729348044132398,
+ "loss": 1.4355,
+ "theoretical_loss": 3.6917566754113387,
+ "tokens_seen": 960430080
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729187562688065,
+ "loss": 1.3899,
+ "theoretical_loss": 3.6917327098539117,
+ "tokens_seen": 960495616
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005729027081243732,
+ "loss": 1.3185,
+ "theoretical_loss": 3.6917087463894553,
+ "tokens_seen": 960561152
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005728866599799399,
+ "loss": 1.2467,
+ "theoretical_loss": 3.6916847850176433,
+ "tokens_seen": 960626688
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005728706118355066,
+ "loss": 1.41,
+ "theoretical_loss": 3.6916608257381505,
+ "tokens_seen": 960692224
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005728545636910733,
+ "loss": 1.3998,
+ "theoretical_loss": 3.6916368685506518,
+ "tokens_seen": 960757760
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.00057283851554664,
+ "loss": 1.3782,
+ "theoretical_loss": 3.691612913454822,
+ "tokens_seen": 960823296
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005728224674022067,
+ "loss": 1.5504,
+ "theoretical_loss": 3.6915889604503356,
+ "tokens_seen": 960888832
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005728064192577734,
+ "loss": 1.3247,
+ "theoretical_loss": 3.6915650095368666,
+ "tokens_seen": 960954368
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.00057279037111334,
+ "loss": 1.3303,
+ "theoretical_loss": 3.6915410607140915,
+ "tokens_seen": 961019904
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005727743229689068,
+ "loss": 1.4098,
+ "theoretical_loss": 3.6915171139816834,
+ "tokens_seen": 961085440
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005727582748244734,
+ "loss": 1.3843,
+ "theoretical_loss": 3.691493169339319,
+ "tokens_seen": 961150976
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005727422266800402,
+ "loss": 1.3701,
+ "theoretical_loss": 3.691469226786672,
+ "tokens_seen": 961216512
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005727261785356068,
+ "loss": 1.3938,
+ "theoretical_loss": 3.6914452863234186,
+ "tokens_seen": 961282048
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005727101303911736,
+ "loss": 1.3824,
+ "theoretical_loss": 3.691421347949234,
+ "tokens_seen": 961347584
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005726940822467402,
+ "loss": 1.4114,
+ "theoretical_loss": 3.691397411663792,
+ "tokens_seen": 961413120
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005726780341023069,
+ "loss": 1.3189,
+ "theoretical_loss": 3.6913734774667706,
+ "tokens_seen": 961478656
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005726619859578736,
+ "loss": 1.3585,
+ "theoretical_loss": 3.691349545357843,
+ "tokens_seen": 961544192
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005726459378134403,
+ "loss": 1.3614,
+ "theoretical_loss": 3.6913256153366856,
+ "tokens_seen": 961609728
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000572629889669007,
+ "loss": 1.3773,
+ "theoretical_loss": 3.691301687402974,
+ "tokens_seen": 961675264
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005726138415245737,
+ "loss": 1.3317,
+ "theoretical_loss": 3.6912777615563837,
+ "tokens_seen": 961740800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725977933801404,
+ "loss": 1.2783,
+ "theoretical_loss": 3.69125383779659,
+ "tokens_seen": 961806336
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725817452357071,
+ "loss": 1.3975,
+ "theoretical_loss": 3.69122991612327,
+ "tokens_seen": 961871872
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725656970912738,
+ "loss": 1.3962,
+ "theoretical_loss": 3.691205996536099,
+ "tokens_seen": 961937408
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725496489468405,
+ "loss": 1.395,
+ "theoretical_loss": 3.6911820790347525,
+ "tokens_seen": 962002944
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725336008024072,
+ "loss": 1.3541,
+ "theoretical_loss": 3.6911581636189066,
+ "tokens_seen": 962068480
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725175526579739,
+ "loss": 1.3784,
+ "theoretical_loss": 3.691134250288238,
+ "tokens_seen": 962134016
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005725015045135406,
+ "loss": 1.3566,
+ "theoretical_loss": 3.6911103390424227,
+ "tokens_seen": 962199552
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005724854563691073,
+ "loss": 1.2805,
+ "theoretical_loss": 3.6910864298811363,
+ "tokens_seen": 962265088
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000572469408224674,
+ "loss": 1.3285,
+ "theoretical_loss": 3.6910625228040557,
+ "tokens_seen": 962330624
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005724533600802407,
+ "loss": 1.4194,
+ "theoretical_loss": 3.6910386178108574,
+ "tokens_seen": 962396160
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005724373119358074,
+ "loss": 1.3384,
+ "theoretical_loss": 3.691014714901218,
+ "tokens_seen": 962461696
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005724212637913741,
+ "loss": 1.4406,
+ "theoretical_loss": 3.6909908140748136,
+ "tokens_seen": 962527232
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005724052156469408,
+ "loss": 1.383,
+ "theoretical_loss": 3.6909669153313205,
+ "tokens_seen": 962592768
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005723891675025075,
+ "loss": 1.2981,
+ "theoretical_loss": 3.690943018670416,
+ "tokens_seen": 962658304
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005723731193580742,
+ "loss": 1.3386,
+ "theoretical_loss": 3.690919124091777,
+ "tokens_seen": 962723840
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005723570712136409,
+ "loss": 1.2929,
+ "theoretical_loss": 3.69089523159508,
+ "tokens_seen": 962789376
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005723410230692077,
+ "loss": 1.4045,
+ "theoretical_loss": 3.690871341180001,
+ "tokens_seen": 962854912
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005723249749247743,
+ "loss": 1.3683,
+ "theoretical_loss": 3.690847452846219,
+ "tokens_seen": 962920448
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000572308926780341,
+ "loss": 1.3836,
+ "theoretical_loss": 3.6908235665934095,
+ "tokens_seen": 962985984
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722928786359077,
+ "loss": 1.3284,
+ "theoretical_loss": 3.6907996824212503,
+ "tokens_seen": 963051520
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722768304914745,
+ "loss": 1.4417,
+ "theoretical_loss": 3.690775800329418,
+ "tokens_seen": 963117056
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722607823470411,
+ "loss": 1.4012,
+ "theoretical_loss": 3.6907519203175907,
+ "tokens_seen": 963182592
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722447342026079,
+ "loss": 1.3698,
+ "theoretical_loss": 3.6907280423854445,
+ "tokens_seen": 963248128
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722286860581745,
+ "loss": 1.3087,
+ "theoretical_loss": 3.690704166532658,
+ "tokens_seen": 963313664
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 326047,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.373966932296753,
+ "objective/train/theoretical_loss": 3.690680292758908,
+ "objective/train/tokens_used": 983839200,
+ "theoretical_loss": 3.690680292758908,
+ "tokens_seen": 963379200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005722126379137413,
+ "loss": 1.437,
+ "theoretical_loss": 3.690680292758908,
+ "tokens_seen": 963379200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721965897693079,
+ "loss": 1.4679,
+ "theoretical_loss": 3.690656421063873,
+ "tokens_seen": 963444736
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721805416248747,
+ "loss": 1.3647,
+ "theoretical_loss": 3.690632551447229,
+ "tokens_seen": 963510272
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721644934804413,
+ "loss": 1.3025,
+ "theoretical_loss": 3.690608683908655,
+ "tokens_seen": 963575808
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721484453360081,
+ "loss": 1.2803,
+ "theoretical_loss": 3.6905848184478285,
+ "tokens_seen": 963641344
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721323971915747,
+ "loss": 1.3523,
+ "theoretical_loss": 3.690560955064427,
+ "tokens_seen": 963706880
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721163490471415,
+ "loss": 1.4114,
+ "theoretical_loss": 3.690537093758129,
+ "tokens_seen": 963772416
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005721003009027081,
+ "loss": 1.4266,
+ "theoretical_loss": 3.690513234528612,
+ "tokens_seen": 963837952
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720842527582749,
+ "loss": 1.3681,
+ "theoretical_loss": 3.690489377375554,
+ "tokens_seen": 963903488
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720682046138415,
+ "loss": 1.3101,
+ "theoretical_loss": 3.6904655222986333,
+ "tokens_seen": 963969024
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720521564694083,
+ "loss": 1.3685,
+ "theoretical_loss": 3.690441669297528,
+ "tokens_seen": 964034560
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720361083249749,
+ "loss": 1.4999,
+ "theoretical_loss": 3.6904178183719165,
+ "tokens_seen": 964100096
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720200601805417,
+ "loss": 1.3722,
+ "theoretical_loss": 3.690393969521477,
+ "tokens_seen": 964165632
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005720040120361083,
+ "loss": 1.4375,
+ "theoretical_loss": 3.6903701227458883,
+ "tokens_seen": 964231168
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719879638916751,
+ "loss": 1.3005,
+ "theoretical_loss": 3.6903462780448284,
+ "tokens_seen": 964296704
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719719157472417,
+ "loss": 1.296,
+ "theoretical_loss": 3.690322435417976,
+ "tokens_seen": 964362240
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719558676028085,
+ "loss": 1.2849,
+ "theoretical_loss": 3.69029859486501,
+ "tokens_seen": 964427776
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719398194583751,
+ "loss": 1.314,
+ "theoretical_loss": 3.690274756385609,
+ "tokens_seen": 964493312
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719237713139419,
+ "loss": 1.4088,
+ "theoretical_loss": 3.6902509199794507,
+ "tokens_seen": 964558848
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005719077231695085,
+ "loss": 1.4873,
+ "theoretical_loss": 3.6902270856462156,
+ "tokens_seen": 964624384
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718916750250753,
+ "loss": 1.3435,
+ "theoretical_loss": 3.690203253385582,
+ "tokens_seen": 964689920
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718756268806419,
+ "loss": 1.4194,
+ "theoretical_loss": 3.690179423197229,
+ "tokens_seen": 964755456
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718595787362087,
+ "loss": 1.4869,
+ "theoretical_loss": 3.690155595080835,
+ "tokens_seen": 964820992
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718435305917753,
+ "loss": 1.3384,
+ "theoretical_loss": 3.6901317690360793,
+ "tokens_seen": 964886528
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718274824473421,
+ "loss": 1.4252,
+ "theoretical_loss": 3.6901079450626417,
+ "tokens_seen": 964952064
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005718114343029087,
+ "loss": 1.4402,
+ "theoretical_loss": 3.6900841231602017,
+ "tokens_seen": 965017600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005717953861584755,
+ "loss": 1.3564,
+ "theoretical_loss": 3.690060303328438,
+ "tokens_seen": 965083136
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005717793380140421,
+ "loss": 1.3953,
+ "theoretical_loss": 3.6900364855670293,
+ "tokens_seen": 965148672
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005717632898696089,
+ "loss": 1.3565,
+ "theoretical_loss": 3.6900126698756566,
+ "tokens_seen": 965214208
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005717472417251756,
+ "loss": 1.3693,
+ "theoretical_loss": 3.689988856253998,
+ "tokens_seen": 965279744
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005717311935807423,
+ "loss": 1.3556,
+ "theoretical_loss": 3.6899650447017347,
+ "tokens_seen": 965345280
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000571715145436309,
+ "loss": 1.403,
+ "theoretical_loss": 3.6899412352185452,
+ "tokens_seen": 965410816
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716990972918757,
+ "loss": 1.2627,
+ "theoretical_loss": 3.68991742780411,
+ "tokens_seen": 965476352
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716830491474424,
+ "loss": 1.3862,
+ "theoretical_loss": 3.689893622458108,
+ "tokens_seen": 965541888
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716670010030091,
+ "loss": 1.406,
+ "theoretical_loss": 3.6898698191802195,
+ "tokens_seen": 965607424
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716509528585758,
+ "loss": 1.3838,
+ "theoretical_loss": 3.689846017970125,
+ "tokens_seen": 965672960
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716349047141425,
+ "loss": 1.404,
+ "theoretical_loss": 3.689822218827505,
+ "tokens_seen": 965738496
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716188565697092,
+ "loss": 1.254,
+ "theoretical_loss": 3.6897984217520383,
+ "tokens_seen": 965804032
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005716028084252759,
+ "loss": 1.4042,
+ "theoretical_loss": 3.6897746267434055,
+ "tokens_seen": 965869568
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005715867602808426,
+ "loss": 1.3868,
+ "theoretical_loss": 3.6897508338012877,
+ "tokens_seen": 965935104
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005715707121364093,
+ "loss": 1.1966,
+ "theoretical_loss": 3.689727042925364,
+ "tokens_seen": 966000640
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000571554663991976,
+ "loss": 1.4443,
+ "theoretical_loss": 3.689703254115315,
+ "tokens_seen": 966066176
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005715386158475427,
+ "loss": 1.4936,
+ "theoretical_loss": 3.689679467370823,
+ "tokens_seen": 966131712
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005715225677031094,
+ "loss": 1.3745,
+ "theoretical_loss": 3.6896556826915656,
+ "tokens_seen": 966197248
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005715065195586761,
+ "loss": 1.3299,
+ "theoretical_loss": 3.689631900077226,
+ "tokens_seen": 966262784
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714904714142428,
+ "loss": 1.5032,
+ "theoretical_loss": 3.6896081195274837,
+ "tokens_seen": 966328320
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714744232698095,
+ "loss": 1.3185,
+ "theoretical_loss": 3.6895843410420195,
+ "tokens_seen": 966393856
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714583751253762,
+ "loss": 1.4487,
+ "theoretical_loss": 3.689560564620514,
+ "tokens_seen": 966459392
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714423269809429,
+ "loss": 1.3979,
+ "theoretical_loss": 3.689536790262649,
+ "tokens_seen": 966524928
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714262788365096,
+ "loss": 1.343,
+ "theoretical_loss": 3.6895130179681046,
+ "tokens_seen": 966590464
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 327277,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.415788173675537,
+ "objective/train/theoretical_loss": 3.689489247736563,
+ "objective/train/tokens_used": 987116000,
+ "theoretical_loss": 3.689489247736563,
+ "tokens_seen": 966656000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005714102306920763,
+ "loss": 1.3681,
+ "theoretical_loss": 3.689489247736563,
+ "tokens_seen": 966656000
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713941825476429,
+ "loss": 1.3016,
+ "theoretical_loss": 3.689465479567703,
+ "tokens_seen": 966721536
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713781344032097,
+ "loss": 1.4658,
+ "theoretical_loss": 3.6894417134612087,
+ "tokens_seen": 966787072
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713620862587763,
+ "loss": 1.4065,
+ "theoretical_loss": 3.6894179494167596,
+ "tokens_seen": 966852608
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713460381143431,
+ "loss": 1.3852,
+ "theoretical_loss": 3.6893941874340372,
+ "tokens_seen": 966918144
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713299899699097,
+ "loss": 1.349,
+ "theoretical_loss": 3.6893704275127233,
+ "tokens_seen": 966983680
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005713139418254765,
+ "loss": 1.3741,
+ "theoretical_loss": 3.6893466696524992,
+ "tokens_seen": 967049216
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712978936810431,
+ "loss": 1.3975,
+ "theoretical_loss": 3.689322913853047,
+ "tokens_seen": 967114752
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712818455366099,
+ "loss": 1.4219,
+ "theoretical_loss": 3.689299160114047,
+ "tokens_seen": 967180288
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712657973921765,
+ "loss": 1.4576,
+ "theoretical_loss": 3.689275408435182,
+ "tokens_seen": 967245824
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712497492477432,
+ "loss": 1.4267,
+ "theoretical_loss": 3.6892516588161333,
+ "tokens_seen": 967311360
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712337011033099,
+ "loss": 1.3833,
+ "theoretical_loss": 3.689227911256583,
+ "tokens_seen": 967376896
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712176529588766,
+ "loss": 1.3428,
+ "theoretical_loss": 3.6892041657562125,
+ "tokens_seen": 967442432
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005712016048144433,
+ "loss": 1.3452,
+ "theoretical_loss": 3.689180422314705,
+ "tokens_seen": 967507968
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.00057118555667001,
+ "loss": 1.3975,
+ "theoretical_loss": 3.689156680931741,
+ "tokens_seen": 967573504
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005711695085255767,
+ "loss": 1.3263,
+ "theoretical_loss": 3.6891329416070033,
+ "tokens_seen": 967639040
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005711534603811434,
+ "loss": 1.3677,
+ "theoretical_loss": 3.6891092043401743,
+ "tokens_seen": 967704576
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005711374122367101,
+ "loss": 1.332,
+ "theoretical_loss": 3.6890854691309354,
+ "tokens_seen": 967770112
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005711213640922768,
+ "loss": 1.3547,
+ "theoretical_loss": 3.6890617359789704,
+ "tokens_seen": 967835648
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005711053159478435,
+ "loss": 1.3289,
+ "theoretical_loss": 3.689038004883961,
+ "tokens_seen": 967901184
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005710892678034102,
+ "loss": 1.3085,
+ "theoretical_loss": 3.6890142758455884,
+ "tokens_seen": 967966720
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000571073219658977,
+ "loss": 1.3303,
+ "theoretical_loss": 3.6889905488635373,
+ "tokens_seen": 968032256
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005710571715145436,
+ "loss": 1.3869,
+ "theoretical_loss": 3.6889668239374886,
+ "tokens_seen": 968097792
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005710411233701103,
+ "loss": 1.3357,
+ "theoretical_loss": 3.6889431010671254,
+ "tokens_seen": 968163328
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000571025075225677,
+ "loss": 1.3994,
+ "theoretical_loss": 3.6889193802521314,
+ "tokens_seen": 968228864
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005710090270812437,
+ "loss": 1.3353,
+ "theoretical_loss": 3.6888956614921886,
+ "tokens_seen": 968294400
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005709929789368104,
+ "loss": 1.4054,
+ "theoretical_loss": 3.68887194478698,
+ "tokens_seen": 968359936
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005709769307923772,
+ "loss": 1.2776,
+ "theoretical_loss": 3.6888482301361885,
+ "tokens_seen": 968425472
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005709608826479438,
+ "loss": 1.3476,
+ "theoretical_loss": 3.688824517539497,
+ "tokens_seen": 968491008
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005709448345035106,
+ "loss": 1.3195,
+ "theoretical_loss": 3.688800806996589,
+ "tokens_seen": 968556544
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005709287863590772,
+ "loss": 1.3728,
+ "theoretical_loss": 3.6887770985071473,
+ "tokens_seen": 968622080
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570912738214644,
+ "loss": 1.4356,
+ "theoretical_loss": 3.688753392070855,
+ "tokens_seen": 968687616
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708966900702106,
+ "loss": 1.3998,
+ "theoretical_loss": 3.6887296876873963,
+ "tokens_seen": 968753152
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708806419257774,
+ "loss": 1.3481,
+ "theoretical_loss": 3.688705985356454,
+ "tokens_seen": 968818688
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570864593781344,
+ "loss": 1.3533,
+ "theoretical_loss": 3.688682285077711,
+ "tokens_seen": 968884224
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708485456369108,
+ "loss": 1.4222,
+ "theoretical_loss": 3.6886585868508517,
+ "tokens_seen": 968949760
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708324974924774,
+ "loss": 1.3566,
+ "theoretical_loss": 3.688634890675559,
+ "tokens_seen": 969015296
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708164493480442,
+ "loss": 1.4013,
+ "theoretical_loss": 3.688611196551517,
+ "tokens_seen": 969080832
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005708004012036108,
+ "loss": 1.4087,
+ "theoretical_loss": 3.68858750447841,
+ "tokens_seen": 969146368
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005707843530591776,
+ "loss": 1.3192,
+ "theoretical_loss": 3.6885638144559207,
+ "tokens_seen": 969211904
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005707683049147442,
+ "loss": 1.3206,
+ "theoretical_loss": 3.6885401264837334,
+ "tokens_seen": 969277440
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570752256770311,
+ "loss": 1.4492,
+ "theoretical_loss": 3.688516440561532,
+ "tokens_seen": 969342976
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005707362086258776,
+ "loss": 1.311,
+ "theoretical_loss": 3.6884927566890005,
+ "tokens_seen": 969408512
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005707201604814444,
+ "loss": 1.2671,
+ "theoretical_loss": 3.6884690748658233,
+ "tokens_seen": 969474048
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570704112337011,
+ "loss": 1.4067,
+ "theoretical_loss": 3.6884453950916845,
+ "tokens_seen": 969539584
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706880641925778,
+ "loss": 1.3854,
+ "theoretical_loss": 3.6884217173662677,
+ "tokens_seen": 969605120
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706720160481444,
+ "loss": 1.486,
+ "theoretical_loss": 3.6883980416892577,
+ "tokens_seen": 969670656
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706559679037112,
+ "loss": 1.3749,
+ "theoretical_loss": 3.688374368060339,
+ "tokens_seen": 969736192
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706399197592778,
+ "loss": 1.2727,
+ "theoretical_loss": 3.688350696479196,
+ "tokens_seen": 969801728
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706238716148446,
+ "loss": 1.3913,
+ "theoretical_loss": 3.6883270269455126,
+ "tokens_seen": 969867264
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 328315,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.420799970626831,
+ "objective/train/theoretical_loss": 3.688303359458974,
+ "objective/train/tokens_used": 990392800,
+ "theoretical_loss": 3.688303359458974,
+ "tokens_seen": 969932800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005706078234704112,
+ "loss": 1.4175,
+ "theoretical_loss": 3.688303359458974,
+ "tokens_seen": 969932800
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570591775325978,
+ "loss": 1.4095,
+ "theoretical_loss": 3.6882796940192644,
+ "tokens_seen": 969998336
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005705757271815446,
+ "loss": 1.3945,
+ "theoretical_loss": 3.6882560306260688,
+ "tokens_seen": 970063872
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005705596790371114,
+ "loss": 1.41,
+ "theoretical_loss": 3.6882323692790724,
+ "tokens_seen": 970129408
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005705436308926781,
+ "loss": 1.345,
+ "theoretical_loss": 3.688208709977959,
+ "tokens_seen": 970194944
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005705275827482448,
+ "loss": 1.3141,
+ "theoretical_loss": 3.6881850527224147,
+ "tokens_seen": 970260480
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005705115346038115,
+ "loss": 1.508,
+ "theoretical_loss": 3.6881613975121232,
+ "tokens_seen": 970326016
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005704954864593782,
+ "loss": 1.3958,
+ "theoretical_loss": 3.6881377443467707,
+ "tokens_seen": 970391552
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005704794383149449,
+ "loss": 1.3226,
+ "theoretical_loss": 3.6881140932260417,
+ "tokens_seen": 970457088
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005704633901705116,
+ "loss": 1.3344,
+ "theoretical_loss": 3.688090444149622,
+ "tokens_seen": 970522624
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005704473420260783,
+ "loss": 1.3384,
+ "theoretical_loss": 3.6880667971171963,
+ "tokens_seen": 970588160
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570431293881645,
+ "loss": 1.3955,
+ "theoretical_loss": 3.68804315212845,
+ "tokens_seen": 970653696
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005704152457372117,
+ "loss": 1.4069,
+ "theoretical_loss": 3.6880195091830688,
+ "tokens_seen": 970719232
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703991975927784,
+ "loss": 1.3192,
+ "theoretical_loss": 3.687995868280738,
+ "tokens_seen": 970784768
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703831494483451,
+ "loss": 1.4352,
+ "theoretical_loss": 3.687972229421143,
+ "tokens_seen": 970850304
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703671013039118,
+ "loss": 1.3864,
+ "theoretical_loss": 3.6879485926039695,
+ "tokens_seen": 970915840
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703510531594785,
+ "loss": 1.3829,
+ "theoretical_loss": 3.687924957828903,
+ "tokens_seen": 970981376
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703350050150452,
+ "loss": 1.3642,
+ "theoretical_loss": 3.6879013250956305,
+ "tokens_seen": 971046912
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703189568706119,
+ "loss": 1.3393,
+ "theoretical_loss": 3.687877694403836,
+ "tokens_seen": 971112448
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005703029087261786,
+ "loss": 1.2679,
+ "theoretical_loss": 3.687854065753206,
+ "tokens_seen": 971177984
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005702868605817453,
+ "loss": 1.3317,
+ "theoretical_loss": 3.6878304391434273,
+ "tokens_seen": 971243520
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570270812437312,
+ "loss": 1.356,
+ "theoretical_loss": 3.687806814574185,
+ "tokens_seen": 971309056
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005702547642928787,
+ "loss": 1.3752,
+ "theoretical_loss": 3.687783192045166,
+ "tokens_seen": 971374592
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005702387161484454,
+ "loss": 1.4202,
+ "theoretical_loss": 3.6877595715560556,
+ "tokens_seen": 971440128
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005702226680040121,
+ "loss": 1.4366,
+ "theoretical_loss": 3.6877359531065412,
+ "tokens_seen": 971505664
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005702066198595788,
+ "loss": 1.3614,
+ "theoretical_loss": 3.6877123366963076,
+ "tokens_seen": 971571200
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005701905717151455,
+ "loss": 1.4171,
+ "theoretical_loss": 3.6876887223250425,
+ "tokens_seen": 971636736
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005701745235707122,
+ "loss": 1.2972,
+ "theoretical_loss": 3.687665109992431,
+ "tokens_seen": 971702272
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005701584754262789,
+ "loss": 1.3607,
+ "theoretical_loss": 3.6876414996981612,
+ "tokens_seen": 971767808
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005701424272818456,
+ "loss": 1.3152,
+ "theoretical_loss": 3.687617891441919,
+ "tokens_seen": 971833344
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005701263791374123,
+ "loss": 1.4115,
+ "theoretical_loss": 3.6875942852233905,
+ "tokens_seen": 971898880
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000570110330992979,
+ "loss": 1.5083,
+ "theoretical_loss": 3.6875706810422635,
+ "tokens_seen": 971964416
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700942828485457,
+ "loss": 1.4294,
+ "theoretical_loss": 3.687547078898224,
+ "tokens_seen": 972029952
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700782347041124,
+ "loss": 1.3117,
+ "theoretical_loss": 3.6875234787909585,
+ "tokens_seen": 972095488
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700621865596791,
+ "loss": 1.463,
+ "theoretical_loss": 3.6874998807201553,
+ "tokens_seen": 972161024
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700461384152458,
+ "loss": 1.4098,
+ "theoretical_loss": 3.6874762846855003,
+ "tokens_seen": 972226560
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700300902708125,
+ "loss": 1.3988,
+ "theoretical_loss": 3.687452690686681,
+ "tokens_seen": 972292096
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005700140421263792,
+ "loss": 1.3265,
+ "theoretical_loss": 3.687429098723385,
+ "tokens_seen": 972357632
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005699979939819458,
+ "loss": 1.3901,
+ "theoretical_loss": 3.687405508795298,
+ "tokens_seen": 972423168
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005699819458375126,
+ "loss": 1.4919,
+ "theoretical_loss": 3.6873819209021086,
+ "tokens_seen": 972488704
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005699658976930792,
+ "loss": 1.3806,
+ "theoretical_loss": 3.687358335043504,
+ "tokens_seen": 972554240
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000569949849548646,
+ "loss": 1.3869,
+ "theoretical_loss": 3.687334751219171,
+ "tokens_seen": 972619776
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005699338014042126,
+ "loss": 1.3302,
+ "theoretical_loss": 3.6873111694287983,
+ "tokens_seen": 972685312
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005699177532597794,
+ "loss": 1.371,
+ "theoretical_loss": 3.6872875896720725,
+ "tokens_seen": 972750848
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000569901705115346,
+ "loss": 1.379,
+ "theoretical_loss": 3.687264011948681,
+ "tokens_seen": 972816384
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698856569709128,
+ "loss": 1.4398,
+ "theoretical_loss": 3.687240436258312,
+ "tokens_seen": 972881920
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698696088264794,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6872168626006534,
+ "tokens_seen": 972947456
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698535606820461,
+ "loss": 1.3912,
+ "theoretical_loss": 3.687193290975393,
+ "tokens_seen": 973012992
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698375125376128,
+ "loss": 1.4236,
+ "theoretical_loss": 3.6871697213822188,
+ "tokens_seen": 973078528
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698214643931795,
+ "loss": 1.3956,
+ "theoretical_loss": 3.6871461538208177,
+ "tokens_seen": 973144064
+ },
+ {
+ "epoch": 0.29,
+ "objective/train/docs_used": 329553,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5348623991012573,
+ "objective/train/theoretical_loss": 3.6871225882908787,
+ "objective/train/tokens_used": 993669600,
+ "theoretical_loss": 3.6871225882908787,
+ "tokens_seen": 973209600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005698054162487462,
+ "loss": 1.45,
+ "theoretical_loss": 3.6871225882908787,
+ "tokens_seen": 973209600
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005697893681043129,
+ "loss": 1.3545,
+ "theoretical_loss": 3.68709902479209,
+ "tokens_seen": 973275136
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005697733199598796,
+ "loss": 1.4072,
+ "theoretical_loss": 3.687075463324139,
+ "tokens_seen": 973340672
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.0005697572718154463,
+ "loss": 1.4714,
+ "theoretical_loss": 3.6870519038867156,
+ "tokens_seen": 973406208
+ },
+ {
+ "epoch": 0.29,
+ "learning_rate": 0.000569741223671013,
+ "loss": 1.4299,
+ "theoretical_loss": 3.687028346479506,
+ "tokens_seen": 973471744
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005697251755265797,
+ "loss": 1.4439,
+ "theoretical_loss": 3.6870047911022,
+ "tokens_seen": 973537280
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005697091273821464,
+ "loss": 1.4044,
+ "theoretical_loss": 3.6869812377544857,
+ "tokens_seen": 973602816
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696930792377131,
+ "loss": 1.3847,
+ "theoretical_loss": 3.6869576864360516,
+ "tokens_seen": 973668352
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696770310932798,
+ "loss": 1.5151,
+ "theoretical_loss": 3.6869341371465865,
+ "tokens_seen": 973733888
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696609829488465,
+ "loss": 1.3391,
+ "theoretical_loss": 3.686910589885778,
+ "tokens_seen": 973799424
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696449348044132,
+ "loss": 1.339,
+ "theoretical_loss": 3.686887044653316,
+ "tokens_seen": 973864960
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696288866599799,
+ "loss": 1.5226,
+ "theoretical_loss": 3.6868635014488893,
+ "tokens_seen": 973930496
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005696128385155467,
+ "loss": 1.3032,
+ "theoretical_loss": 3.686839960272186,
+ "tokens_seen": 973996032
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695967903711133,
+ "loss": 1.4417,
+ "theoretical_loss": 3.686816421122896,
+ "tokens_seen": 974061568
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056958074222668,
+ "loss": 1.4024,
+ "theoretical_loss": 3.686792884000708,
+ "tokens_seen": 974127104
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695646940822467,
+ "loss": 1.2908,
+ "theoretical_loss": 3.68676934890531,
+ "tokens_seen": 974192640
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695486459378135,
+ "loss": 1.3637,
+ "theoretical_loss": 3.6867458158363924,
+ "tokens_seen": 974258176
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695325977933801,
+ "loss": 1.294,
+ "theoretical_loss": 3.6867222847936443,
+ "tokens_seen": 974323712
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695165496489469,
+ "loss": 1.4052,
+ "theoretical_loss": 3.686698755776754,
+ "tokens_seen": 974389248
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005695005015045135,
+ "loss": 1.5044,
+ "theoretical_loss": 3.686675228785412,
+ "tokens_seen": 974454784
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694844533600803,
+ "loss": 1.2905,
+ "theoretical_loss": 3.686651703819307,
+ "tokens_seen": 974520320
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694684052156469,
+ "loss": 1.3481,
+ "theoretical_loss": 3.686628180878129,
+ "tokens_seen": 974585856
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694523570712137,
+ "loss": 1.408,
+ "theoretical_loss": 3.686604659961567,
+ "tokens_seen": 974651392
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694363089267803,
+ "loss": 1.4492,
+ "theoretical_loss": 3.6865811410693117,
+ "tokens_seen": 974716928
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694202607823471,
+ "loss": 1.3604,
+ "theoretical_loss": 3.6865576242010514,
+ "tokens_seen": 974782464
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005694042126379137,
+ "loss": 1.3904,
+ "theoretical_loss": 3.6865341093564767,
+ "tokens_seen": 974848000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005693881644934805,
+ "loss": 1.2957,
+ "theoretical_loss": 3.6865105965352765,
+ "tokens_seen": 974913536
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005693721163490471,
+ "loss": 1.3714,
+ "theoretical_loss": 3.6864870857371423,
+ "tokens_seen": 974979072
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005693560682046139,
+ "loss": 1.3326,
+ "theoretical_loss": 3.6864635769617626,
+ "tokens_seen": 975044608
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005693400200601806,
+ "loss": 1.5433,
+ "theoretical_loss": 3.6864400702088282,
+ "tokens_seen": 975110144
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005693239719157473,
+ "loss": 1.3617,
+ "theoretical_loss": 3.686416565478029,
+ "tokens_seen": 975175680
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000569307923771314,
+ "loss": 1.2906,
+ "theoretical_loss": 3.6863930627690555,
+ "tokens_seen": 975241216
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692918756268807,
+ "loss": 1.4415,
+ "theoretical_loss": 3.6863695620815973,
+ "tokens_seen": 975306752
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692758274824474,
+ "loss": 1.504,
+ "theoretical_loss": 3.686346063415345,
+ "tokens_seen": 975372288
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692597793380141,
+ "loss": 1.3098,
+ "theoretical_loss": 3.686322566769989,
+ "tokens_seen": 975437824
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692437311935808,
+ "loss": 1.2811,
+ "theoretical_loss": 3.6862990721452196,
+ "tokens_seen": 975503360
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692276830491475,
+ "loss": 1.3407,
+ "theoretical_loss": 3.6862755795407276,
+ "tokens_seen": 975568896
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005692116349047142,
+ "loss": 1.3383,
+ "theoretical_loss": 3.686252088956203,
+ "tokens_seen": 975634432
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005691955867602809,
+ "loss": 1.4899,
+ "theoretical_loss": 3.6862286003913374,
+ "tokens_seen": 975699968
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005691795386158476,
+ "loss": 1.3076,
+ "theoretical_loss": 3.686205113845821,
+ "tokens_seen": 975765504
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005691634904714143,
+ "loss": 1.4034,
+ "theoretical_loss": 3.686181629319344,
+ "tokens_seen": 975831040
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000569147442326981,
+ "loss": 1.3919,
+ "theoretical_loss": 3.6861581468115983,
+ "tokens_seen": 975896576
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005691313941825477,
+ "loss": 1.3594,
+ "theoretical_loss": 3.6861346663222743,
+ "tokens_seen": 975962112
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005691153460381144,
+ "loss": 1.2964,
+ "theoretical_loss": 3.686111187851063,
+ "tokens_seen": 976027648
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690992978936811,
+ "loss": 1.5076,
+ "theoretical_loss": 3.686087711397655,
+ "tokens_seen": 976093184
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690832497492478,
+ "loss": 1.3465,
+ "theoretical_loss": 3.6860642369617427,
+ "tokens_seen": 976158720
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690672016048145,
+ "loss": 1.3509,
+ "theoretical_loss": 3.686040764543016,
+ "tokens_seen": 976224256
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690511534603812,
+ "loss": 1.2808,
+ "theoretical_loss": 3.686017294141167,
+ "tokens_seen": 976289792
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690351053159479,
+ "loss": 1.3149,
+ "theoretical_loss": 3.6859938257558866,
+ "tokens_seen": 976355328
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690190571715146,
+ "loss": 1.3197,
+ "theoretical_loss": 3.6859703593868662,
+ "tokens_seen": 976420864
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 330391,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5027896165847778,
+ "objective/train/theoretical_loss": 3.6859468950337977,
+ "objective/train/tokens_used": 996946400,
+ "theoretical_loss": 3.6859468950337977,
+ "tokens_seen": 976486400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005690030090270813,
+ "loss": 1.4294,
+ "theoretical_loss": 3.6859468950337977,
+ "tokens_seen": 976486400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568986960882648,
+ "loss": 1.4013,
+ "theoretical_loss": 3.685923432696372,
+ "tokens_seen": 976551936
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005689709127382147,
+ "loss": 1.4,
+ "theoretical_loss": 3.6858999723742807,
+ "tokens_seen": 976617472
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005689548645937814,
+ "loss": 1.3802,
+ "theoretical_loss": 3.685876514067216,
+ "tokens_seen": 976683008
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005689388164493481,
+ "loss": 1.4382,
+ "theoretical_loss": 3.685853057774869,
+ "tokens_seen": 976748544
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005689227683049148,
+ "loss": 1.3721,
+ "theoretical_loss": 3.6858296034969324,
+ "tokens_seen": 976814080
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005689067201604815,
+ "loss": 1.4985,
+ "theoretical_loss": 3.6858061512330975,
+ "tokens_seen": 976879616
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005688906720160482,
+ "loss": 1.4508,
+ "theoretical_loss": 3.685782700983056,
+ "tokens_seen": 976945152
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005688746238716149,
+ "loss": 1.3566,
+ "theoretical_loss": 3.6857592527465006,
+ "tokens_seen": 977010688
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005688585757271816,
+ "loss": 1.4136,
+ "theoretical_loss": 3.6857358065231227,
+ "tokens_seen": 977076224
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005688425275827483,
+ "loss": 1.4871,
+ "theoretical_loss": 3.685712362312615,
+ "tokens_seen": 977141760
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568826479438315,
+ "loss": 1.2774,
+ "theoretical_loss": 3.6856889201146688,
+ "tokens_seen": 977207296
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005688104312938817,
+ "loss": 1.3291,
+ "theoretical_loss": 3.6856654799289776,
+ "tokens_seen": 977272832
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687943831494484,
+ "loss": 1.4611,
+ "theoretical_loss": 3.6856420417552336,
+ "tokens_seen": 977338368
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687783350050151,
+ "loss": 1.3763,
+ "theoretical_loss": 3.685618605593128,
+ "tokens_seen": 977403904
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687622868605818,
+ "loss": 1.4502,
+ "theoretical_loss": 3.6855951714423543,
+ "tokens_seen": 977469440
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687462387161485,
+ "loss": 1.35,
+ "theoretical_loss": 3.6855717393026044,
+ "tokens_seen": 977534976
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687301905717152,
+ "loss": 1.3829,
+ "theoretical_loss": 3.685548309173572,
+ "tokens_seen": 977600512
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005687141424272819,
+ "loss": 1.3952,
+ "theoretical_loss": 3.6855248810549486,
+ "tokens_seen": 977666048
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686980942828486,
+ "loss": 1.4016,
+ "theoretical_loss": 3.6855014549464276,
+ "tokens_seen": 977731584
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686820461384153,
+ "loss": 1.4011,
+ "theoretical_loss": 3.685478030847702,
+ "tokens_seen": 977797120
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568665997993982,
+ "loss": 1.3656,
+ "theoretical_loss": 3.685454608758464,
+ "tokens_seen": 977862656
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686499498495487,
+ "loss": 1.4097,
+ "theoretical_loss": 3.6854311886784066,
+ "tokens_seen": 977928192
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686339017051154,
+ "loss": 1.4644,
+ "theoretical_loss": 3.685407770607224,
+ "tokens_seen": 977993728
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686178535606821,
+ "loss": 1.353,
+ "theoretical_loss": 3.685384354544608,
+ "tokens_seen": 978059264
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005686018054162487,
+ "loss": 1.3915,
+ "theoretical_loss": 3.685360940490252,
+ "tokens_seen": 978124800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685857572718155,
+ "loss": 1.3349,
+ "theoretical_loss": 3.685337528443849,
+ "tokens_seen": 978190336
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685697091273821,
+ "loss": 1.3566,
+ "theoretical_loss": 3.6853141184050937,
+ "tokens_seen": 978255872
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685536609829489,
+ "loss": 1.4095,
+ "theoretical_loss": 3.6852907103736774,
+ "tokens_seen": 978321408
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685376128385155,
+ "loss": 1.403,
+ "theoretical_loss": 3.6852673043492956,
+ "tokens_seen": 978386944
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685215646940823,
+ "loss": 1.2823,
+ "theoretical_loss": 3.68524390033164,
+ "tokens_seen": 978452480
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005685055165496489,
+ "loss": 1.3668,
+ "theoretical_loss": 3.6852204983204055,
+ "tokens_seen": 978518016
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005684894684052157,
+ "loss": 1.3809,
+ "theoretical_loss": 3.6851970983152844,
+ "tokens_seen": 978583552
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005684734202607823,
+ "loss": 1.3704,
+ "theoretical_loss": 3.685173700315972,
+ "tokens_seen": 978649088
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568457372116349,
+ "loss": 1.3677,
+ "theoretical_loss": 3.6851503043221605,
+ "tokens_seen": 978714624
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005684413239719157,
+ "loss": 1.4072,
+ "theoretical_loss": 3.685126910333545,
+ "tokens_seen": 978780160
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005684252758274824,
+ "loss": 1.3737,
+ "theoretical_loss": 3.6851035183498184,
+ "tokens_seen": 978845696
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005684092276830491,
+ "loss": 1.3461,
+ "theoretical_loss": 3.6850801283706756,
+ "tokens_seen": 978911232
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005683931795386158,
+ "loss": 1.3565,
+ "theoretical_loss": 3.68505674039581,
+ "tokens_seen": 978976768
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005683771313941825,
+ "loss": 1.3611,
+ "theoretical_loss": 3.6850333544249154,
+ "tokens_seen": 979042304
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005683610832497492,
+ "loss": 1.4217,
+ "theoretical_loss": 3.6850099704576866,
+ "tokens_seen": 979107840
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568345035105316,
+ "loss": 1.332,
+ "theoretical_loss": 3.684986588493818,
+ "tokens_seen": 979173376
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005683289869608826,
+ "loss": 1.2867,
+ "theoretical_loss": 3.684963208533003,
+ "tokens_seen": 979238912
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005683129388164493,
+ "loss": 1.4204,
+ "theoretical_loss": 3.684939830574937,
+ "tokens_seen": 979304448
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568296890672016,
+ "loss": 1.3366,
+ "theoretical_loss": 3.6849164546193136,
+ "tokens_seen": 979369984
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682808425275827,
+ "loss": 1.3858,
+ "theoretical_loss": 3.6848930806658275,
+ "tokens_seen": 979435520
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682647943831494,
+ "loss": 1.4454,
+ "theoretical_loss": 3.684869708714174,
+ "tokens_seen": 979501056
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682487462387162,
+ "loss": 1.3815,
+ "theoretical_loss": 3.684846338764047,
+ "tokens_seen": 979566592
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682326980942828,
+ "loss": 1.3057,
+ "theoretical_loss": 3.6848229708151408,
+ "tokens_seen": 979632128
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682166499498496,
+ "loss": 1.3947,
+ "theoretical_loss": 3.684799604867151,
+ "tokens_seen": 979697664
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 331668,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3969978094100952,
+ "objective/train/theoretical_loss": 3.6847762409197724,
+ "objective/train/tokens_used": 1000223200,
+ "theoretical_loss": 3.6847762409197724,
+ "tokens_seen": 979763200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005682006018054162,
+ "loss": 1.2919,
+ "theoretical_loss": 3.6847762409197724,
+ "tokens_seen": 979763200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000568184553660983,
+ "loss": 1.3791,
+ "theoretical_loss": 3.6847528789726995,
+ "tokens_seen": 979828736
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005681685055165496,
+ "loss": 1.3665,
+ "theoretical_loss": 3.6847295190256273,
+ "tokens_seen": 979894272
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005681524573721164,
+ "loss": 1.3441,
+ "theoretical_loss": 3.684706161078251,
+ "tokens_seen": 979959808
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005681364092276831,
+ "loss": 1.3108,
+ "theoretical_loss": 3.6846828051302656,
+ "tokens_seen": 980025344
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005681203610832498,
+ "loss": 1.3302,
+ "theoretical_loss": 3.6846594511813664,
+ "tokens_seen": 980090880
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005681043129388165,
+ "loss": 1.4192,
+ "theoretical_loss": 3.6846360992312484,
+ "tokens_seen": 980156416
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005680882647943832,
+ "loss": 1.2893,
+ "theoretical_loss": 3.6846127492796077,
+ "tokens_seen": 980221952
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005680722166499499,
+ "loss": 1.3537,
+ "theoretical_loss": 3.6845894013261384,
+ "tokens_seen": 980287488
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005680561685055166,
+ "loss": 1.3803,
+ "theoretical_loss": 3.684566055370537,
+ "tokens_seen": 980353024
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005680401203610833,
+ "loss": 1.3628,
+ "theoretical_loss": 3.6845427114124982,
+ "tokens_seen": 980418560
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056802407221665,
+ "loss": 1.4395,
+ "theoretical_loss": 3.684519369451718,
+ "tokens_seen": 980484096
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005680080240722167,
+ "loss": 1.3303,
+ "theoretical_loss": 3.6844960294878923,
+ "tokens_seen": 980549632
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679919759277834,
+ "loss": 1.2955,
+ "theoretical_loss": 3.684472691520716,
+ "tokens_seen": 980615168
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679759277833501,
+ "loss": 1.4059,
+ "theoretical_loss": 3.684449355549886,
+ "tokens_seen": 980680704
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679598796389168,
+ "loss": 1.3102,
+ "theoretical_loss": 3.6844260215750975,
+ "tokens_seen": 980746240
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679438314944835,
+ "loss": 1.4076,
+ "theoretical_loss": 3.684402689596046,
+ "tokens_seen": 980811776
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679277833500502,
+ "loss": 1.2741,
+ "theoretical_loss": 3.6843793596124277,
+ "tokens_seen": 980877312
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005679117352056169,
+ "loss": 1.3524,
+ "theoretical_loss": 3.684356031623939,
+ "tokens_seen": 980942848
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005678956870611836,
+ "loss": 1.3458,
+ "theoretical_loss": 3.6843327056302764,
+ "tokens_seen": 981008384
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005678796389167503,
+ "loss": 1.5072,
+ "theoretical_loss": 3.6843093816311354,
+ "tokens_seen": 981073920
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567863590772317,
+ "loss": 1.3678,
+ "theoretical_loss": 3.6842860596262117,
+ "tokens_seen": 981139456
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005678475426278837,
+ "loss": 1.4274,
+ "theoretical_loss": 3.6842627396152023,
+ "tokens_seen": 981204992
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005678314944834504,
+ "loss": 1.3302,
+ "theoretical_loss": 3.684239421597804,
+ "tokens_seen": 981270528
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005678154463390171,
+ "loss": 1.3208,
+ "theoretical_loss": 3.6842161055737126,
+ "tokens_seen": 981336064
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677993981945838,
+ "loss": 1.2496,
+ "theoretical_loss": 3.6841927915426247,
+ "tokens_seen": 981401600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677833500501505,
+ "loss": 1.4826,
+ "theoretical_loss": 3.684169479504237,
+ "tokens_seen": 981467136
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677673019057172,
+ "loss": 1.3356,
+ "theoretical_loss": 3.684146169458246,
+ "tokens_seen": 981532672
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677512537612839,
+ "loss": 1.3267,
+ "theoretical_loss": 3.6841228614043486,
+ "tokens_seen": 981598208
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677352056168506,
+ "loss": 1.3417,
+ "theoretical_loss": 3.6840995553422413,
+ "tokens_seen": 981663744
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005677191574724173,
+ "loss": 1.2657,
+ "theoretical_loss": 3.6840762512716214,
+ "tokens_seen": 981729280
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567703109327984,
+ "loss": 1.4828,
+ "theoretical_loss": 3.6840529491921847,
+ "tokens_seen": 981794816
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676870611835507,
+ "loss": 1.4698,
+ "theoretical_loss": 3.68402964910363,
+ "tokens_seen": 981860352
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676710130391174,
+ "loss": 1.4686,
+ "theoretical_loss": 3.6840063510056527,
+ "tokens_seen": 981925888
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676549648946841,
+ "loss": 1.416,
+ "theoretical_loss": 3.6839830548979506,
+ "tokens_seen": 981991424
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676389167502508,
+ "loss": 1.2738,
+ "theoretical_loss": 3.683959760780221,
+ "tokens_seen": 982056960
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676228686058175,
+ "loss": 1.3382,
+ "theoretical_loss": 3.68393646865216,
+ "tokens_seen": 982122496
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005676068204613842,
+ "loss": 1.3057,
+ "theoretical_loss": 3.6839131785134667,
+ "tokens_seen": 982188032
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005675907723169509,
+ "loss": 1.4602,
+ "theoretical_loss": 3.683889890363837,
+ "tokens_seen": 982253568
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005675747241725176,
+ "loss": 1.3691,
+ "theoretical_loss": 3.6838666042029695,
+ "tokens_seen": 982319104
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005675586760280843,
+ "loss": 1.3261,
+ "theoretical_loss": 3.6838433200305607,
+ "tokens_seen": 982384640
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567542627883651,
+ "loss": 1.3717,
+ "theoretical_loss": 3.683820037846308,
+ "tokens_seen": 982450176
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005675265797392177,
+ "loss": 1.297,
+ "theoretical_loss": 3.6837967576499104,
+ "tokens_seen": 982515712
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005675105315947844,
+ "loss": 1.3108,
+ "theoretical_loss": 3.6837734794410646,
+ "tokens_seen": 982581248
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674944834503511,
+ "loss": 1.3577,
+ "theoretical_loss": 3.6837502032194687,
+ "tokens_seen": 982646784
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674784353059178,
+ "loss": 1.3823,
+ "theoretical_loss": 3.68372692898482,
+ "tokens_seen": 982712320
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674623871614845,
+ "loss": 1.3033,
+ "theoretical_loss": 3.6837036567368173,
+ "tokens_seen": 982777856
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674463390170512,
+ "loss": 1.3632,
+ "theoretical_loss": 3.6836803864751575,
+ "tokens_seen": 982843392
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674302908726179,
+ "loss": 1.2767,
+ "theoretical_loss": 3.6836571181995392,
+ "tokens_seen": 982908928
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005674142427281846,
+ "loss": 1.3832,
+ "theoretical_loss": 3.6836338519096605,
+ "tokens_seen": 982974464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.7061842754091193,
+ "debugging/entropy-1-grams": 4.557400890096247,
+ "debugging/length": 580.2,
+ "debugging/num_segments": 5,
+ "debugging/score": 0.06456836857494044,
+ "debugging/score_std": 0.019067822519855632,
+ "epoch": 0.3,
+ "objective/train/docs_used": 332958,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4429404735565186,
+ "objective/train/theoretical_loss": 3.68361058760522,
+ "objective/train/tokens_used": 1003500000,
+ "theoretical_loss": 3.68361058760522,
+ "tokens_seen": 983040000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673981945837513,
+ "loss": 1.3946,
+ "theoretical_loss": 3.68361058760522,
+ "tokens_seen": 983040000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567382146439318,
+ "loss": 1.2636,
+ "theoretical_loss": 3.683587325285915,
+ "tokens_seen": 983105536
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673660982948847,
+ "loss": 1.402,
+ "theoretical_loss": 3.6835640649514443,
+ "tokens_seen": 983171072
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673500501504514,
+ "loss": 1.3532,
+ "theoretical_loss": 3.6835408066015063,
+ "tokens_seen": 983236608
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673340020060181,
+ "loss": 1.3186,
+ "theoretical_loss": 3.6835175502357993,
+ "tokens_seen": 983302144
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673179538615848,
+ "loss": 1.3792,
+ "theoretical_loss": 3.6834942958540218,
+ "tokens_seen": 983367680
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005673019057171515,
+ "loss": 1.4187,
+ "theoretical_loss": 3.683471043455872,
+ "tokens_seen": 983433216
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005672858575727182,
+ "loss": 1.3978,
+ "theoretical_loss": 3.6834477930410494,
+ "tokens_seen": 983498752
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005672698094282849,
+ "loss": 1.277,
+ "theoretical_loss": 3.683424544609252,
+ "tokens_seen": 983564288
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005672537612838516,
+ "loss": 1.2866,
+ "theoretical_loss": 3.6834012981601787,
+ "tokens_seen": 983629824
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005672377131394183,
+ "loss": 1.4135,
+ "theoretical_loss": 3.683378053693528,
+ "tokens_seen": 983695360
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567221664994985,
+ "loss": 1.3519,
+ "theoretical_loss": 3.683354811209,
+ "tokens_seen": 983760896
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005672056168505516,
+ "loss": 1.4097,
+ "theoretical_loss": 3.6833315707062924,
+ "tokens_seen": 983826432
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005671895687061184,
+ "loss": 1.4026,
+ "theoretical_loss": 3.6833083321851046,
+ "tokens_seen": 983891968
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567173520561685,
+ "loss": 1.4019,
+ "theoretical_loss": 3.6832850956451355,
+ "tokens_seen": 983957504
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005671574724172518,
+ "loss": 1.344,
+ "theoretical_loss": 3.6832618610860846,
+ "tokens_seen": 984023040
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005671414242728184,
+ "loss": 1.3308,
+ "theoretical_loss": 3.6832386285076506,
+ "tokens_seen": 984088576
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005671253761283852,
+ "loss": 1.3346,
+ "theoretical_loss": 3.6832153979095335,
+ "tokens_seen": 984154112
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005671093279839518,
+ "loss": 1.3408,
+ "theoretical_loss": 3.6831921692914324,
+ "tokens_seen": 984219648
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005670932798395186,
+ "loss": 1.4718,
+ "theoretical_loss": 3.6831689426530465,
+ "tokens_seen": 984285184
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005670772316950852,
+ "loss": 1.365,
+ "theoretical_loss": 3.683145717994075,
+ "tokens_seen": 984350720
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005670611835506519,
+ "loss": 1.2906,
+ "theoretical_loss": 3.683122495314218,
+ "tokens_seen": 984416256
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005670451354062186,
+ "loss": 1.3021,
+ "theoretical_loss": 3.683099274613175,
+ "tokens_seen": 984481792
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005670290872617853,
+ "loss": 1.4236,
+ "theoretical_loss": 3.6830760558906457,
+ "tokens_seen": 984547328
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000567013039117352,
+ "loss": 1.377,
+ "theoretical_loss": 3.6830528391463293,
+ "tokens_seen": 984612864
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669969909729187,
+ "loss": 1.3042,
+ "theoretical_loss": 3.6830296243799263,
+ "tokens_seen": 984678400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669809428284854,
+ "loss": 1.3744,
+ "theoretical_loss": 3.6830064115911365,
+ "tokens_seen": 984743936
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669648946840522,
+ "loss": 1.3447,
+ "theoretical_loss": 3.6829832007796597,
+ "tokens_seen": 984809472
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669488465396188,
+ "loss": 1.4428,
+ "theoretical_loss": 3.6829599919451956,
+ "tokens_seen": 984875008
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669327983951856,
+ "loss": 1.331,
+ "theoretical_loss": 3.682936785087444,
+ "tokens_seen": 984940544
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005669167502507522,
+ "loss": 1.393,
+ "theoretical_loss": 3.6829135802061064,
+ "tokens_seen": 985006080
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566900702106319,
+ "loss": 1.3394,
+ "theoretical_loss": 3.6828903773008816,
+ "tokens_seen": 985071616
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005668846539618856,
+ "loss": 1.4251,
+ "theoretical_loss": 3.6828671763714707,
+ "tokens_seen": 985137152
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005668686058174524,
+ "loss": 1.4133,
+ "theoretical_loss": 3.6828439774175736,
+ "tokens_seen": 985202688
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566852557673019,
+ "loss": 1.433,
+ "theoretical_loss": 3.682820780438891,
+ "tokens_seen": 985268224
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005668365095285858,
+ "loss": 1.4331,
+ "theoretical_loss": 3.6827975854351225,
+ "tokens_seen": 985333760
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005668204613841525,
+ "loss": 1.3828,
+ "theoretical_loss": 3.68277439240597,
+ "tokens_seen": 985399296
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005668044132397192,
+ "loss": 1.3815,
+ "theoretical_loss": 3.682751201351133,
+ "tokens_seen": 985464832
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005667883650952859,
+ "loss": 1.3695,
+ "theoretical_loss": 3.6827280122703128,
+ "tokens_seen": 985530368
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005667723169508526,
+ "loss": 1.3144,
+ "theoretical_loss": 3.68270482516321,
+ "tokens_seen": 985595904
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005667562688064193,
+ "loss": 1.4188,
+ "theoretical_loss": 3.682681640029525,
+ "tokens_seen": 985661440
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566740220661986,
+ "loss": 1.3489,
+ "theoretical_loss": 3.6826584568689587,
+ "tokens_seen": 985726976
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005667241725175527,
+ "loss": 1.4191,
+ "theoretical_loss": 3.6826352756812124,
+ "tokens_seen": 985792512
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005667081243731194,
+ "loss": 1.3455,
+ "theoretical_loss": 3.6826120964659874,
+ "tokens_seen": 985858048
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666920762286861,
+ "loss": 1.3907,
+ "theoretical_loss": 3.682588919222984,
+ "tokens_seen": 985923584
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666760280842528,
+ "loss": 1.4821,
+ "theoretical_loss": 3.682565743951903,
+ "tokens_seen": 985989120
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666599799398195,
+ "loss": 1.3521,
+ "theoretical_loss": 3.682542570652447,
+ "tokens_seen": 986054656
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666439317953862,
+ "loss": 1.489,
+ "theoretical_loss": 3.6825193993243155,
+ "tokens_seen": 986120192
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666278836509529,
+ "loss": 1.3119,
+ "theoretical_loss": 3.682496229967211,
+ "tokens_seen": 986185728
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005666118355065196,
+ "loss": 1.3445,
+ "theoretical_loss": 3.6824730625808346,
+ "tokens_seen": 986251264
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 333817,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.331803798675537,
+ "objective/train/theoretical_loss": 3.6824498971648882,
+ "objective/train/tokens_used": 1006776800,
+ "theoretical_loss": 3.6824498971648882,
+ "tokens_seen": 986316800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005665957873620863,
+ "loss": 1.3384,
+ "theoretical_loss": 3.6824498971648882,
+ "tokens_seen": 986316800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566579739217653,
+ "loss": 1.4972,
+ "theoretical_loss": 3.682426733719072,
+ "tokens_seen": 986382336
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005665636910732197,
+ "loss": 1.4241,
+ "theoretical_loss": 3.6824035722430892,
+ "tokens_seen": 986447872
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005665476429287864,
+ "loss": 1.3828,
+ "theoretical_loss": 3.6823804127366397,
+ "tokens_seen": 986513408
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005665315947843531,
+ "loss": 1.369,
+ "theoretical_loss": 3.6823572551994266,
+ "tokens_seen": 986578944
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005665155466399198,
+ "loss": 1.3784,
+ "theoretical_loss": 3.6823340996311513,
+ "tokens_seen": 986644480
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664994984954865,
+ "loss": 1.3649,
+ "theoretical_loss": 3.682310946031515,
+ "tokens_seen": 986710016
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664834503510532,
+ "loss": 1.4104,
+ "theoretical_loss": 3.6822877944002204,
+ "tokens_seen": 986775552
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664674022066199,
+ "loss": 1.3924,
+ "theoretical_loss": 3.682264644736969,
+ "tokens_seen": 986841088
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664513540621866,
+ "loss": 1.3395,
+ "theoretical_loss": 3.6822414970414634,
+ "tokens_seen": 986906624
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664353059177533,
+ "loss": 1.355,
+ "theoretical_loss": 3.682218351313405,
+ "tokens_seen": 986972160
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056641925777332,
+ "loss": 1.3924,
+ "theoretical_loss": 3.6821952075524957,
+ "tokens_seen": 987037696
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005664032096288867,
+ "loss": 1.4015,
+ "theoretical_loss": 3.6821720657584387,
+ "tokens_seen": 987103232
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663871614844534,
+ "loss": 1.404,
+ "theoretical_loss": 3.6821489259309357,
+ "tokens_seen": 987168768
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663711133400201,
+ "loss": 1.2869,
+ "theoretical_loss": 3.682125788069689,
+ "tokens_seen": 987234304
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663550651955868,
+ "loss": 1.4381,
+ "theoretical_loss": 3.682102652174401,
+ "tokens_seen": 987299840
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663390170511535,
+ "loss": 1.4067,
+ "theoretical_loss": 3.682079518244775,
+ "tokens_seen": 987365376
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663229689067202,
+ "loss": 1.3896,
+ "theoretical_loss": 3.6820563862805127,
+ "tokens_seen": 987430912
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005663069207622869,
+ "loss": 1.3642,
+ "theoretical_loss": 3.6820332562813167,
+ "tokens_seen": 987496448
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005662908726178536,
+ "loss": 1.4163,
+ "theoretical_loss": 3.6820101282468896,
+ "tokens_seen": 987561984
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005662748244734203,
+ "loss": 1.3182,
+ "theoretical_loss": 3.6819870021769345,
+ "tokens_seen": 987627520
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566258776328987,
+ "loss": 1.3719,
+ "theoretical_loss": 3.6819638780711546,
+ "tokens_seen": 987693056
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005662427281845537,
+ "loss": 1.3108,
+ "theoretical_loss": 3.6819407559292516,
+ "tokens_seen": 987758592
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005662266800401204,
+ "loss": 1.3835,
+ "theoretical_loss": 3.6819176357509296,
+ "tokens_seen": 987824128
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005662106318956871,
+ "loss": 1.2571,
+ "theoretical_loss": 3.681894517535891,
+ "tokens_seen": 987889664
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661945837512538,
+ "loss": 1.2335,
+ "theoretical_loss": 3.6818714012838383,
+ "tokens_seen": 987955200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661785356068205,
+ "loss": 1.3365,
+ "theoretical_loss": 3.6818482869944757,
+ "tokens_seen": 988020736
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661624874623872,
+ "loss": 1.3487,
+ "theoretical_loss": 3.681825174667506,
+ "tokens_seen": 988086272
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661464393179539,
+ "loss": 1.3836,
+ "theoretical_loss": 3.6818020643026323,
+ "tokens_seen": 988151808
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661303911735206,
+ "loss": 1.3399,
+ "theoretical_loss": 3.681778955899558,
+ "tokens_seen": 988217344
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005661143430290873,
+ "loss": 1.3863,
+ "theoretical_loss": 3.6817558494579865,
+ "tokens_seen": 988282880
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000566098294884654,
+ "loss": 1.4341,
+ "theoretical_loss": 3.681732744977621,
+ "tokens_seen": 988348416
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660822467402207,
+ "loss": 1.3173,
+ "theoretical_loss": 3.681709642458166,
+ "tokens_seen": 988413952
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660661985957874,
+ "loss": 1.2942,
+ "theoretical_loss": 3.6816865418993237,
+ "tokens_seen": 988479488
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660501504513541,
+ "loss": 1.3678,
+ "theoretical_loss": 3.681663443300798,
+ "tokens_seen": 988545024
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660341023069208,
+ "loss": 1.2993,
+ "theoretical_loss": 3.6816403466622942,
+ "tokens_seen": 988610560
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660180541624875,
+ "loss": 1.322,
+ "theoretical_loss": 3.681617251983514,
+ "tokens_seen": 988676096
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005660020060180542,
+ "loss": 1.3447,
+ "theoretical_loss": 3.681594159264162,
+ "tokens_seen": 988741632
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005659859578736209,
+ "loss": 1.3304,
+ "theoretical_loss": 3.681571068503942,
+ "tokens_seen": 988807168
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005659699097291876,
+ "loss": 1.3252,
+ "theoretical_loss": 3.6815479797025588,
+ "tokens_seen": 988872704
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005659538615847543,
+ "loss": 1.3822,
+ "theoretical_loss": 3.6815248928597155,
+ "tokens_seen": 988938240
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000565937813440321,
+ "loss": 1.3233,
+ "theoretical_loss": 3.681501807975116,
+ "tokens_seen": 989003776
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005659217652958877,
+ "loss": 1.3444,
+ "theoretical_loss": 3.681478725048465,
+ "tokens_seen": 989069312
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005659057171514544,
+ "loss": 1.3964,
+ "theoretical_loss": 3.6814556440794672,
+ "tokens_seen": 989134848
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658896690070211,
+ "loss": 1.3694,
+ "theoretical_loss": 3.681432565067826,
+ "tokens_seen": 989200384
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658736208625878,
+ "loss": 1.407,
+ "theoretical_loss": 3.681409488013246,
+ "tokens_seen": 989265920
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658575727181545,
+ "loss": 1.225,
+ "theoretical_loss": 3.6813864129154314,
+ "tokens_seen": 989331456
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658415245737212,
+ "loss": 1.4289,
+ "theoretical_loss": 3.6813633397740873,
+ "tokens_seen": 989396992
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658254764292879,
+ "loss": 1.3986,
+ "theoretical_loss": 3.681340268588918,
+ "tokens_seen": 989462528
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005658094282848546,
+ "loss": 1.367,
+ "theoretical_loss": 3.6813171993596274,
+ "tokens_seen": 989528064
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 334953,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3690037727355957,
+ "objective/train/theoretical_loss": 3.681294132085921,
+ "objective/train/tokens_used": 1010053600,
+ "theoretical_loss": 3.681294132085921,
+ "tokens_seen": 989593600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657933801404213,
+ "loss": 1.3338,
+ "theoretical_loss": 3.681294132085921,
+ "tokens_seen": 989593600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657773319959879,
+ "loss": 1.4075,
+ "theoretical_loss": 3.6812710667675033,
+ "tokens_seen": 989659136
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657612838515547,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6812480034040793,
+ "tokens_seen": 989724672
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657452357071213,
+ "loss": 1.3045,
+ "theoretical_loss": 3.681224941995353,
+ "tokens_seen": 989790208
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657291875626881,
+ "loss": 1.3002,
+ "theoretical_loss": 3.681201882541031,
+ "tokens_seen": 989855744
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005657131394182547,
+ "loss": 1.3496,
+ "theoretical_loss": 3.681178825040816,
+ "tokens_seen": 989921280
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656970912738215,
+ "loss": 1.3146,
+ "theoretical_loss": 3.6811557694944153,
+ "tokens_seen": 989986816
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656810431293881,
+ "loss": 1.3166,
+ "theoretical_loss": 3.6811327159015326,
+ "tokens_seen": 990052352
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656649949849549,
+ "loss": 1.3282,
+ "theoretical_loss": 3.6811096642618732,
+ "tokens_seen": 990117888
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656489468405215,
+ "loss": 1.4342,
+ "theoretical_loss": 3.6810866145751433,
+ "tokens_seen": 990183424
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656328986960883,
+ "loss": 1.2805,
+ "theoretical_loss": 3.6810635668410474,
+ "tokens_seen": 990248960
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656168505516549,
+ "loss": 1.4442,
+ "theoretical_loss": 3.6810405210592907,
+ "tokens_seen": 990314496
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005656008024072217,
+ "loss": 1.3321,
+ "theoretical_loss": 3.681017477229579,
+ "tokens_seen": 990380032
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655847542627883,
+ "loss": 1.306,
+ "theoretical_loss": 3.680994435351618,
+ "tokens_seen": 990445568
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655687061183551,
+ "loss": 1.286,
+ "theoretical_loss": 3.6809713954251126,
+ "tokens_seen": 990511104
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655526579739217,
+ "loss": 1.3902,
+ "theoretical_loss": 3.6809483574497692,
+ "tokens_seen": 990576640
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655366098294885,
+ "loss": 1.3559,
+ "theoretical_loss": 3.6809253214252933,
+ "tokens_seen": 990642176
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655205616850551,
+ "loss": 1.3252,
+ "theoretical_loss": 3.6809022873513904,
+ "tokens_seen": 990707712
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005655045135406219,
+ "loss": 1.351,
+ "theoretical_loss": 3.680879255227766,
+ "tokens_seen": 990773248
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005654884653961886,
+ "loss": 1.3565,
+ "theoretical_loss": 3.680856225054127,
+ "tokens_seen": 990838784
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005654724172517553,
+ "loss": 1.388,
+ "theoretical_loss": 3.680833196830178,
+ "tokens_seen": 990904320
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000565456369107322,
+ "loss": 1.3968,
+ "theoretical_loss": 3.6808101705556266,
+ "tokens_seen": 990969856
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005654403209628887,
+ "loss": 1.3266,
+ "theoretical_loss": 3.6807871462301778,
+ "tokens_seen": 991035392
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005654242728184554,
+ "loss": 1.3703,
+ "theoretical_loss": 3.6807641238535376,
+ "tokens_seen": 991100928
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005654082246740221,
+ "loss": 1.2335,
+ "theoretical_loss": 3.6807411034254125,
+ "tokens_seen": 991166464
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653921765295888,
+ "loss": 1.4172,
+ "theoretical_loss": 3.6807180849455094,
+ "tokens_seen": 991232000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653761283851555,
+ "loss": 1.334,
+ "theoretical_loss": 3.680695068413534,
+ "tokens_seen": 991297536
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653600802407222,
+ "loss": 1.3617,
+ "theoretical_loss": 3.680672053829192,
+ "tokens_seen": 991363072
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653440320962889,
+ "loss": 1.3674,
+ "theoretical_loss": 3.6806490411921917,
+ "tokens_seen": 991428608
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653279839518556,
+ "loss": 1.3773,
+ "theoretical_loss": 3.6806260305022382,
+ "tokens_seen": 991494144
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005653119358074223,
+ "loss": 1.3921,
+ "theoretical_loss": 3.680603021759038,
+ "tokens_seen": 991559680
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000565295887662989,
+ "loss": 1.4225,
+ "theoretical_loss": 3.680580014962299,
+ "tokens_seen": 991625216
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005652798395185557,
+ "loss": 1.3689,
+ "theoretical_loss": 3.680557010111726,
+ "tokens_seen": 991690752
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005652637913741224,
+ "loss": 1.3267,
+ "theoretical_loss": 3.6805340072070276,
+ "tokens_seen": 991756288
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005652477432296891,
+ "loss": 1.2849,
+ "theoretical_loss": 3.68051100624791,
+ "tokens_seen": 991821824
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005652316950852558,
+ "loss": 1.2479,
+ "theoretical_loss": 3.6804880072340795,
+ "tokens_seen": 991887360
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005652156469408225,
+ "loss": 1.283,
+ "theoretical_loss": 3.680465010165244,
+ "tokens_seen": 991952896
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651995987963892,
+ "loss": 1.3904,
+ "theoretical_loss": 3.68044201504111,
+ "tokens_seen": 992018432
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651835506519559,
+ "loss": 1.3015,
+ "theoretical_loss": 3.680419021861385,
+ "tokens_seen": 992083968
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651675025075226,
+ "loss": 1.3007,
+ "theoretical_loss": 3.6803960306257752,
+ "tokens_seen": 992149504
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651514543630893,
+ "loss": 1.4119,
+ "theoretical_loss": 3.680373041333989,
+ "tokens_seen": 992215040
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000565135406218656,
+ "loss": 1.3537,
+ "theoretical_loss": 3.680350053985733,
+ "tokens_seen": 992280576
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651193580742227,
+ "loss": 1.3171,
+ "theoretical_loss": 3.6803270685807146,
+ "tokens_seen": 992346112
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005651033099297894,
+ "loss": 1.292,
+ "theoretical_loss": 3.680304085118641,
+ "tokens_seen": 992411648
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650872617853561,
+ "loss": 1.3236,
+ "theoretical_loss": 3.6802811035992207,
+ "tokens_seen": 992477184
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650712136409228,
+ "loss": 1.3134,
+ "theoretical_loss": 3.68025812402216,
+ "tokens_seen": 992542720
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650551654964895,
+ "loss": 1.3056,
+ "theoretical_loss": 3.680235146387167,
+ "tokens_seen": 992608256
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650391173520562,
+ "loss": 1.3597,
+ "theoretical_loss": 3.6802121706939497,
+ "tokens_seen": 992673792
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650230692076229,
+ "loss": 1.4204,
+ "theoretical_loss": 3.680189196942216,
+ "tokens_seen": 992739328
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005650070210631896,
+ "loss": 1.3497,
+ "theoretical_loss": 3.6801662251316722,
+ "tokens_seen": 992804864
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 335764,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3931187391281128,
+ "objective/train/theoretical_loss": 3.6801432552620277,
+ "objective/train/tokens_used": 1013330400,
+ "theoretical_loss": 3.6801432552620277,
+ "tokens_seen": 992870400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005649909729187563,
+ "loss": 1.4383,
+ "theoretical_loss": 3.6801432552620277,
+ "tokens_seen": 992870400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564974924774323,
+ "loss": 1.422,
+ "theoretical_loss": 3.68012028733299,
+ "tokens_seen": 992935936
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005649588766298897,
+ "loss": 1.4616,
+ "theoretical_loss": 3.6800973213442667,
+ "tokens_seen": 993001472
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005649428284854564,
+ "loss": 1.2986,
+ "theoretical_loss": 3.680074357295566,
+ "tokens_seen": 993067008
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005649267803410231,
+ "loss": 1.4569,
+ "theoretical_loss": 3.680051395186596,
+ "tokens_seen": 993132544
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005649107321965898,
+ "loss": 1.2824,
+ "theoretical_loss": 3.680028435017065,
+ "tokens_seen": 993198080
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005648946840521565,
+ "loss": 1.2645,
+ "theoretical_loss": 3.6800054767866817,
+ "tokens_seen": 993263616
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005648786359077232,
+ "loss": 1.2743,
+ "theoretical_loss": 3.6799825204951535,
+ "tokens_seen": 993329152
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005648625877632899,
+ "loss": 1.3661,
+ "theoretical_loss": 3.6799595661421893,
+ "tokens_seen": 993394688
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005648465396188566,
+ "loss": 1.3528,
+ "theoretical_loss": 3.679936613727498,
+ "tokens_seen": 993460224
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005648304914744233,
+ "loss": 1.3753,
+ "theoretical_loss": 3.6799136632507867,
+ "tokens_seen": 993525760
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056481444332999,
+ "loss": 1.3209,
+ "theoretical_loss": 3.6798907147117648,
+ "tokens_seen": 993591296
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647983951855567,
+ "loss": 1.2764,
+ "theoretical_loss": 3.6798677681101406,
+ "tokens_seen": 993656832
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647823470411234,
+ "loss": 1.2637,
+ "theoretical_loss": 3.6798448234456234,
+ "tokens_seen": 993722368
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647662988966901,
+ "loss": 1.3528,
+ "theoretical_loss": 3.6798218807179213,
+ "tokens_seen": 993787904
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647502507522568,
+ "loss": 1.3155,
+ "theoretical_loss": 3.6797989399267435,
+ "tokens_seen": 993853440
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647342026078235,
+ "loss": 1.2588,
+ "theoretical_loss": 3.679776001071799,
+ "tokens_seen": 993918976
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647181544633902,
+ "loss": 1.2973,
+ "theoretical_loss": 3.679753064152796,
+ "tokens_seen": 993984512
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005647021063189569,
+ "loss": 1.3545,
+ "theoretical_loss": 3.679730129169444,
+ "tokens_seen": 994050048
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005646860581745236,
+ "loss": 1.2825,
+ "theoretical_loss": 3.679707196121452,
+ "tokens_seen": 994115584
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005646700100300903,
+ "loss": 1.372,
+ "theoretical_loss": 3.6796842650085293,
+ "tokens_seen": 994181120
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564653961885657,
+ "loss": 1.3828,
+ "theoretical_loss": 3.6796613358303842,
+ "tokens_seen": 994246656
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005646379137412237,
+ "loss": 1.3242,
+ "theoretical_loss": 3.6796384085867277,
+ "tokens_seen": 994312192
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005646218655967904,
+ "loss": 1.3462,
+ "theoretical_loss": 3.679615483277267,
+ "tokens_seen": 994377728
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005646058174523571,
+ "loss": 1.3769,
+ "theoretical_loss": 3.679592559901713,
+ "tokens_seen": 994443264
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645897693079238,
+ "loss": 1.3495,
+ "theoretical_loss": 3.6795696384597747,
+ "tokens_seen": 994508800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645737211634905,
+ "loss": 1.3529,
+ "theoretical_loss": 3.679546718951161,
+ "tokens_seen": 994574336
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645576730190573,
+ "loss": 1.3009,
+ "theoretical_loss": 3.6795238013755824,
+ "tokens_seen": 994639872
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645416248746239,
+ "loss": 1.3876,
+ "theoretical_loss": 3.6795008857327476,
+ "tokens_seen": 994705408
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645255767301906,
+ "loss": 1.3747,
+ "theoretical_loss": 3.6794779720223674,
+ "tokens_seen": 994770944
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005645095285857573,
+ "loss": 1.3314,
+ "theoretical_loss": 3.6794550602441505,
+ "tokens_seen": 994836480
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564493480441324,
+ "loss": 1.3172,
+ "theoretical_loss": 3.6794321503978074,
+ "tokens_seen": 994902016
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005644774322968907,
+ "loss": 1.3119,
+ "theoretical_loss": 3.679409242483047,
+ "tokens_seen": 994967552
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005644613841524574,
+ "loss": 1.3156,
+ "theoretical_loss": 3.6793863364995802,
+ "tokens_seen": 995033088
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005644453360080241,
+ "loss": 1.382,
+ "theoretical_loss": 3.6793634324471167,
+ "tokens_seen": 995098624
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005644292878635908,
+ "loss": 1.4278,
+ "theoretical_loss": 3.6793405303253675,
+ "tokens_seen": 995164160
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005644132397191575,
+ "loss": 1.3369,
+ "theoretical_loss": 3.6793176301340407,
+ "tokens_seen": 995229696
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643971915747242,
+ "loss": 1.3507,
+ "theoretical_loss": 3.6792947318728473,
+ "tokens_seen": 995295232
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643811434302908,
+ "loss": 1.3575,
+ "theoretical_loss": 3.679271835541499,
+ "tokens_seen": 995360768
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643650952858576,
+ "loss": 1.3269,
+ "theoretical_loss": 3.6792489411397042,
+ "tokens_seen": 995426304
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643490471414242,
+ "loss": 1.3141,
+ "theoretical_loss": 3.6792260486671737,
+ "tokens_seen": 995491840
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564332998996991,
+ "loss": 1.291,
+ "theoretical_loss": 3.6792031581236184,
+ "tokens_seen": 995557376
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643169508525576,
+ "loss": 1.3886,
+ "theoretical_loss": 3.6791802695087483,
+ "tokens_seen": 995622912
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005643009027081244,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6791573828222752,
+ "tokens_seen": 995688448
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564284854563691,
+ "loss": 1.2696,
+ "theoretical_loss": 3.679134498063908,
+ "tokens_seen": 995753984
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005642688064192578,
+ "loss": 1.276,
+ "theoretical_loss": 3.6791116152333583,
+ "tokens_seen": 995819520
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005642527582748244,
+ "loss": 1.3491,
+ "theoretical_loss": 3.6790887343303367,
+ "tokens_seen": 995885056
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005642367101303912,
+ "loss": 1.3067,
+ "theoretical_loss": 3.679065855354554,
+ "tokens_seen": 995950592
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005642206619859578,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6790429783057212,
+ "tokens_seen": 996016128
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005642046138415246,
+ "loss": 1.2713,
+ "theoretical_loss": 3.679020103183549,
+ "tokens_seen": 996081664
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 336954,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1566932201385498,
+ "objective/train/theoretical_loss": 3.6789972299877487,
+ "objective/train/tokens_used": 1016607200,
+ "theoretical_loss": 3.6789972299877487,
+ "tokens_seen": 996147200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005641885656970912,
+ "loss": 1.2733,
+ "theoretical_loss": 3.6789972299877487,
+ "tokens_seen": 996147200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564172517552658,
+ "loss": 1.4281,
+ "theoretical_loss": 3.678974358718031,
+ "tokens_seen": 996212736
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005641564694082246,
+ "loss": 1.4024,
+ "theoretical_loss": 3.678951489374107,
+ "tokens_seen": 996278272
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005641404212637914,
+ "loss": 1.4178,
+ "theoretical_loss": 3.6789286219556887,
+ "tokens_seen": 996343808
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564124373119358,
+ "loss": 1.3535,
+ "theoretical_loss": 3.678905756462486,
+ "tokens_seen": 996409344
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005641083249749248,
+ "loss": 1.4022,
+ "theoretical_loss": 3.6788828928942108,
+ "tokens_seen": 996474880
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005640922768304915,
+ "loss": 1.3485,
+ "theoretical_loss": 3.6788600312505753,
+ "tokens_seen": 996540416
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005640762286860582,
+ "loss": 1.3762,
+ "theoretical_loss": 3.6788371715312898,
+ "tokens_seen": 996605952
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005640601805416249,
+ "loss": 1.2325,
+ "theoretical_loss": 3.6788143137360665,
+ "tokens_seen": 996671488
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005640441323971916,
+ "loss": 1.4216,
+ "theoretical_loss": 3.678791457864616,
+ "tokens_seen": 996737024
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005640280842527583,
+ "loss": 1.36,
+ "theoretical_loss": 3.678768603916651,
+ "tokens_seen": 996802560
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000564012036108325,
+ "loss": 1.2724,
+ "theoretical_loss": 3.678745751891883,
+ "tokens_seen": 996868096
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639959879638917,
+ "loss": 1.3032,
+ "theoretical_loss": 3.678722901790023,
+ "tokens_seen": 996933632
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639799398194584,
+ "loss": 1.3816,
+ "theoretical_loss": 3.6787000536107834,
+ "tokens_seen": 996999168
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639638916750251,
+ "loss": 1.3505,
+ "theoretical_loss": 3.6786772073538763,
+ "tokens_seen": 997064704
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639478435305918,
+ "loss": 1.3334,
+ "theoretical_loss": 3.678654363019013,
+ "tokens_seen": 997130240
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639317953861585,
+ "loss": 1.2945,
+ "theoretical_loss": 3.6786315206059057,
+ "tokens_seen": 997195776
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005639157472417252,
+ "loss": 1.2427,
+ "theoretical_loss": 3.678608680114267,
+ "tokens_seen": 997261312
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638996990972919,
+ "loss": 1.2997,
+ "theoretical_loss": 3.678585841543808,
+ "tokens_seen": 997326848
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638836509528586,
+ "loss": 1.3328,
+ "theoretical_loss": 3.678563004894242,
+ "tokens_seen": 997392384
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638676028084253,
+ "loss": 1.498,
+ "theoretical_loss": 3.67854017016528,
+ "tokens_seen": 997457920
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000563851554663992,
+ "loss": 1.4053,
+ "theoretical_loss": 3.6785173373566353,
+ "tokens_seen": 997523456
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638355065195587,
+ "loss": 1.2954,
+ "theoretical_loss": 3.6784945064680197,
+ "tokens_seen": 997588992
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638194583751254,
+ "loss": 1.3759,
+ "theoretical_loss": 3.6784716774991457,
+ "tokens_seen": 997654528
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005638034102306921,
+ "loss": 1.4184,
+ "theoretical_loss": 3.6784488504497266,
+ "tokens_seen": 997720064
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637873620862588,
+ "loss": 1.3446,
+ "theoretical_loss": 3.6784260253194736,
+ "tokens_seen": 997785600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637713139418255,
+ "loss": 1.3536,
+ "theoretical_loss": 3.6784032021081003,
+ "tokens_seen": 997851136
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637552657973922,
+ "loss": 1.2899,
+ "theoretical_loss": 3.6783803808153195,
+ "tokens_seen": 997916672
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637392176529589,
+ "loss": 1.3224,
+ "theoretical_loss": 3.678357561440843,
+ "tokens_seen": 997982208
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637231695085256,
+ "loss": 1.4113,
+ "theoretical_loss": 3.678334743984384,
+ "tokens_seen": 998047744
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005637071213640923,
+ "loss": 1.3151,
+ "theoretical_loss": 3.6783119284456554,
+ "tokens_seen": 998113280
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000563691073219659,
+ "loss": 1.3028,
+ "theoretical_loss": 3.6782891148243704,
+ "tokens_seen": 998178816
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005636750250752257,
+ "loss": 1.3659,
+ "theoretical_loss": 3.6782663031202416,
+ "tokens_seen": 998244352
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005636589769307924,
+ "loss": 1.3531,
+ "theoretical_loss": 3.6782434933329826,
+ "tokens_seen": 998309888
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005636429287863591,
+ "loss": 1.4025,
+ "theoretical_loss": 3.678220685462305,
+ "tokens_seen": 998375424
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005636268806419258,
+ "loss": 1.2639,
+ "theoretical_loss": 3.678197879507924,
+ "tokens_seen": 998440960
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005636108324974925,
+ "loss": 1.4739,
+ "theoretical_loss": 3.678175075469552,
+ "tokens_seen": 998506496
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005635947843530592,
+ "loss": 1.3225,
+ "theoretical_loss": 3.678152273346902,
+ "tokens_seen": 998572032
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005635787362086259,
+ "loss": 1.2327,
+ "theoretical_loss": 3.6781294731396876,
+ "tokens_seen": 998637568
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005635626880641926,
+ "loss": 1.4357,
+ "theoretical_loss": 3.678106674847622,
+ "tokens_seen": 998703104
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005635466399197593,
+ "loss": 1.3422,
+ "theoretical_loss": 3.678083878470418,
+ "tokens_seen": 998768640
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000563530591775326,
+ "loss": 1.3486,
+ "theoretical_loss": 3.6780610840077914,
+ "tokens_seen": 998834176
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005635145436308927,
+ "loss": 1.2121,
+ "theoretical_loss": 3.6780382914594534,
+ "tokens_seen": 998899712
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634984954864594,
+ "loss": 1.2469,
+ "theoretical_loss": 3.678015500825119,
+ "tokens_seen": 998965248
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634824473420261,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6779927121045013,
+ "tokens_seen": 999030784
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634663991975928,
+ "loss": 1.3937,
+ "theoretical_loss": 3.6779699252973144,
+ "tokens_seen": 999096320
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634503510531595,
+ "loss": 1.3069,
+ "theoretical_loss": 3.6779471404032718,
+ "tokens_seen": 999161856
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634343029087262,
+ "loss": 1.3422,
+ "theoretical_loss": 3.6779243574220883,
+ "tokens_seen": 999227392
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634182547642929,
+ "loss": 1.3918,
+ "theoretical_loss": 3.677901576353477,
+ "tokens_seen": 999292928
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005634022066198596,
+ "loss": 1.3897,
+ "theoretical_loss": 3.6778787971971516,
+ "tokens_seen": 999358464
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 337911,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3854376077651978,
+ "objective/train/theoretical_loss": 3.677856019952827,
+ "objective/train/tokens_used": 1019884000,
+ "theoretical_loss": 3.677856019952827,
+ "tokens_seen": 999424000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005633861584754263,
+ "loss": 1.3928,
+ "theoretical_loss": 3.677856019952827,
+ "tokens_seen": 999424000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000563370110330993,
+ "loss": 1.4098,
+ "theoretical_loss": 3.677833244620218,
+ "tokens_seen": 999489536
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005633540621865598,
+ "loss": 1.3254,
+ "theoretical_loss": 3.677810471199037,
+ "tokens_seen": 999555072
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005633380140421264,
+ "loss": 1.3597,
+ "theoretical_loss": 3.6777876996889995,
+ "tokens_seen": 999620608
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005633219658976932,
+ "loss": 1.3813,
+ "theoretical_loss": 3.677764930089819,
+ "tokens_seen": 999686144
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005633059177532598,
+ "loss": 1.4033,
+ "theoretical_loss": 3.677742162401211,
+ "tokens_seen": 999751680
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005632898696088266,
+ "loss": 1.3265,
+ "theoretical_loss": 3.6777193966228894,
+ "tokens_seen": 999817216
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005632738214643932,
+ "loss": 1.221,
+ "theoretical_loss": 3.677696632754569,
+ "tokens_seen": 999882752
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056325777331996,
+ "loss": 1.3621,
+ "theoretical_loss": 3.6776738707959633,
+ "tokens_seen": 999948288
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005632417251755266,
+ "loss": 1.4019,
+ "theoretical_loss": 3.677651110746788,
+ "tokens_seen": 1000013824
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005632256770310934,
+ "loss": 1.3801,
+ "theoretical_loss": 3.6776283526067575,
+ "tokens_seen": 1000079360
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.00056320962888666,
+ "loss": 1.23,
+ "theoretical_loss": 3.677605596375587,
+ "tokens_seen": 1000144896
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631935807422268,
+ "loss": 1.3374,
+ "theoretical_loss": 3.6775828420529906,
+ "tokens_seen": 1000210432
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631775325977934,
+ "loss": 1.2955,
+ "theoretical_loss": 3.6775600896386837,
+ "tokens_seen": 1000275968
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631614844533602,
+ "loss": 1.3149,
+ "theoretical_loss": 3.677537339132381,
+ "tokens_seen": 1000341504
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631454363089268,
+ "loss": 1.3515,
+ "theoretical_loss": 3.6775145905337974,
+ "tokens_seen": 1000407040
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631293881644935,
+ "loss": 1.3775,
+ "theoretical_loss": 3.6774918438426485,
+ "tokens_seen": 1000472576
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005631133400200602,
+ "loss": 1.3265,
+ "theoretical_loss": 3.6774690990586487,
+ "tokens_seen": 1000538112
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630972918756269,
+ "loss": 1.2553,
+ "theoretical_loss": 3.677446356181514,
+ "tokens_seen": 1000603648
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630812437311936,
+ "loss": 1.2805,
+ "theoretical_loss": 3.6774236152109587,
+ "tokens_seen": 1000669184
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630651955867603,
+ "loss": 1.4398,
+ "theoretical_loss": 3.6774008761466987,
+ "tokens_seen": 1000734720
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000563049147442327,
+ "loss": 1.3433,
+ "theoretical_loss": 3.67737813898845,
+ "tokens_seen": 1000800256
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630330992978937,
+ "loss": 1.3674,
+ "theoretical_loss": 3.677355403735927,
+ "tokens_seen": 1000865792
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630170511534604,
+ "loss": 1.3955,
+ "theoretical_loss": 3.6773326703888456,
+ "tokens_seen": 1000931328
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005630010030090271,
+ "loss": 1.4065,
+ "theoretical_loss": 3.6773099389469213,
+ "tokens_seen": 1000996864
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629849548645937,
+ "loss": 1.4244,
+ "theoretical_loss": 3.6772872094098696,
+ "tokens_seen": 1001062400
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629689067201605,
+ "loss": 1.3244,
+ "theoretical_loss": 3.6772644817774065,
+ "tokens_seen": 1001127936
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629528585757271,
+ "loss": 1.2779,
+ "theoretical_loss": 3.6772417560492476,
+ "tokens_seen": 1001193472
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629368104312939,
+ "loss": 1.388,
+ "theoretical_loss": 3.677219032225109,
+ "tokens_seen": 1001259008
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629207622868605,
+ "loss": 1.3168,
+ "theoretical_loss": 3.677196310304706,
+ "tokens_seen": 1001324544
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005629047141424273,
+ "loss": 1.3445,
+ "theoretical_loss": 3.6771735902877545,
+ "tokens_seen": 1001390080
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628886659979939,
+ "loss": 1.4859,
+ "theoretical_loss": 3.6771508721739714,
+ "tokens_seen": 1001455616
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628726178535607,
+ "loss": 1.2793,
+ "theoretical_loss": 3.6771281559630715,
+ "tokens_seen": 1001521152
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628565697091273,
+ "loss": 1.2923,
+ "theoretical_loss": 3.677105441654772,
+ "tokens_seen": 1001586688
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628405215646941,
+ "loss": 1.2933,
+ "theoretical_loss": 3.6770827292487884,
+ "tokens_seen": 1001652224
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628244734202607,
+ "loss": 1.2476,
+ "theoretical_loss": 3.677060018744837,
+ "tokens_seen": 1001717760
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005628084252758275,
+ "loss": 1.3712,
+ "theoretical_loss": 3.6770373101426346,
+ "tokens_seen": 1001783296
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005627923771313941,
+ "loss": 1.3297,
+ "theoretical_loss": 3.677014603441897,
+ "tokens_seen": 1001848832
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005627763289869609,
+ "loss": 1.4308,
+ "theoretical_loss": 3.6769918986423407,
+ "tokens_seen": 1001914368
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005627602808425276,
+ "loss": 1.4035,
+ "theoretical_loss": 3.6769691957436823,
+ "tokens_seen": 1001979904
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005627442326980943,
+ "loss": 1.4258,
+ "theoretical_loss": 3.6769464947456383,
+ "tokens_seen": 1002045440
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000562728184553661,
+ "loss": 1.3288,
+ "theoretical_loss": 3.6769237956479257,
+ "tokens_seen": 1002110976
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005627121364092277,
+ "loss": 1.2861,
+ "theoretical_loss": 3.6769010984502604,
+ "tokens_seen": 1002176512
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626960882647944,
+ "loss": 1.3088,
+ "theoretical_loss": 3.67687840315236,
+ "tokens_seen": 1002242048
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626800401203611,
+ "loss": 1.1965,
+ "theoretical_loss": 3.67685570975394,
+ "tokens_seen": 1002307584
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626639919759278,
+ "loss": 1.3618,
+ "theoretical_loss": 3.676833018254719,
+ "tokens_seen": 1002373120
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626479438314945,
+ "loss": 1.3594,
+ "theoretical_loss": 3.676810328654412,
+ "tokens_seen": 1002438656
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626318956870612,
+ "loss": 1.3654,
+ "theoretical_loss": 3.676787640952737,
+ "tokens_seen": 1002504192
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005626158475426279,
+ "loss": 1.3873,
+ "theoretical_loss": 3.6767649551494115,
+ "tokens_seen": 1002569728
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625997993981946,
+ "loss": 1.2778,
+ "theoretical_loss": 3.6767422712441515,
+ "tokens_seen": 1002635264
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 339058,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4166014194488525,
+ "objective/train/theoretical_loss": 3.676719589236675,
+ "objective/train/tokens_used": 1023160800,
+ "theoretical_loss": 3.676719589236675,
+ "tokens_seen": 1002700800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625837512537613,
+ "loss": 1.3904,
+ "theoretical_loss": 3.676719589236675,
+ "tokens_seen": 1002700800
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000562567703109328,
+ "loss": 1.2092,
+ "theoretical_loss": 3.676696909126698,
+ "tokens_seen": 1002766336
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625516549648947,
+ "loss": 1.32,
+ "theoretical_loss": 3.6766742309139397,
+ "tokens_seen": 1002831872
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625356068204614,
+ "loss": 1.4426,
+ "theoretical_loss": 3.6766515545981155,
+ "tokens_seen": 1002897408
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625195586760281,
+ "loss": 1.3065,
+ "theoretical_loss": 3.676628880178944,
+ "tokens_seen": 1002962944
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005625035105315948,
+ "loss": 1.3856,
+ "theoretical_loss": 3.676606207656142,
+ "tokens_seen": 1003028480
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005624874623871615,
+ "loss": 1.4278,
+ "theoretical_loss": 3.6765835370294275,
+ "tokens_seen": 1003094016
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005624714142427282,
+ "loss": 1.4277,
+ "theoretical_loss": 3.676560868298518,
+ "tokens_seen": 1003159552
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005624553660982949,
+ "loss": 1.2708,
+ "theoretical_loss": 3.6765382014631305,
+ "tokens_seen": 1003225088
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005624393179538616,
+ "loss": 1.279,
+ "theoretical_loss": 3.6765155365229836,
+ "tokens_seen": 1003290624
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005624232698094283,
+ "loss": 1.3475,
+ "theoretical_loss": 3.6764928734777946,
+ "tokens_seen": 1003356160
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000562407221664995,
+ "loss": 1.4427,
+ "theoretical_loss": 3.676470212327281,
+ "tokens_seen": 1003421696
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623911735205617,
+ "loss": 1.4315,
+ "theoretical_loss": 3.6764475530711613,
+ "tokens_seen": 1003487232
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623751253761284,
+ "loss": 1.3445,
+ "theoretical_loss": 3.676424895709153,
+ "tokens_seen": 1003552768
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623590772316951,
+ "loss": 1.286,
+ "theoretical_loss": 3.676402240240974,
+ "tokens_seen": 1003618304
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623430290872618,
+ "loss": 1.3735,
+ "theoretical_loss": 3.676379586666343,
+ "tokens_seen": 1003683840
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623269809428285,
+ "loss": 1.4462,
+ "theoretical_loss": 3.6763569349849776,
+ "tokens_seen": 1003749376
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005623109327983952,
+ "loss": 1.3635,
+ "theoretical_loss": 3.676334285196596,
+ "tokens_seen": 1003814912
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005622948846539619,
+ "loss": 1.4446,
+ "theoretical_loss": 3.676311637300916,
+ "tokens_seen": 1003880448
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005622788365095286,
+ "loss": 1.3577,
+ "theoretical_loss": 3.676288991297657,
+ "tokens_seen": 1003945984
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005622627883650953,
+ "loss": 1.3886,
+ "theoretical_loss": 3.6762663471865364,
+ "tokens_seen": 1004011520
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.000562246740220662,
+ "loss": 1.2953,
+ "theoretical_loss": 3.676243704967273,
+ "tokens_seen": 1004077056
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005622306920762287,
+ "loss": 1.3162,
+ "theoretical_loss": 3.676221064639585,
+ "tokens_seen": 1004142592
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005622146439317954,
+ "loss": 1.3495,
+ "theoretical_loss": 3.6761984262031917,
+ "tokens_seen": 1004208128
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621985957873621,
+ "loss": 1.4397,
+ "theoretical_loss": 3.676175789657811,
+ "tokens_seen": 1004273664
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621825476429288,
+ "loss": 1.2939,
+ "theoretical_loss": 3.676153155003161,
+ "tokens_seen": 1004339200
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621664994984955,
+ "loss": 1.3325,
+ "theoretical_loss": 3.676130522238962,
+ "tokens_seen": 1004404736
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621504513540623,
+ "loss": 1.2272,
+ "theoretical_loss": 3.6761078913649317,
+ "tokens_seen": 1004470272
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621344032096289,
+ "loss": 1.4954,
+ "theoretical_loss": 3.6760852623807887,
+ "tokens_seen": 1004535808
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621183550651957,
+ "loss": 1.3522,
+ "theoretical_loss": 3.676062635286253,
+ "tokens_seen": 1004601344
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005621023069207623,
+ "loss": 1.4191,
+ "theoretical_loss": 3.676040010081042,
+ "tokens_seen": 1004666880
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620862587763291,
+ "loss": 1.2587,
+ "theoretical_loss": 3.6760173867648764,
+ "tokens_seen": 1004732416
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620702106318957,
+ "loss": 1.2593,
+ "theoretical_loss": 3.6759947653374745,
+ "tokens_seen": 1004797952
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620541624874625,
+ "loss": 1.3543,
+ "theoretical_loss": 3.6759721457985552,
+ "tokens_seen": 1004863488
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620381143430291,
+ "loss": 1.3352,
+ "theoretical_loss": 3.6759495281478376,
+ "tokens_seen": 1004929024
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620220661985959,
+ "loss": 1.3456,
+ "theoretical_loss": 3.6759269123850413,
+ "tokens_seen": 1004994560
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005620060180541625,
+ "loss": 1.365,
+ "theoretical_loss": 3.6759042985098858,
+ "tokens_seen": 1005060096
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619899699097293,
+ "loss": 1.4014,
+ "theoretical_loss": 3.6758816865220902,
+ "tokens_seen": 1005125632
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619739217652959,
+ "loss": 1.4408,
+ "theoretical_loss": 3.675859076421374,
+ "tokens_seen": 1005191168
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619578736208627,
+ "loss": 1.4228,
+ "theoretical_loss": 3.6758364682074567,
+ "tokens_seen": 1005256704
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619418254764293,
+ "loss": 1.3138,
+ "theoretical_loss": 3.6758138618800578,
+ "tokens_seen": 1005322240
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619257773319961,
+ "loss": 1.2984,
+ "theoretical_loss": 3.675791257438897,
+ "tokens_seen": 1005387776
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005619097291875627,
+ "loss": 1.3153,
+ "theoretical_loss": 3.675768654883694,
+ "tokens_seen": 1005453312
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618936810431295,
+ "loss": 1.3233,
+ "theoretical_loss": 3.6757460542141684,
+ "tokens_seen": 1005518848
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618776328986961,
+ "loss": 1.3306,
+ "theoretical_loss": 3.67572345543004,
+ "tokens_seen": 1005584384
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618615847542629,
+ "loss": 1.3512,
+ "theoretical_loss": 3.675700858531029,
+ "tokens_seen": 1005649920
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618455366098295,
+ "loss": 1.4213,
+ "theoretical_loss": 3.675678263516855,
+ "tokens_seen": 1005715456
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618294884653963,
+ "loss": 1.3265,
+ "theoretical_loss": 3.675655670387238,
+ "tokens_seen": 1005780992
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005618134403209629,
+ "loss": 1.2062,
+ "theoretical_loss": 3.675633079141898,
+ "tokens_seen": 1005846528
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617973921765297,
+ "loss": 1.327,
+ "theoretical_loss": 3.6756104897805546,
+ "tokens_seen": 1005912064
+ },
+ {
+ "epoch": 0.3,
+ "objective/train/docs_used": 339945,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3827526569366455,
+ "objective/train/theoretical_loss": 3.6755879023029294,
+ "objective/train/tokens_used": 1026437600,
+ "theoretical_loss": 3.6755879023029294,
+ "tokens_seen": 1005977600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617813440320963,
+ "loss": 1.2794,
+ "theoretical_loss": 3.6755879023029294,
+ "tokens_seen": 1005977600
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617652958876631,
+ "loss": 1.3544,
+ "theoretical_loss": 3.675565316708741,
+ "tokens_seen": 1006043136
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617492477432297,
+ "loss": 1.3751,
+ "theoretical_loss": 3.6755427329977106,
+ "tokens_seen": 1006108672
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617331995987964,
+ "loss": 1.4148,
+ "theoretical_loss": 3.6755201511695588,
+ "tokens_seen": 1006174208
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617171514543631,
+ "loss": 1.2918,
+ "theoretical_loss": 3.675497571224005,
+ "tokens_seen": 1006239744
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005617011033099298,
+ "loss": 1.3348,
+ "theoretical_loss": 3.675474993160771,
+ "tokens_seen": 1006305280
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005616850551654965,
+ "loss": 1.3271,
+ "theoretical_loss": 3.6754524169795753,
+ "tokens_seen": 1006370816
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 0.0005616690070210632,
+ "loss": 1.404,
+ "theoretical_loss": 3.67542984268014,
+ "tokens_seen": 1006436352
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00056165295887663,
+ "loss": 1.4002,
+ "theoretical_loss": 3.675407270262186,
+ "tokens_seen": 1006501888
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005616369107321966,
+ "loss": 1.4938,
+ "theoretical_loss": 3.675384699725433,
+ "tokens_seen": 1006567424
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005616208625877633,
+ "loss": 1.3689,
+ "theoretical_loss": 3.675362131069602,
+ "tokens_seen": 1006632960
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00056160481444333,
+ "loss": 1.2742,
+ "theoretical_loss": 3.675339564294414,
+ "tokens_seen": 1006698496
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005615887662988966,
+ "loss": 1.4429,
+ "theoretical_loss": 3.6753169993995902,
+ "tokens_seen": 1006764032
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005615727181544634,
+ "loss": 1.2831,
+ "theoretical_loss": 3.675294436384851,
+ "tokens_seen": 1006829568
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00056155667001003,
+ "loss": 1.3226,
+ "theoretical_loss": 3.6752718752499174,
+ "tokens_seen": 1006895104
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005615406218655968,
+ "loss": 1.2452,
+ "theoretical_loss": 3.6752493159945105,
+ "tokens_seen": 1006960640
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005615245737211634,
+ "loss": 1.4371,
+ "theoretical_loss": 3.675226758618352,
+ "tokens_seen": 1007026176
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005615085255767302,
+ "loss": 1.399,
+ "theoretical_loss": 3.6752042031211625,
+ "tokens_seen": 1007091712
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005614924774322968,
+ "loss": 1.379,
+ "theoretical_loss": 3.6751816495026626,
+ "tokens_seen": 1007157248
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005614764292878636,
+ "loss": 1.4198,
+ "theoretical_loss": 3.6751590977625748,
+ "tokens_seen": 1007222784
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005614603811434302,
+ "loss": 1.338,
+ "theoretical_loss": 3.67513654790062,
+ "tokens_seen": 1007288320
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000561444332998997,
+ "loss": 1.2114,
+ "theoretical_loss": 3.67511399991652,
+ "tokens_seen": 1007353856
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005614282848545636,
+ "loss": 1.4396,
+ "theoretical_loss": 3.675091453809995,
+ "tokens_seen": 1007419392
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005614122367101304,
+ "loss": 1.3514,
+ "theoretical_loss": 3.6750689095807676,
+ "tokens_seen": 1007484928
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000561396188565697,
+ "loss": 1.26,
+ "theoretical_loss": 3.6750463672285596,
+ "tokens_seen": 1007550464
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005613801404212638,
+ "loss": 1.4395,
+ "theoretical_loss": 3.6750238267530917,
+ "tokens_seen": 1007616000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005613640922768305,
+ "loss": 1.3453,
+ "theoretical_loss": 3.6750012881540863,
+ "tokens_seen": 1007681536
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005613480441323972,
+ "loss": 1.3833,
+ "theoretical_loss": 3.674978751431265,
+ "tokens_seen": 1007747072
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005613319959879639,
+ "loss": 1.5011,
+ "theoretical_loss": 3.674956216584349,
+ "tokens_seen": 1007812608
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005613159478435306,
+ "loss": 1.3377,
+ "theoretical_loss": 3.6749336836130615,
+ "tokens_seen": 1007878144
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612998996990973,
+ "loss": 1.3894,
+ "theoretical_loss": 3.674911152517123,
+ "tokens_seen": 1007943680
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000561283851554664,
+ "loss": 1.4529,
+ "theoretical_loss": 3.674888623296257,
+ "tokens_seen": 1008009216
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612678034102307,
+ "loss": 1.274,
+ "theoretical_loss": 3.6748660959501844,
+ "tokens_seen": 1008074752
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612517552657974,
+ "loss": 1.2635,
+ "theoretical_loss": 3.674843570478627,
+ "tokens_seen": 1008140288
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612357071213641,
+ "loss": 1.4344,
+ "theoretical_loss": 3.6748210468813087,
+ "tokens_seen": 1008205824
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612196589769308,
+ "loss": 1.5382,
+ "theoretical_loss": 3.6747985251579505,
+ "tokens_seen": 1008271360
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005612036108324975,
+ "loss": 1.2626,
+ "theoretical_loss": 3.6747760053082743,
+ "tokens_seen": 1008336896
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005611875626880642,
+ "loss": 1.4066,
+ "theoretical_loss": 3.674753487332004,
+ "tokens_seen": 1008402432
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005611715145436309,
+ "loss": 1.3055,
+ "theoretical_loss": 3.6747309712288603,
+ "tokens_seen": 1008467968
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005611554663991976,
+ "loss": 1.3769,
+ "theoretical_loss": 3.674708456998567,
+ "tokens_seen": 1008533504
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005611394182547643,
+ "loss": 1.4245,
+ "theoretical_loss": 3.6746859446408457,
+ "tokens_seen": 1008599040
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000561123370110331,
+ "loss": 1.4843,
+ "theoretical_loss": 3.674663434155419,
+ "tokens_seen": 1008664576
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005611073219658977,
+ "loss": 1.306,
+ "theoretical_loss": 3.6746409255420103,
+ "tokens_seen": 1008730112
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610912738214644,
+ "loss": 1.2671,
+ "theoretical_loss": 3.674618418800342,
+ "tokens_seen": 1008795648
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610752256770311,
+ "loss": 1.3045,
+ "theoretical_loss": 3.674595913930137,
+ "tokens_seen": 1008861184
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610591775325978,
+ "loss": 1.3975,
+ "theoretical_loss": 3.6745734109311177,
+ "tokens_seen": 1008926720
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610431293881645,
+ "loss": 1.4402,
+ "theoretical_loss": 3.674550909803007,
+ "tokens_seen": 1008992256
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610270812437312,
+ "loss": 1.451,
+ "theoretical_loss": 3.6745284105455287,
+ "tokens_seen": 1009057792
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005610110330992979,
+ "loss": 1.5097,
+ "theoretical_loss": 3.674505913158405,
+ "tokens_seen": 1009123328
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005609949849548646,
+ "loss": 1.2531,
+ "theoretical_loss": 3.6744834176413588,
+ "tokens_seen": 1009188864
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 341239,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2152767181396484,
+ "objective/train/theoretical_loss": 3.6744609239941135,
+ "objective/train/tokens_used": 1029714400,
+ "theoretical_loss": 3.6744609239941135,
+ "tokens_seen": 1009254400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005609789368104313,
+ "loss": 1.2052,
+ "theoretical_loss": 3.6744609239941135,
+ "tokens_seen": 1009254400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560962888665998,
+ "loss": 1.3659,
+ "theoretical_loss": 3.6744384322163928,
+ "tokens_seen": 1009319936
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005609468405215648,
+ "loss": 1.3328,
+ "theoretical_loss": 3.674415942307919,
+ "tokens_seen": 1009385472
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005609307923771314,
+ "loss": 1.2977,
+ "theoretical_loss": 3.674393454268416,
+ "tokens_seen": 1009451008
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005609147442326982,
+ "loss": 1.3546,
+ "theoretical_loss": 3.674370968097607,
+ "tokens_seen": 1009516544
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005608986960882648,
+ "loss": 1.4168,
+ "theoretical_loss": 3.6743484837952165,
+ "tokens_seen": 1009582080
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005608826479438316,
+ "loss": 1.3549,
+ "theoretical_loss": 3.674326001360966,
+ "tokens_seen": 1009647616
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005608665997993982,
+ "loss": 1.4207,
+ "theoretical_loss": 3.67430352079458,
+ "tokens_seen": 1009713152
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560850551654965,
+ "loss": 1.2511,
+ "theoretical_loss": 3.6742810420957825,
+ "tokens_seen": 1009778688
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005608345035105316,
+ "loss": 1.4117,
+ "theoretical_loss": 3.6742585652642967,
+ "tokens_seen": 1009844224
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005608184553660984,
+ "loss": 1.3136,
+ "theoretical_loss": 3.6742360902998454,
+ "tokens_seen": 1009909760
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560802407221665,
+ "loss": 1.3801,
+ "theoretical_loss": 3.6742136172021542,
+ "tokens_seen": 1009975296
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607863590772318,
+ "loss": 1.3655,
+ "theoretical_loss": 3.6741911459709464,
+ "tokens_seen": 1010040832
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607703109327984,
+ "loss": 1.3055,
+ "theoretical_loss": 3.674168676605945,
+ "tokens_seen": 1010106368
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607542627883652,
+ "loss": 1.3593,
+ "theoretical_loss": 3.674146209106875,
+ "tokens_seen": 1010171904
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607382146439318,
+ "loss": 1.3674,
+ "theoretical_loss": 3.6741237434734595,
+ "tokens_seen": 1010237440
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607221664994986,
+ "loss": 1.3816,
+ "theoretical_loss": 3.6741012797054227,
+ "tokens_seen": 1010302976
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005607061183550652,
+ "loss": 1.3807,
+ "theoretical_loss": 3.6740788178024895,
+ "tokens_seen": 1010368512
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560690070210632,
+ "loss": 1.4319,
+ "theoretical_loss": 3.6740563577643837,
+ "tokens_seen": 1010434048
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005606740220661986,
+ "loss": 1.3671,
+ "theoretical_loss": 3.674033899590829,
+ "tokens_seen": 1010499584
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005606579739217654,
+ "loss": 1.3143,
+ "theoretical_loss": 3.6740114432815503,
+ "tokens_seen": 1010565120
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560641925777332,
+ "loss": 1.3913,
+ "theoretical_loss": 3.6739889888362716,
+ "tokens_seen": 1010630656
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005606258776328988,
+ "loss": 1.3219,
+ "theoretical_loss": 3.6739665362547176,
+ "tokens_seen": 1010696192
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005606098294884654,
+ "loss": 1.2968,
+ "theoretical_loss": 3.673944085536613,
+ "tokens_seen": 1010761728
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005605937813440322,
+ "loss": 1.3181,
+ "theoretical_loss": 3.6739216366816816,
+ "tokens_seen": 1010827264
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005605777331995988,
+ "loss": 1.4921,
+ "theoretical_loss": 3.6738991896896485,
+ "tokens_seen": 1010892800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005605616850551656,
+ "loss": 1.2868,
+ "theoretical_loss": 3.6738767445602383,
+ "tokens_seen": 1010958336
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005605456369107322,
+ "loss": 1.2886,
+ "theoretical_loss": 3.6738543012931757,
+ "tokens_seen": 1011023872
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560529588766299,
+ "loss": 1.3538,
+ "theoretical_loss": 3.6738318598881854,
+ "tokens_seen": 1011089408
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005605135406218656,
+ "loss": 1.354,
+ "theoretical_loss": 3.673809420344993,
+ "tokens_seen": 1011154944
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604974924774324,
+ "loss": 1.3059,
+ "theoretical_loss": 3.6737869826633216,
+ "tokens_seen": 1011220480
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560481444332999,
+ "loss": 1.2208,
+ "theoretical_loss": 3.6737645468428974,
+ "tokens_seen": 1011286016
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604653961885658,
+ "loss": 1.358,
+ "theoretical_loss": 3.6737421128834455,
+ "tokens_seen": 1011351552
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604493480441324,
+ "loss": 1.3231,
+ "theoretical_loss": 3.673719680784691,
+ "tokens_seen": 1011417088
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604332998996992,
+ "loss": 1.4374,
+ "theoretical_loss": 3.673697250546358,
+ "tokens_seen": 1011482624
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604172517552658,
+ "loss": 1.4058,
+ "theoretical_loss": 3.6736748221681728,
+ "tokens_seen": 1011548160
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005604012036108326,
+ "loss": 1.2774,
+ "theoretical_loss": 3.67365239564986,
+ "tokens_seen": 1011613696
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005603851554663992,
+ "loss": 1.4357,
+ "theoretical_loss": 3.6736299709911453,
+ "tokens_seen": 1011679232
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560369107321966,
+ "loss": 1.431,
+ "theoretical_loss": 3.673607548191754,
+ "tokens_seen": 1011744768
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005603530591775326,
+ "loss": 1.3777,
+ "theoretical_loss": 3.6735851272514113,
+ "tokens_seen": 1011810304
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005603370110330994,
+ "loss": 1.2492,
+ "theoretical_loss": 3.6735627081698423,
+ "tokens_seen": 1011875840
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000560320962888666,
+ "loss": 1.3058,
+ "theoretical_loss": 3.6735402909467734,
+ "tokens_seen": 1011941376
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005603049147442327,
+ "loss": 1.3889,
+ "theoretical_loss": 3.6735178755819295,
+ "tokens_seen": 1012006912
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602888665997994,
+ "loss": 1.441,
+ "theoretical_loss": 3.673495462075037,
+ "tokens_seen": 1012072448
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602728184553661,
+ "loss": 1.2577,
+ "theoretical_loss": 3.6734730504258204,
+ "tokens_seen": 1012137984
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602567703109328,
+ "loss": 1.4346,
+ "theoretical_loss": 3.6734506406340066,
+ "tokens_seen": 1012203520
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602407221664995,
+ "loss": 1.2212,
+ "theoretical_loss": 3.673428232699321,
+ "tokens_seen": 1012269056
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602246740220663,
+ "loss": 1.379,
+ "theoretical_loss": 3.6734058266214893,
+ "tokens_seen": 1012334592
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005602086258776329,
+ "loss": 1.4254,
+ "theoretical_loss": 3.6733834224002377,
+ "tokens_seen": 1012400128
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601925777331995,
+ "loss": 1.2654,
+ "theoretical_loss": 3.6733610200352924,
+ "tokens_seen": 1012465664
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 342041,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2200353145599365,
+ "objective/train/theoretical_loss": 3.673338619526379,
+ "objective/train/tokens_used": 1032991200,
+ "theoretical_loss": 3.673338619526379,
+ "tokens_seen": 1012531200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601765295887663,
+ "loss": 1.2664,
+ "theoretical_loss": 3.673338619526379,
+ "tokens_seen": 1012531200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601604814443329,
+ "loss": 1.309,
+ "theoretical_loss": 3.6733162208732235,
+ "tokens_seen": 1012596736
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601444332998997,
+ "loss": 1.3172,
+ "theoretical_loss": 3.6732938240755524,
+ "tokens_seen": 1012662272
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601283851554663,
+ "loss": 1.283,
+ "theoretical_loss": 3.673271429133092,
+ "tokens_seen": 1012727808
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005601123370110331,
+ "loss": 1.192,
+ "theoretical_loss": 3.6732490360455685,
+ "tokens_seen": 1012793344
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600962888665997,
+ "loss": 1.3267,
+ "theoretical_loss": 3.6732266448127078,
+ "tokens_seen": 1012858880
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600802407221665,
+ "loss": 1.221,
+ "theoretical_loss": 3.6732042554342375,
+ "tokens_seen": 1012924416
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600641925777331,
+ "loss": 1.3396,
+ "theoretical_loss": 3.673181867909882,
+ "tokens_seen": 1012989952
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600481444332999,
+ "loss": 1.3371,
+ "theoretical_loss": 3.6731594822393703,
+ "tokens_seen": 1013055488
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600320962888665,
+ "loss": 1.3225,
+ "theoretical_loss": 3.673137098422427,
+ "tokens_seen": 1013121024
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005600160481444333,
+ "loss": 1.3604,
+ "theoretical_loss": 3.6731147164587803,
+ "tokens_seen": 1013186560
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00056,
+ "loss": 1.262,
+ "theoretical_loss": 3.6730923363481556,
+ "tokens_seen": 1013252096
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599839518555667,
+ "loss": 1.3062,
+ "theoretical_loss": 3.67306995809028,
+ "tokens_seen": 1013317632
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599679037111334,
+ "loss": 1.3287,
+ "theoretical_loss": 3.6730475816848807,
+ "tokens_seen": 1013383168
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599518555667001,
+ "loss": 1.4042,
+ "theoretical_loss": 3.6730252071316842,
+ "tokens_seen": 1013448704
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599358074222668,
+ "loss": 1.3411,
+ "theoretical_loss": 3.6730028344304175,
+ "tokens_seen": 1013514240
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599197592778335,
+ "loss": 1.3035,
+ "theoretical_loss": 3.6729804635808074,
+ "tokens_seen": 1013579776
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005599037111334002,
+ "loss": 1.4123,
+ "theoretical_loss": 3.6729580945825813,
+ "tokens_seen": 1013645312
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005598876629889669,
+ "loss": 1.2369,
+ "theoretical_loss": 3.6729357274354664,
+ "tokens_seen": 1013710848
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005598716148445336,
+ "loss": 1.399,
+ "theoretical_loss": 3.672913362139189,
+ "tokens_seen": 1013776384
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005598555667001003,
+ "loss": 1.3412,
+ "theoretical_loss": 3.6728909986934775,
+ "tokens_seen": 1013841920
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000559839518555667,
+ "loss": 1.3273,
+ "theoretical_loss": 3.6728686370980586,
+ "tokens_seen": 1013907456
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005598234704112337,
+ "loss": 1.4403,
+ "theoretical_loss": 3.6728462773526593,
+ "tokens_seen": 1013972992
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005598074222668004,
+ "loss": 1.3912,
+ "theoretical_loss": 3.6728239194570076,
+ "tokens_seen": 1014038528
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597913741223671,
+ "loss": 1.3262,
+ "theoretical_loss": 3.67280156341083,
+ "tokens_seen": 1014104064
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597753259779338,
+ "loss": 1.2509,
+ "theoretical_loss": 3.6727792092138554,
+ "tokens_seen": 1014169600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597592778335005,
+ "loss": 1.3865,
+ "theoretical_loss": 3.6727568568658104,
+ "tokens_seen": 1014235136
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597432296890673,
+ "loss": 1.2653,
+ "theoretical_loss": 3.6727345063664227,
+ "tokens_seen": 1014300672
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597271815446339,
+ "loss": 1.3199,
+ "theoretical_loss": 3.67271215771542,
+ "tokens_seen": 1014366208
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005597111334002007,
+ "loss": 1.3486,
+ "theoretical_loss": 3.67268981091253,
+ "tokens_seen": 1014431744
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596950852557673,
+ "loss": 1.3965,
+ "theoretical_loss": 3.672667465957481,
+ "tokens_seen": 1014497280
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596790371113341,
+ "loss": 1.3149,
+ "theoretical_loss": 3.6726451228500006,
+ "tokens_seen": 1014562816
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596629889669007,
+ "loss": 1.2876,
+ "theoretical_loss": 3.6726227815898165,
+ "tokens_seen": 1014628352
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596469408224675,
+ "loss": 1.4492,
+ "theoretical_loss": 3.6726004421766567,
+ "tokens_seen": 1014693888
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596308926780341,
+ "loss": 1.3578,
+ "theoretical_loss": 3.672578104610249,
+ "tokens_seen": 1014759424
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005596148445336009,
+ "loss": 1.2204,
+ "theoretical_loss": 3.672555768890322,
+ "tokens_seen": 1014824960
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595987963891675,
+ "loss": 1.3516,
+ "theoretical_loss": 3.6725334350166037,
+ "tokens_seen": 1014890496
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595827482447343,
+ "loss": 1.3082,
+ "theoretical_loss": 3.672511102988822,
+ "tokens_seen": 1014956032
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595667001003009,
+ "loss": 1.3985,
+ "theoretical_loss": 3.6724887728067053,
+ "tokens_seen": 1015021568
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595506519558677,
+ "loss": 1.3852,
+ "theoretical_loss": 3.672466444469982,
+ "tokens_seen": 1015087104
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595346038114343,
+ "loss": 1.251,
+ "theoretical_loss": 3.6724441179783804,
+ "tokens_seen": 1015152640
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595185556670011,
+ "loss": 1.3817,
+ "theoretical_loss": 3.672421793331629,
+ "tokens_seen": 1015218176
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005595025075225677,
+ "loss": 1.3173,
+ "theoretical_loss": 3.672399470529456,
+ "tokens_seen": 1015283712
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594864593781345,
+ "loss": 1.4168,
+ "theoretical_loss": 3.6723771495715907,
+ "tokens_seen": 1015349248
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594704112337011,
+ "loss": 1.3295,
+ "theoretical_loss": 3.6723548304577607,
+ "tokens_seen": 1015414784
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594543630892679,
+ "loss": 1.4062,
+ "theoretical_loss": 3.672332513187695,
+ "tokens_seen": 1015480320
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594383149448345,
+ "loss": 1.3011,
+ "theoretical_loss": 3.6723101977611226,
+ "tokens_seen": 1015545856
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594222668004013,
+ "loss": 1.4394,
+ "theoretical_loss": 3.672287884177772,
+ "tokens_seen": 1015611392
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005594062186559679,
+ "loss": 1.348,
+ "theoretical_loss": 3.6722655724373725,
+ "tokens_seen": 1015676928
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593901705115347,
+ "loss": 1.3012,
+ "theoretical_loss": 3.672243262539652,
+ "tokens_seen": 1015742464
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 343196,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3883546590805054,
+ "objective/train/theoretical_loss": 3.6722209544843407,
+ "objective/train/tokens_used": 1036268000,
+ "theoretical_loss": 3.6722209544843407,
+ "tokens_seen": 1015808000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593741223671013,
+ "loss": 1.3079,
+ "theoretical_loss": 3.6722209544843407,
+ "tokens_seen": 1015808000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593580742226681,
+ "loss": 1.3882,
+ "theoretical_loss": 3.6721986482711664,
+ "tokens_seen": 1015873536
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593420260782347,
+ "loss": 1.4233,
+ "theoretical_loss": 3.6721763438998587,
+ "tokens_seen": 1015939072
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593259779338015,
+ "loss": 1.2578,
+ "theoretical_loss": 3.672154041370147,
+ "tokens_seen": 1016004608
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005593099297893681,
+ "loss": 1.2674,
+ "theoretical_loss": 3.6721317406817597,
+ "tokens_seen": 1016070144
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592938816449349,
+ "loss": 1.3096,
+ "theoretical_loss": 3.6721094418344276,
+ "tokens_seen": 1016135680
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592778335005015,
+ "loss": 1.3513,
+ "theoretical_loss": 3.672087144827878,
+ "tokens_seen": 1016201216
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592617853560683,
+ "loss": 1.338,
+ "theoretical_loss": 3.672064849661841,
+ "tokens_seen": 1016266752
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592457372116349,
+ "loss": 1.32,
+ "theoretical_loss": 3.672042556336047,
+ "tokens_seen": 1016332288
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592296890672017,
+ "loss": 1.3809,
+ "theoretical_loss": 3.6720202648502234,
+ "tokens_seen": 1016397824
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005592136409227683,
+ "loss": 1.386,
+ "theoretical_loss": 3.6719979752041016,
+ "tokens_seen": 1016463360
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591975927783351,
+ "loss": 1.2795,
+ "theoretical_loss": 3.6719756873974103,
+ "tokens_seen": 1016528896
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591815446339017,
+ "loss": 1.3663,
+ "theoretical_loss": 3.6719534014298794,
+ "tokens_seen": 1016594432
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591654964894685,
+ "loss": 1.3,
+ "theoretical_loss": 3.671931117301239,
+ "tokens_seen": 1016659968
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591494483450351,
+ "loss": 1.459,
+ "theoretical_loss": 3.6719088350112177,
+ "tokens_seen": 1016725504
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591334002006019,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6718865545595456,
+ "tokens_seen": 1016791040
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591173520561685,
+ "loss": 1.355,
+ "theoretical_loss": 3.6718642759459534,
+ "tokens_seen": 1016856576
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005591013039117353,
+ "loss": 1.4298,
+ "theoretical_loss": 3.6718419991701703,
+ "tokens_seen": 1016922112
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590852557673019,
+ "loss": 1.3993,
+ "theoretical_loss": 3.671819724231927,
+ "tokens_seen": 1016987648
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590692076228687,
+ "loss": 1.3459,
+ "theoretical_loss": 3.671797451130953,
+ "tokens_seen": 1017053184
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590531594784353,
+ "loss": 1.2541,
+ "theoretical_loss": 3.6717751798669775,
+ "tokens_seen": 1017118720
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590371113340021,
+ "loss": 1.4025,
+ "theoretical_loss": 3.671752910439732,
+ "tokens_seen": 1017184256
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590210631895687,
+ "loss": 1.4173,
+ "theoretical_loss": 3.6717306428489467,
+ "tokens_seen": 1017249792
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005590050150451355,
+ "loss": 1.3877,
+ "theoretical_loss": 3.671708377094351,
+ "tokens_seen": 1017315328
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005589889669007021,
+ "loss": 1.3171,
+ "theoretical_loss": 3.6716861131756753,
+ "tokens_seen": 1017380864
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005589729187562689,
+ "loss": 1.2392,
+ "theoretical_loss": 3.671663851092651,
+ "tokens_seen": 1017446400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005589568706118355,
+ "loss": 1.4565,
+ "theoretical_loss": 3.6716415908450073,
+ "tokens_seen": 1017511936
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005589408224674023,
+ "loss": 1.4564,
+ "theoretical_loss": 3.671619332432475,
+ "tokens_seen": 1017577472
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558924774322969,
+ "loss": 1.3598,
+ "theoretical_loss": 3.671597075854785,
+ "tokens_seen": 1017643008
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005589087261785356,
+ "loss": 1.3916,
+ "theoretical_loss": 3.671574821111668,
+ "tokens_seen": 1017708544
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005588926780341023,
+ "loss": 1.298,
+ "theoretical_loss": 3.6715525682028547,
+ "tokens_seen": 1017774080
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558876629889669,
+ "loss": 1.3317,
+ "theoretical_loss": 3.6715303171280746,
+ "tokens_seen": 1017839616
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005588605817452358,
+ "loss": 1.4106,
+ "theoretical_loss": 3.6715080678870597,
+ "tokens_seen": 1017905152
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005588445336008024,
+ "loss": 1.4136,
+ "theoretical_loss": 3.671485820479541,
+ "tokens_seen": 1017970688
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005588284854563692,
+ "loss": 1.407,
+ "theoretical_loss": 3.671463574905248,
+ "tokens_seen": 1018036224
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005588124373119358,
+ "loss": 1.2389,
+ "theoretical_loss": 3.671441331163913,
+ "tokens_seen": 1018101760
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587963891675026,
+ "loss": 1.2802,
+ "theoretical_loss": 3.6714190892552665,
+ "tokens_seen": 1018167296
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587803410230692,
+ "loss": 1.2805,
+ "theoretical_loss": 3.6713968491790396,
+ "tokens_seen": 1018232832
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587642928786358,
+ "loss": 1.3192,
+ "theoretical_loss": 3.671374610934963,
+ "tokens_seen": 1018298368
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587482447342026,
+ "loss": 1.4082,
+ "theoretical_loss": 3.671352374522769,
+ "tokens_seen": 1018363904
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587321965897692,
+ "loss": 1.3361,
+ "theoretical_loss": 3.671330139942188,
+ "tokens_seen": 1018429440
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558716148445336,
+ "loss": 1.3375,
+ "theoretical_loss": 3.6713079071929506,
+ "tokens_seen": 1018494976
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005587001003009026,
+ "loss": 1.303,
+ "theoretical_loss": 3.671285676274789,
+ "tokens_seen": 1018560512
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005586840521564694,
+ "loss": 1.2425,
+ "theoretical_loss": 3.671263447187435,
+ "tokens_seen": 1018626048
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558668004012036,
+ "loss": 1.2442,
+ "theoretical_loss": 3.6712412199306192,
+ "tokens_seen": 1018691584
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005586519558676028,
+ "loss": 1.3234,
+ "theoretical_loss": 3.6712189945040734,
+ "tokens_seen": 1018757120
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005586359077231695,
+ "loss": 1.3611,
+ "theoretical_loss": 3.6711967709075295,
+ "tokens_seen": 1018822656
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005586198595787362,
+ "loss": 1.3453,
+ "theoretical_loss": 3.6711745491407184,
+ "tokens_seen": 1018888192
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005586038114343029,
+ "loss": 1.3404,
+ "theoretical_loss": 3.6711523292033723,
+ "tokens_seen": 1018953728
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005585877632898696,
+ "loss": 1.431,
+ "theoretical_loss": 3.671130111095223,
+ "tokens_seen": 1019019264
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 343997,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2892177104949951,
+ "objective/train/theoretical_loss": 3.671107894816002,
+ "objective/train/tokens_used": 1039544800,
+ "theoretical_loss": 3.671107894816002,
+ "tokens_seen": 1019084800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005585717151454363,
+ "loss": 1.2274,
+ "theoretical_loss": 3.671107894816002,
+ "tokens_seen": 1019084800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558555667001003,
+ "loss": 1.3134,
+ "theoretical_loss": 3.671085680365441,
+ "tokens_seen": 1019150336
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005585396188565698,
+ "loss": 1.3146,
+ "theoretical_loss": 3.6710634677432727,
+ "tokens_seen": 1019215872
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005585235707121364,
+ "loss": 1.3461,
+ "theoretical_loss": 3.6710412569492283,
+ "tokens_seen": 1019281408
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005585075225677032,
+ "loss": 1.3037,
+ "theoretical_loss": 3.67101904798304,
+ "tokens_seen": 1019346944
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005584914744232698,
+ "loss": 1.2787,
+ "theoretical_loss": 3.67099684084444,
+ "tokens_seen": 1019412480
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005584754262788366,
+ "loss": 1.3193,
+ "theoretical_loss": 3.6709746355331605,
+ "tokens_seen": 1019478016
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005584593781344032,
+ "loss": 1.4775,
+ "theoretical_loss": 3.6709524320489333,
+ "tokens_seen": 1019543552
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00055844332998997,
+ "loss": 1.31,
+ "theoretical_loss": 3.6709302303914906,
+ "tokens_seen": 1019609088
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005584272818455366,
+ "loss": 1.3576,
+ "theoretical_loss": 3.6709080305605655,
+ "tokens_seen": 1019674624
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005584112337011034,
+ "loss": 1.3663,
+ "theoretical_loss": 3.6708858325558893,
+ "tokens_seen": 1019740160
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00055839518555667,
+ "loss": 1.2277,
+ "theoretical_loss": 3.670863636377195,
+ "tokens_seen": 1019805696
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005583791374122368,
+ "loss": 1.4093,
+ "theoretical_loss": 3.6708414420242157,
+ "tokens_seen": 1019871232
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005583630892678034,
+ "loss": 1.3397,
+ "theoretical_loss": 3.6708192494966827,
+ "tokens_seen": 1019936768
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005583470411233702,
+ "loss": 1.4206,
+ "theoretical_loss": 3.670797058794329,
+ "tokens_seen": 1020002304
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005583309929789368,
+ "loss": 1.3876,
+ "theoretical_loss": 3.6707748699168876,
+ "tokens_seen": 1020067840
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005583149448345036,
+ "loss": 1.4703,
+ "theoretical_loss": 3.670752682864091,
+ "tokens_seen": 1020133376
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005582988966900702,
+ "loss": 1.3421,
+ "theoretical_loss": 3.670730497635671,
+ "tokens_seen": 1020198912
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558282848545637,
+ "loss": 1.4444,
+ "theoretical_loss": 3.670708314231362,
+ "tokens_seen": 1020264448
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005582668004012036,
+ "loss": 1.3564,
+ "theoretical_loss": 3.6706861326508964,
+ "tokens_seen": 1020329984
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005582507522567704,
+ "loss": 1.3278,
+ "theoretical_loss": 3.6706639528940066,
+ "tokens_seen": 1020395520
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558234704112337,
+ "loss": 1.3452,
+ "theoretical_loss": 3.6706417749604263,
+ "tokens_seen": 1020461056
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005582186559679038,
+ "loss": 1.3457,
+ "theoretical_loss": 3.6706195988498873,
+ "tokens_seen": 1020526592
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005582026078234704,
+ "loss": 1.4581,
+ "theoretical_loss": 3.670597424562124,
+ "tokens_seen": 1020592128
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005581865596790372,
+ "loss": 1.2731,
+ "theoretical_loss": 3.670575252096869,
+ "tokens_seen": 1020657664
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005581705115346038,
+ "loss": 1.3628,
+ "theoretical_loss": 3.670553081453855,
+ "tokens_seen": 1020723200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005581544633901706,
+ "loss": 1.3148,
+ "theoretical_loss": 3.6705309126328163,
+ "tokens_seen": 1020788736
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005581384152457372,
+ "loss": 1.3534,
+ "theoretical_loss": 3.6705087456334855,
+ "tokens_seen": 1020854272
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558122367101304,
+ "loss": 1.4729,
+ "theoretical_loss": 3.6704865804555964,
+ "tokens_seen": 1020919808
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005581063189568706,
+ "loss": 1.4037,
+ "theoretical_loss": 3.6704644170988816,
+ "tokens_seen": 1020985344
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005580902708124374,
+ "loss": 1.2846,
+ "theoretical_loss": 3.6704422555630756,
+ "tokens_seen": 1021050880
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000558074222668004,
+ "loss": 1.2909,
+ "theoretical_loss": 3.6704200958479114,
+ "tokens_seen": 1021116416
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005580581745235708,
+ "loss": 1.5132,
+ "theoretical_loss": 3.6703979379531226,
+ "tokens_seen": 1021181952
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005580421263791374,
+ "loss": 1.3148,
+ "theoretical_loss": 3.6703757818784433,
+ "tokens_seen": 1021247488
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005580260782347042,
+ "loss": 1.3601,
+ "theoretical_loss": 3.6703536276236064,
+ "tokens_seen": 1021313024
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005580100300902708,
+ "loss": 1.299,
+ "theoretical_loss": 3.6703314751883465,
+ "tokens_seen": 1021378560
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005579939819458376,
+ "loss": 1.3861,
+ "theoretical_loss": 3.670309324572396,
+ "tokens_seen": 1021444096
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005579779338014042,
+ "loss": 1.3621,
+ "theoretical_loss": 3.670287175775491,
+ "tokens_seen": 1021509632
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557961885656971,
+ "loss": 1.3271,
+ "theoretical_loss": 3.6702650287973637,
+ "tokens_seen": 1021575168
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005579458375125376,
+ "loss": 1.2815,
+ "theoretical_loss": 3.6702428836377488,
+ "tokens_seen": 1021640704
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005579297893681044,
+ "loss": 1.4229,
+ "theoretical_loss": 3.67022074029638,
+ "tokens_seen": 1021706240
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557913741223671,
+ "loss": 1.2939,
+ "theoretical_loss": 3.6701985987729913,
+ "tokens_seen": 1021771776
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578976930792378,
+ "loss": 1.4508,
+ "theoretical_loss": 3.670176459067317,
+ "tokens_seen": 1021837312
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578816449348044,
+ "loss": 1.353,
+ "theoretical_loss": 3.6701543211790923,
+ "tokens_seen": 1021902848
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578655967903712,
+ "loss": 1.2911,
+ "theoretical_loss": 3.6701321851080495,
+ "tokens_seen": 1021968384
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578495486459378,
+ "loss": 1.3805,
+ "theoretical_loss": 3.670110050853925,
+ "tokens_seen": 1022033920
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578335005015046,
+ "loss": 1.2826,
+ "theoretical_loss": 3.670087918416451,
+ "tokens_seen": 1022099456
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005578174523570712,
+ "loss": 1.391,
+ "theoretical_loss": 3.670065787795364,
+ "tokens_seen": 1022164992
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557801404212638,
+ "loss": 1.3321,
+ "theoretical_loss": 3.670043658990397,
+ "tokens_seen": 1022230528
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005577853560682046,
+ "loss": 1.4022,
+ "theoretical_loss": 3.670021532001286,
+ "tokens_seen": 1022296064
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 345138,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3124229907989502,
+ "objective/train/theoretical_loss": 3.6699994068277637,
+ "objective/train/tokens_used": 1042821600,
+ "theoretical_loss": 3.6699994068277637,
+ "tokens_seen": 1022361600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005577693079237714,
+ "loss": 1.3542,
+ "theoretical_loss": 3.6699994068277637,
+ "tokens_seen": 1022361600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557753259779338,
+ "loss": 1.326,
+ "theoretical_loss": 3.6699772834695663,
+ "tokens_seen": 1022427136
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005577372116349048,
+ "loss": 1.4202,
+ "theoretical_loss": 3.6699551619264277,
+ "tokens_seen": 1022492672
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005577211634904714,
+ "loss": 1.3727,
+ "theoretical_loss": 3.6699330421980836,
+ "tokens_seen": 1022558208
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005577051153460382,
+ "loss": 1.2421,
+ "theoretical_loss": 3.6699109242842676,
+ "tokens_seen": 1022623744
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005576890672016048,
+ "loss": 1.3208,
+ "theoretical_loss": 3.669888808184715,
+ "tokens_seen": 1022689280
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005576730190571716,
+ "loss": 1.3568,
+ "theoretical_loss": 3.6698666938991615,
+ "tokens_seen": 1022754816
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005576569709127382,
+ "loss": 1.3501,
+ "theoretical_loss": 3.6698445814273413,
+ "tokens_seen": 1022820352
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557640922768305,
+ "loss": 1.3411,
+ "theoretical_loss": 3.669822470768989,
+ "tokens_seen": 1022885888
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005576248746238716,
+ "loss": 1.4406,
+ "theoretical_loss": 3.669800361923841,
+ "tokens_seen": 1022951424
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005576088264794384,
+ "loss": 1.2884,
+ "theoretical_loss": 3.669778254891632,
+ "tokens_seen": 1023016960
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557592778335005,
+ "loss": 1.3368,
+ "theoretical_loss": 3.6697561496720965,
+ "tokens_seen": 1023082496
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005575767301905718,
+ "loss": 1.4343,
+ "theoretical_loss": 3.6697340462649706,
+ "tokens_seen": 1023148032
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005575606820461384,
+ "loss": 1.4026,
+ "theoretical_loss": 3.669711944669989,
+ "tokens_seen": 1023213568
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005575446339017052,
+ "loss": 1.3179,
+ "theoretical_loss": 3.669689844886887,
+ "tokens_seen": 1023279104
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005575285857572718,
+ "loss": 1.2564,
+ "theoretical_loss": 3.669667746915401,
+ "tokens_seen": 1023344640
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005575125376128385,
+ "loss": 1.4114,
+ "theoretical_loss": 3.6696456507552657,
+ "tokens_seen": 1023410176
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574964894684052,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6696235564062167,
+ "tokens_seen": 1023475712
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574804413239719,
+ "loss": 1.292,
+ "theoretical_loss": 3.6696014638679904,
+ "tokens_seen": 1023541248
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574643931795387,
+ "loss": 1.3808,
+ "theoretical_loss": 3.669579373140321,
+ "tokens_seen": 1023606784
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574483450351053,
+ "loss": 1.4791,
+ "theoretical_loss": 3.669557284222946,
+ "tokens_seen": 1023672320
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000557432296890672,
+ "loss": 1.3604,
+ "theoretical_loss": 3.669535197115599,
+ "tokens_seen": 1023737856
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574162487462387,
+ "loss": 1.4341,
+ "theoretical_loss": 3.6695131118180173,
+ "tokens_seen": 1023803392
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005574002006018055,
+ "loss": 1.3428,
+ "theoretical_loss": 3.6694910283299373,
+ "tokens_seen": 1023868928
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573841524573721,
+ "loss": 1.4249,
+ "theoretical_loss": 3.6694689466510937,
+ "tokens_seen": 1023934464
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573681043129387,
+ "loss": 1.3218,
+ "theoretical_loss": 3.6694468667812226,
+ "tokens_seen": 1024000000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573520561685055,
+ "loss": 1.3406,
+ "theoretical_loss": 3.66942478872006,
+ "tokens_seen": 1024065536
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573360080240723,
+ "loss": 1.4468,
+ "theoretical_loss": 3.669402712467343,
+ "tokens_seen": 1024131072
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573199598796389,
+ "loss": 1.2285,
+ "theoretical_loss": 3.6693806380228073,
+ "tokens_seen": 1024196608
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005573039117352057,
+ "loss": 1.238,
+ "theoretical_loss": 3.6693585653861884,
+ "tokens_seen": 1024262144
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572878635907723,
+ "loss": 1.471,
+ "theoretical_loss": 3.669336494557223,
+ "tokens_seen": 1024327680
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572718154463391,
+ "loss": 1.4066,
+ "theoretical_loss": 3.6693144255356485,
+ "tokens_seen": 1024393216
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572557673019057,
+ "loss": 1.3275,
+ "theoretical_loss": 3.6692923583211994,
+ "tokens_seen": 1024458752
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572397191574725,
+ "loss": 1.3698,
+ "theoretical_loss": 3.6692702929136134,
+ "tokens_seen": 1024524288
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572236710130391,
+ "loss": 1.3979,
+ "theoretical_loss": 3.6692482293126263,
+ "tokens_seen": 1024589824
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005572076228686059,
+ "loss": 1.2864,
+ "theoretical_loss": 3.669226167517975,
+ "tokens_seen": 1024655360
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571915747241725,
+ "loss": 1.3593,
+ "theoretical_loss": 3.6692041075293966,
+ "tokens_seen": 1024720896
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571755265797393,
+ "loss": 1.3714,
+ "theoretical_loss": 3.6691820493466265,
+ "tokens_seen": 1024786432
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571594784353059,
+ "loss": 1.3756,
+ "theoretical_loss": 3.6691599929694023,
+ "tokens_seen": 1024851968
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571434302908727,
+ "loss": 1.3525,
+ "theoretical_loss": 3.6691379383974607,
+ "tokens_seen": 1024917504
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571273821464393,
+ "loss": 1.2336,
+ "theoretical_loss": 3.6691158856305384,
+ "tokens_seen": 1024983040
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005571113340020061,
+ "loss": 1.3104,
+ "theoretical_loss": 3.6690938346683724,
+ "tokens_seen": 1025048576
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570952858575727,
+ "loss": 1.3472,
+ "theoretical_loss": 3.6690717855106993,
+ "tokens_seen": 1025114112
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570792377131395,
+ "loss": 1.3492,
+ "theoretical_loss": 3.6690497381572564,
+ "tokens_seen": 1025179648
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570631895687061,
+ "loss": 1.4162,
+ "theoretical_loss": 3.6690276926077807,
+ "tokens_seen": 1025245184
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570471414242729,
+ "loss": 1.2798,
+ "theoretical_loss": 3.669005648862009,
+ "tokens_seen": 1025310720
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570310932798395,
+ "loss": 1.3499,
+ "theoretical_loss": 3.668983606919679,
+ "tokens_seen": 1025376256
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005570150451354063,
+ "loss": 1.4369,
+ "theoretical_loss": 3.668961566780527,
+ "tokens_seen": 1025441792
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569989969909729,
+ "loss": 1.4385,
+ "theoretical_loss": 3.6689395284442914,
+ "tokens_seen": 1025507328
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569829488465397,
+ "loss": 1.4733,
+ "theoretical_loss": 3.668917491910709,
+ "tokens_seen": 1025572864
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 345966,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3591279983520508,
+ "objective/train/theoretical_loss": 3.6688954571795165,
+ "objective/train/tokens_used": 1046098400,
+ "theoretical_loss": 3.6688954571795165,
+ "tokens_seen": 1025638400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569669007021063,
+ "loss": 1.3544,
+ "theoretical_loss": 3.6688954571795165,
+ "tokens_seen": 1025638400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569508525576731,
+ "loss": 1.2817,
+ "theoretical_loss": 3.6688734242504526,
+ "tokens_seen": 1025703936
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569348044132397,
+ "loss": 1.442,
+ "theoretical_loss": 3.668851393123254,
+ "tokens_seen": 1025769472
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569187562688065,
+ "loss": 1.3131,
+ "theoretical_loss": 3.6688293637976583,
+ "tokens_seen": 1025835008
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005569027081243731,
+ "loss": 1.3058,
+ "theoretical_loss": 3.668807336273403,
+ "tokens_seen": 1025900544
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568866599799399,
+ "loss": 1.3676,
+ "theoretical_loss": 3.668785310550226,
+ "tokens_seen": 1025966080
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568706118355065,
+ "loss": 1.438,
+ "theoretical_loss": 3.6687632866278657,
+ "tokens_seen": 1026031616
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568545636910733,
+ "loss": 1.3054,
+ "theoretical_loss": 3.668741264506058,
+ "tokens_seen": 1026097152
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568385155466399,
+ "loss": 1.432,
+ "theoretical_loss": 3.668719244184542,
+ "tokens_seen": 1026162688
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568224674022067,
+ "loss": 1.3624,
+ "theoretical_loss": 3.668697225663056,
+ "tokens_seen": 1026228224
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005568064192577733,
+ "loss": 1.431,
+ "theoretical_loss": 3.6686752089413366,
+ "tokens_seen": 1026293760
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567903711133401,
+ "loss": 1.3012,
+ "theoretical_loss": 3.6686531940191225,
+ "tokens_seen": 1026359296
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567743229689067,
+ "loss": 1.4499,
+ "theoretical_loss": 3.668631180896152,
+ "tokens_seen": 1026424832
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567582748244735,
+ "loss": 1.3879,
+ "theoretical_loss": 3.6686091695721625,
+ "tokens_seen": 1026490368
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567422266800401,
+ "loss": 1.311,
+ "theoretical_loss": 3.6685871600468927,
+ "tokens_seen": 1026555904
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567261785356069,
+ "loss": 1.3203,
+ "theoretical_loss": 3.6685651523200806,
+ "tokens_seen": 1026621440
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005567101303911735,
+ "loss": 1.3811,
+ "theoretical_loss": 3.6685431463914644,
+ "tokens_seen": 1026686976
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566940822467403,
+ "loss": 1.3512,
+ "theoretical_loss": 3.668521142260783,
+ "tokens_seen": 1026752512
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566780341023069,
+ "loss": 1.4013,
+ "theoretical_loss": 3.668499139927773,
+ "tokens_seen": 1026818048
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566619859578737,
+ "loss": 1.2922,
+ "theoretical_loss": 3.668477139392175,
+ "tokens_seen": 1026883584
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566459378134403,
+ "loss": 1.3866,
+ "theoretical_loss": 3.668455140653726,
+ "tokens_seen": 1026949120
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566298896690071,
+ "loss": 1.3262,
+ "theoretical_loss": 3.668433143712165,
+ "tokens_seen": 1027014656
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005566138415245737,
+ "loss": 1.4521,
+ "theoretical_loss": 3.6684111485672313,
+ "tokens_seen": 1027080192
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565977933801405,
+ "loss": 1.3321,
+ "theoretical_loss": 3.6683891552186623,
+ "tokens_seen": 1027145728
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565817452357071,
+ "loss": 1.3675,
+ "theoretical_loss": 3.668367163666197,
+ "tokens_seen": 1027211264
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565656970912739,
+ "loss": 1.3501,
+ "theoretical_loss": 3.6683451739095747,
+ "tokens_seen": 1027276800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565496489468405,
+ "loss": 1.3871,
+ "theoretical_loss": 3.668323185948534,
+ "tokens_seen": 1027342336
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565336008024073,
+ "loss": 1.3615,
+ "theoretical_loss": 3.668301199782813,
+ "tokens_seen": 1027407872
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565175526579739,
+ "loss": 1.3568,
+ "theoretical_loss": 3.6682792154121517,
+ "tokens_seen": 1027473408
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005565015045135407,
+ "loss": 1.3667,
+ "theoretical_loss": 3.6682572328362886,
+ "tokens_seen": 1027538944
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564854563691073,
+ "loss": 1.3261,
+ "theoretical_loss": 3.6682352520549624,
+ "tokens_seen": 1027604480
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564694082246741,
+ "loss": 1.2886,
+ "theoretical_loss": 3.668213273067913,
+ "tokens_seen": 1027670016
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564533600802407,
+ "loss": 1.4568,
+ "theoretical_loss": 3.668191295874878,
+ "tokens_seen": 1027735552
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564373119358075,
+ "loss": 1.3701,
+ "theoretical_loss": 3.6681693204755987,
+ "tokens_seen": 1027801088
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564212637913741,
+ "loss": 1.4025,
+ "theoretical_loss": 3.6681473468698123,
+ "tokens_seen": 1027866624
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005564052156469409,
+ "loss": 1.3558,
+ "theoretical_loss": 3.6681253750572593,
+ "tokens_seen": 1027932160
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563891675025075,
+ "loss": 1.3646,
+ "theoretical_loss": 3.668103405037679,
+ "tokens_seen": 1027997696
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563731193580743,
+ "loss": 1.3556,
+ "theoretical_loss": 3.66808143681081,
+ "tokens_seen": 1028063232
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563570712136409,
+ "loss": 1.2993,
+ "theoretical_loss": 3.668059470376393,
+ "tokens_seen": 1028128768
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563410230692077,
+ "loss": 1.3762,
+ "theoretical_loss": 3.6680375057341665,
+ "tokens_seen": 1028194304
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563249749247743,
+ "loss": 1.401,
+ "theoretical_loss": 3.66801554288387,
+ "tokens_seen": 1028259840
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005563089267803411,
+ "loss": 1.2452,
+ "theoretical_loss": 3.6679935818252436,
+ "tokens_seen": 1028325376
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562928786359077,
+ "loss": 1.3394,
+ "theoretical_loss": 3.667971622558027,
+ "tokens_seen": 1028390912
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562768304914745,
+ "loss": 1.4385,
+ "theoretical_loss": 3.6679496650819594,
+ "tokens_seen": 1028456448
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562607823470411,
+ "loss": 1.3868,
+ "theoretical_loss": 3.6679277093967815,
+ "tokens_seen": 1028521984
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562447342026079,
+ "loss": 1.2272,
+ "theoretical_loss": 3.6679057555022325,
+ "tokens_seen": 1028587520
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562286860581745,
+ "loss": 1.233,
+ "theoretical_loss": 3.667883803398052,
+ "tokens_seen": 1028653056
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005562126379137413,
+ "loss": 1.3053,
+ "theoretical_loss": 3.6678618530839806,
+ "tokens_seen": 1028718592
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000556196589769308,
+ "loss": 1.2998,
+ "theoretical_loss": 3.6678399045597576,
+ "tokens_seen": 1028784128
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561805416248747,
+ "loss": 1.1746,
+ "theoretical_loss": 3.6678179578251235,
+ "tokens_seen": 1028849664
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 347142,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3573336601257324,
+ "objective/train/theoretical_loss": 3.667796012879819,
+ "objective/train/tokens_used": 1049375200,
+ "theoretical_loss": 3.667796012879819,
+ "tokens_seen": 1028915200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561644934804413,
+ "loss": 1.3217,
+ "theoretical_loss": 3.667796012879819,
+ "tokens_seen": 1028915200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561484453360081,
+ "loss": 1.4169,
+ "theoretical_loss": 3.667774069723583,
+ "tokens_seen": 1028980736
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561323971915747,
+ "loss": 1.2895,
+ "theoretical_loss": 3.667752128356156,
+ "tokens_seen": 1029046272
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561163490471414,
+ "loss": 1.3405,
+ "theoretical_loss": 3.6677301887772793,
+ "tokens_seen": 1029111808
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005561003009027082,
+ "loss": 1.3382,
+ "theoretical_loss": 3.667708250986692,
+ "tokens_seen": 1029177344
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005560842527582748,
+ "loss": 1.3336,
+ "theoretical_loss": 3.6676863149841354,
+ "tokens_seen": 1029242880
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005560682046138416,
+ "loss": 1.333,
+ "theoretical_loss": 3.6676643807693496,
+ "tokens_seen": 1029308416
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005560521564694082,
+ "loss": 1.3367,
+ "theoretical_loss": 3.6676424483420744,
+ "tokens_seen": 1029373952
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000556036108324975,
+ "loss": 1.3724,
+ "theoretical_loss": 3.6676205177020513,
+ "tokens_seen": 1029439488
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005560200601805416,
+ "loss": 1.2695,
+ "theoretical_loss": 3.6675985888490206,
+ "tokens_seen": 1029505024
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005560040120361084,
+ "loss": 1.3062,
+ "theoretical_loss": 3.667576661782723,
+ "tokens_seen": 1029570560
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555987963891675,
+ "loss": 1.3483,
+ "theoretical_loss": 3.6675547365028986,
+ "tokens_seen": 1029636096
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005559719157472418,
+ "loss": 1.2031,
+ "theoretical_loss": 3.6675328130092892,
+ "tokens_seen": 1029701632
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005559558676028084,
+ "loss": 1.4142,
+ "theoretical_loss": 3.6675108913016348,
+ "tokens_seen": 1029767168
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005559398194583752,
+ "loss": 1.261,
+ "theoretical_loss": 3.6674889713796763,
+ "tokens_seen": 1029832704
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005559237713139418,
+ "loss": 1.3173,
+ "theoretical_loss": 3.667467053243155,
+ "tokens_seen": 1029898240
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005559077231695086,
+ "loss": 1.4307,
+ "theoretical_loss": 3.667445136891812,
+ "tokens_seen": 1029963776
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005558916750250752,
+ "loss": 1.2764,
+ "theoretical_loss": 3.6674232223253878,
+ "tokens_seen": 1030029312
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555875626880642,
+ "loss": 1.3415,
+ "theoretical_loss": 3.6674013095436235,
+ "tokens_seen": 1030094848
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005558595787362086,
+ "loss": 1.3558,
+ "theoretical_loss": 3.6673793985462604,
+ "tokens_seen": 1030160384
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005558435305917754,
+ "loss": 1.3806,
+ "theoretical_loss": 3.6673574893330403,
+ "tokens_seen": 1030225920
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555827482447342,
+ "loss": 1.2824,
+ "theoretical_loss": 3.6673355819037035,
+ "tokens_seen": 1030291456
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005558114343029088,
+ "loss": 1.4197,
+ "theoretical_loss": 3.6673136762579914,
+ "tokens_seen": 1030356992
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005557953861584754,
+ "loss": 1.3481,
+ "theoretical_loss": 3.6672917723956457,
+ "tokens_seen": 1030422528
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005557793380140422,
+ "loss": 1.3848,
+ "theoretical_loss": 3.6672698703164084,
+ "tokens_seen": 1030488064
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005557632898696088,
+ "loss": 1.2509,
+ "theoretical_loss": 3.6672479700200196,
+ "tokens_seen": 1030553600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005557472417251756,
+ "loss": 1.4003,
+ "theoretical_loss": 3.6672260715062217,
+ "tokens_seen": 1030619136
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005557311935807422,
+ "loss": 1.3734,
+ "theoretical_loss": 3.667204174774756,
+ "tokens_seen": 1030684672
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555715145436309,
+ "loss": 1.4423,
+ "theoretical_loss": 3.6671822798253637,
+ "tokens_seen": 1030750208
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556990972918756,
+ "loss": 1.3854,
+ "theoretical_loss": 3.6671603866577875,
+ "tokens_seen": 1030815744
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556830491474424,
+ "loss": 1.4254,
+ "theoretical_loss": 3.6671384952717685,
+ "tokens_seen": 1030881280
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555667001003009,
+ "loss": 1.4466,
+ "theoretical_loss": 3.6671166056670486,
+ "tokens_seen": 1030946816
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556509528585758,
+ "loss": 1.2626,
+ "theoretical_loss": 3.6670947178433693,
+ "tokens_seen": 1031012352
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556349047141424,
+ "loss": 1.3451,
+ "theoretical_loss": 3.667072831800473,
+ "tokens_seen": 1031077888
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556188565697092,
+ "loss": 1.4075,
+ "theoretical_loss": 3.6670509475381015,
+ "tokens_seen": 1031143424
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005556028084252758,
+ "loss": 1.2638,
+ "theoretical_loss": 3.6670290650559965,
+ "tokens_seen": 1031208960
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005555867602808426,
+ "loss": 1.3928,
+ "theoretical_loss": 3.6670071843539005,
+ "tokens_seen": 1031274496
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005555707121364092,
+ "loss": 1.3528,
+ "theoretical_loss": 3.666985305431555,
+ "tokens_seen": 1031340032
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555554663991976,
+ "loss": 1.3304,
+ "theoretical_loss": 3.666963428288703,
+ "tokens_seen": 1031405568
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005555386158475426,
+ "loss": 1.2793,
+ "theoretical_loss": 3.666941552925086,
+ "tokens_seen": 1031471104
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005555225677031094,
+ "loss": 1.2503,
+ "theoretical_loss": 3.6669196793404466,
+ "tokens_seen": 1031536640
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555506519558676,
+ "loss": 1.2333,
+ "theoretical_loss": 3.666897807534527,
+ "tokens_seen": 1031602176
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554904714142428,
+ "loss": 1.3717,
+ "theoretical_loss": 3.6668759375070694,
+ "tokens_seen": 1031667712
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554744232698094,
+ "loss": 1.3454,
+ "theoretical_loss": 3.6668540692578167,
+ "tokens_seen": 1031733248
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554583751253762,
+ "loss": 1.3405,
+ "theoretical_loss": 3.6668322027865106,
+ "tokens_seen": 1031798784
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554423269809428,
+ "loss": 1.3685,
+ "theoretical_loss": 3.666810338092895,
+ "tokens_seen": 1031864320
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554262788365096,
+ "loss": 1.4335,
+ "theoretical_loss": 3.666788475176711,
+ "tokens_seen": 1031929856
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005554102306920762,
+ "loss": 1.3048,
+ "theoretical_loss": 3.6667666140377015,
+ "tokens_seen": 1031995392
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555394182547643,
+ "loss": 1.388,
+ "theoretical_loss": 3.6667447546756105,
+ "tokens_seen": 1032060928
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005553781344032096,
+ "loss": 1.2973,
+ "theoretical_loss": 3.666722897090179,
+ "tokens_seen": 1032126464
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 348090,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3344188928604126,
+ "objective/train/theoretical_loss": 3.6667010412811507,
+ "objective/train/tokens_used": 1052652000,
+ "theoretical_loss": 3.6667010412811507,
+ "tokens_seen": 1032192000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005553620862587764,
+ "loss": 1.2576,
+ "theoretical_loss": 3.6667010412811507,
+ "tokens_seen": 1032192000
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555346038114343,
+ "loss": 1.31,
+ "theoretical_loss": 3.6666791872482687,
+ "tokens_seen": 1032257536
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005553299899699098,
+ "loss": 1.3013,
+ "theoretical_loss": 3.6666573349912754,
+ "tokens_seen": 1032323072
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005553139418254764,
+ "loss": 1.4247,
+ "theoretical_loss": 3.6666354845099143,
+ "tokens_seen": 1032388608
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552978936810432,
+ "loss": 1.2885,
+ "theoretical_loss": 3.6666136358039276,
+ "tokens_seen": 1032454144
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552818455366098,
+ "loss": 1.4111,
+ "theoretical_loss": 3.666591788873059,
+ "tokens_seen": 1032519680
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552657973921766,
+ "loss": 1.2379,
+ "theoretical_loss": 3.666569943717051,
+ "tokens_seen": 1032585216
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552497492477432,
+ "loss": 1.3001,
+ "theoretical_loss": 3.666548100335648,
+ "tokens_seen": 1032650752
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00055523370110331,
+ "loss": 1.3514,
+ "theoretical_loss": 3.6665262587285916,
+ "tokens_seen": 1032716288
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552176529588766,
+ "loss": 1.3702,
+ "theoretical_loss": 3.6665044188956264,
+ "tokens_seen": 1032781824
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005552016048144434,
+ "loss": 1.3201,
+ "theoretical_loss": 3.6664825808364956,
+ "tokens_seen": 1032847360
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00055518555667001,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6664607445509416,
+ "tokens_seen": 1032912896
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005551695085255768,
+ "loss": 1.4063,
+ "theoretical_loss": 3.6664389100387087,
+ "tokens_seen": 1032978432
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005551534603811434,
+ "loss": 1.4044,
+ "theoretical_loss": 3.66641707729954,
+ "tokens_seen": 1033043968
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005551374122367102,
+ "loss": 1.3637,
+ "theoretical_loss": 3.6663952463331797,
+ "tokens_seen": 1033109504
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005551213640922768,
+ "loss": 1.3576,
+ "theoretical_loss": 3.6663734171393707,
+ "tokens_seen": 1033175040
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005551053159478436,
+ "loss": 1.2985,
+ "theoretical_loss": 3.666351589717857,
+ "tokens_seen": 1033240576
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005550892678034102,
+ "loss": 1.3548,
+ "theoretical_loss": 3.666329764068382,
+ "tokens_seen": 1033306112
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555073219658977,
+ "loss": 1.3824,
+ "theoretical_loss": 3.6663079401906895,
+ "tokens_seen": 1033371648
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005550571715145436,
+ "loss": 1.3748,
+ "theoretical_loss": 3.666286118084524,
+ "tokens_seen": 1033437184
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005550411233701104,
+ "loss": 1.5217,
+ "theoretical_loss": 3.6662642977496285,
+ "tokens_seen": 1033502720
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000555025075225677,
+ "loss": 1.3761,
+ "theoretical_loss": 3.6662424791857475,
+ "tokens_seen": 1033568256
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005550090270812438,
+ "loss": 1.267,
+ "theoretical_loss": 3.666220662392625,
+ "tokens_seen": 1033633792
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005549929789368104,
+ "loss": 1.4167,
+ "theoretical_loss": 3.6661988473700045,
+ "tokens_seen": 1033699328
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005549769307923772,
+ "loss": 1.4184,
+ "theoretical_loss": 3.66617703411763,
+ "tokens_seen": 1033764864
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005549608826479438,
+ "loss": 1.5055,
+ "theoretical_loss": 3.6661552226352465,
+ "tokens_seen": 1033830400
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005549448345035106,
+ "loss": 1.4087,
+ "theoretical_loss": 3.6661334129225978,
+ "tokens_seen": 1033895936
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005549287863590774,
+ "loss": 1.3579,
+ "theoretical_loss": 3.6661116049794273,
+ "tokens_seen": 1033961472
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000554912738214644,
+ "loss": 1.3957,
+ "theoretical_loss": 3.666089798805481,
+ "tokens_seen": 1034027008
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548966900702108,
+ "loss": 1.2851,
+ "theoretical_loss": 3.6660679944005015,
+ "tokens_seen": 1034092544
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548806419257774,
+ "loss": 1.3388,
+ "theoretical_loss": 3.6660461917642344,
+ "tokens_seen": 1034158080
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000554864593781344,
+ "loss": 1.4184,
+ "theoretical_loss": 3.666024390896424,
+ "tokens_seen": 1034223616
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548485456369108,
+ "loss": 1.369,
+ "theoretical_loss": 3.666002591796814,
+ "tokens_seen": 1034289152
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548324974924774,
+ "loss": 1.3376,
+ "theoretical_loss": 3.665980794465149,
+ "tokens_seen": 1034354688
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548164493480442,
+ "loss": 1.3469,
+ "theoretical_loss": 3.6659589989011754,
+ "tokens_seen": 1034420224
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005548004012036108,
+ "loss": 1.3072,
+ "theoretical_loss": 3.665937205104636,
+ "tokens_seen": 1034485760
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005547843530591776,
+ "loss": 1.3264,
+ "theoretical_loss": 3.665915413075276,
+ "tokens_seen": 1034551296
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005547683049147442,
+ "loss": 1.469,
+ "theoretical_loss": 3.66589362281284,
+ "tokens_seen": 1034616832
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000554752256770311,
+ "loss": 1.3863,
+ "theoretical_loss": 3.6658718343170738,
+ "tokens_seen": 1034682368
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005547362086258777,
+ "loss": 1.3203,
+ "theoretical_loss": 3.665850047587721,
+ "tokens_seen": 1034747904
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005547201604814443,
+ "loss": 1.4613,
+ "theoretical_loss": 3.6658282626245278,
+ "tokens_seen": 1034813440
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.000554704112337011,
+ "loss": 1.3122,
+ "theoretical_loss": 3.6658064794272383,
+ "tokens_seen": 1034878976
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546880641925777,
+ "loss": 1.3626,
+ "theoretical_loss": 3.6657846979955973,
+ "tokens_seen": 1034944512
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546720160481445,
+ "loss": 1.3732,
+ "theoretical_loss": 3.6657629183293508,
+ "tokens_seen": 1035010048
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546559679037111,
+ "loss": 1.4149,
+ "theoretical_loss": 3.665741140428243,
+ "tokens_seen": 1035075584
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546399197592779,
+ "loss": 1.3765,
+ "theoretical_loss": 3.66571936429202,
+ "tokens_seen": 1035141120
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546238716148445,
+ "loss": 1.3002,
+ "theoretical_loss": 3.6656975899204265,
+ "tokens_seen": 1035206656
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005546078234704113,
+ "loss": 1.3946,
+ "theoretical_loss": 3.665675817313208,
+ "tokens_seen": 1035272192
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545917753259779,
+ "loss": 1.4278,
+ "theoretical_loss": 3.6656540464701095,
+ "tokens_seen": 1035337728
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545757271815447,
+ "loss": 1.4373,
+ "theoretical_loss": 3.665632277390877,
+ "tokens_seen": 1035403264
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 349230,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5953205823898315,
+ "objective/train/theoretical_loss": 3.665610510075256,
+ "objective/train/tokens_used": 1055928800,
+ "theoretical_loss": 3.665610510075256,
+ "tokens_seen": 1035468800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545596790371113,
+ "loss": 1.5163,
+ "theoretical_loss": 3.665610510075256,
+ "tokens_seen": 1035468800
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545436308926781,
+ "loss": 1.3309,
+ "theoretical_loss": 3.6655887445229913,
+ "tokens_seen": 1035534336
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545275827482447,
+ "loss": 1.3277,
+ "theoretical_loss": 3.6655669807338285,
+ "tokens_seen": 1035599872
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005545115346038115,
+ "loss": 1.296,
+ "theoretical_loss": 3.665545218707514,
+ "tokens_seen": 1035665408
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544954864593781,
+ "loss": 1.3078,
+ "theoretical_loss": 3.6655234584437926,
+ "tokens_seen": 1035730944
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544794383149449,
+ "loss": 1.4387,
+ "theoretical_loss": 3.665501699942411,
+ "tokens_seen": 1035796480
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544633901705115,
+ "loss": 1.3512,
+ "theoretical_loss": 3.665479943203114,
+ "tokens_seen": 1035862016
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544473420260783,
+ "loss": 1.2364,
+ "theoretical_loss": 3.6654581882256485,
+ "tokens_seen": 1035927552
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544312938816449,
+ "loss": 1.2795,
+ "theoretical_loss": 3.6654364350097595,
+ "tokens_seen": 1035993088
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005544152457372117,
+ "loss": 1.3323,
+ "theoretical_loss": 3.665414683555193,
+ "tokens_seen": 1036058624
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543991975927783,
+ "loss": 1.3176,
+ "theoretical_loss": 3.665392933861695,
+ "tokens_seen": 1036124160
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543831494483451,
+ "loss": 1.3466,
+ "theoretical_loss": 3.665371185929012,
+ "tokens_seen": 1036189696
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543671013039117,
+ "loss": 1.2921,
+ "theoretical_loss": 3.6653494397568895,
+ "tokens_seen": 1036255232
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543510531594785,
+ "loss": 1.3309,
+ "theoretical_loss": 3.6653276953450744,
+ "tokens_seen": 1036320768
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543350050150451,
+ "loss": 1.3214,
+ "theoretical_loss": 3.6653059526933123,
+ "tokens_seen": 1036386304
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543189568706119,
+ "loss": 1.342,
+ "theoretical_loss": 3.6652842118013496,
+ "tokens_seen": 1036451840
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005543029087261785,
+ "loss": 1.342,
+ "theoretical_loss": 3.6652624726689327,
+ "tokens_seen": 1036517376
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542868605817453,
+ "loss": 1.4497,
+ "theoretical_loss": 3.6652407352958076,
+ "tokens_seen": 1036582912
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542708124373119,
+ "loss": 1.3414,
+ "theoretical_loss": 3.665218999681721,
+ "tokens_seen": 1036648448
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542547642928787,
+ "loss": 1.4255,
+ "theoretical_loss": 3.6651972658264196,
+ "tokens_seen": 1036713984
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542387161484453,
+ "loss": 1.3207,
+ "theoretical_loss": 3.66517553372965,
+ "tokens_seen": 1036779520
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542226680040121,
+ "loss": 1.302,
+ "theoretical_loss": 3.6651538033911577,
+ "tokens_seen": 1036845056
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005542066198595787,
+ "loss": 1.3584,
+ "theoretical_loss": 3.6651320748106904,
+ "tokens_seen": 1036910592
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541905717151455,
+ "loss": 1.2684,
+ "theoretical_loss": 3.6651103479879943,
+ "tokens_seen": 1036976128
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541745235707121,
+ "loss": 1.2881,
+ "theoretical_loss": 3.665088622922817,
+ "tokens_seen": 1037041664
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541584754262789,
+ "loss": 1.4304,
+ "theoretical_loss": 3.665066899614904,
+ "tokens_seen": 1037107200
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541424272818455,
+ "loss": 1.363,
+ "theoretical_loss": 3.665045178064002,
+ "tokens_seen": 1037172736
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541263791374123,
+ "loss": 1.2961,
+ "theoretical_loss": 3.6650234582698595,
+ "tokens_seen": 1037238272
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005541103309929789,
+ "loss": 1.3131,
+ "theoretical_loss": 3.665001740232222,
+ "tokens_seen": 1037303808
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540942828485457,
+ "loss": 1.3062,
+ "theoretical_loss": 3.664980023950837,
+ "tokens_seen": 1037369344
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540782347041123,
+ "loss": 1.3362,
+ "theoretical_loss": 3.664958309425452,
+ "tokens_seen": 1037434880
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540621865596791,
+ "loss": 1.3453,
+ "theoretical_loss": 3.664936596655813,
+ "tokens_seen": 1037500416
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540461384152457,
+ "loss": 1.4069,
+ "theoretical_loss": 3.664914885641668,
+ "tokens_seen": 1037565952
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540300902708125,
+ "loss": 1.3398,
+ "theoretical_loss": 3.664893176382764,
+ "tokens_seen": 1037631488
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005540140421263791,
+ "loss": 1.4761,
+ "theoretical_loss": 3.6648714688788484,
+ "tokens_seen": 1037697024
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539979939819459,
+ "loss": 1.3642,
+ "theoretical_loss": 3.664849763129668,
+ "tokens_seen": 1037762560
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539819458375125,
+ "loss": 1.3796,
+ "theoretical_loss": 3.6648280591349707,
+ "tokens_seen": 1037828096
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539658976930793,
+ "loss": 1.3271,
+ "theoretical_loss": 3.6648063568945033,
+ "tokens_seen": 1037893632
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539498495486459,
+ "loss": 1.3468,
+ "theoretical_loss": 3.6647846564080138,
+ "tokens_seen": 1037959168
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539338014042127,
+ "loss": 1.2307,
+ "theoretical_loss": 3.66476295767525,
+ "tokens_seen": 1038024704
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539177532597793,
+ "loss": 1.2928,
+ "theoretical_loss": 3.6647412606959584,
+ "tokens_seen": 1038090240
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005539017051153461,
+ "loss": 1.29,
+ "theoretical_loss": 3.664719565469887,
+ "tokens_seen": 1038155776
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538856569709127,
+ "loss": 1.3733,
+ "theoretical_loss": 3.6646978719967844,
+ "tokens_seen": 1038221312
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538696088264795,
+ "loss": 1.1986,
+ "theoretical_loss": 3.664676180276397,
+ "tokens_seen": 1038286848
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538535606820461,
+ "loss": 1.3375,
+ "theoretical_loss": 3.6646544903084735,
+ "tokens_seen": 1038352384
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538375125376129,
+ "loss": 1.2587,
+ "theoretical_loss": 3.664632802092761,
+ "tokens_seen": 1038417920
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538214643931795,
+ "loss": 1.2895,
+ "theoretical_loss": 3.6646111156290084,
+ "tokens_seen": 1038483456
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005538054162487463,
+ "loss": 1.2788,
+ "theoretical_loss": 3.664589430916963,
+ "tokens_seen": 1038548992
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537893681043129,
+ "loss": 1.2994,
+ "theoretical_loss": 3.6645677479563723,
+ "tokens_seen": 1038614528
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537733199598797,
+ "loss": 1.3,
+ "theoretical_loss": 3.6645460667469854,
+ "tokens_seen": 1038680064
+ },
+ {
+ "epoch": 0.31,
+ "objective/train/docs_used": 350251,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3899097442626953,
+ "objective/train/theoretical_loss": 3.66452438728855,
+ "objective/train/tokens_used": 1059205600,
+ "theoretical_loss": 3.66452438728855,
+ "tokens_seen": 1038745600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537572718154463,
+ "loss": 1.332,
+ "theoretical_loss": 3.66452438728855,
+ "tokens_seen": 1038745600
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537412236710131,
+ "loss": 1.3857,
+ "theoretical_loss": 3.6645027095808134,
+ "tokens_seen": 1038811136
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537251755265798,
+ "loss": 1.2491,
+ "theoretical_loss": 3.6644810336235243,
+ "tokens_seen": 1038876672
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005537091273821465,
+ "loss": 1.3745,
+ "theoretical_loss": 3.6644593594164316,
+ "tokens_seen": 1038942208
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005536930792377132,
+ "loss": 1.3818,
+ "theoretical_loss": 3.6644376869592827,
+ "tokens_seen": 1039007744
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005536770310932799,
+ "loss": 1.2186,
+ "theoretical_loss": 3.6644160162518267,
+ "tokens_seen": 1039073280
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005536609829488466,
+ "loss": 1.3415,
+ "theoretical_loss": 3.664394347293811,
+ "tokens_seen": 1039138816
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005536449348044133,
+ "loss": 1.358,
+ "theoretical_loss": 3.664372680084986,
+ "tokens_seen": 1039204352
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.00055362888665998,
+ "loss": 1.3312,
+ "theoretical_loss": 3.6643510146250975,
+ "tokens_seen": 1039269888
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005536128385155467,
+ "loss": 1.2917,
+ "theoretical_loss": 3.664329350913896,
+ "tokens_seen": 1039335424
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005535967903711134,
+ "loss": 1.2901,
+ "theoretical_loss": 3.6643076889511295,
+ "tokens_seen": 1039400960
+ },
+ {
+ "epoch": 0.31,
+ "learning_rate": 0.0005535807422266801,
+ "loss": 1.3579,
+ "theoretical_loss": 3.6642860287365466,
+ "tokens_seen": 1039466496
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005535646940822469,
+ "loss": 1.3679,
+ "theoretical_loss": 3.6642643702698963,
+ "tokens_seen": 1039532032
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005535486459378135,
+ "loss": 1.3271,
+ "theoretical_loss": 3.6642427135509275,
+ "tokens_seen": 1039597568
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005535325977933803,
+ "loss": 1.3528,
+ "theoretical_loss": 3.664221058579388,
+ "tokens_seen": 1039663104
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005535165496489469,
+ "loss": 1.3484,
+ "theoretical_loss": 3.6641994053550277,
+ "tokens_seen": 1039728640
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005535005015045137,
+ "loss": 1.3224,
+ "theoretical_loss": 3.664177753877595,
+ "tokens_seen": 1039794176
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534844533600803,
+ "loss": 1.2996,
+ "theoretical_loss": 3.664156104146839,
+ "tokens_seen": 1039859712
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534684052156471,
+ "loss": 1.2883,
+ "theoretical_loss": 3.664134456162509,
+ "tokens_seen": 1039925248
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534523570712137,
+ "loss": 1.4707,
+ "theoretical_loss": 3.6641128099243536,
+ "tokens_seen": 1039990784
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534363089267803,
+ "loss": 1.3435,
+ "theoretical_loss": 3.664091165432122,
+ "tokens_seen": 1040056320
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534202607823471,
+ "loss": 1.2943,
+ "theoretical_loss": 3.6640695226855646,
+ "tokens_seen": 1040121856
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005534042126379137,
+ "loss": 1.4396,
+ "theoretical_loss": 3.6640478816844286,
+ "tokens_seen": 1040187392
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005533881644934805,
+ "loss": 1.3399,
+ "theoretical_loss": 3.6640262424284646,
+ "tokens_seen": 1040252928
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005533721163490472,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6640046049174213,
+ "tokens_seen": 1040318464
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005533560682046139,
+ "loss": 1.3841,
+ "theoretical_loss": 3.663982969151049,
+ "tokens_seen": 1040384000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005533400200601806,
+ "loss": 1.3785,
+ "theoretical_loss": 3.6639613351290956,
+ "tokens_seen": 1040449536
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005533239719157473,
+ "loss": 1.4335,
+ "theoretical_loss": 3.6639397028513123,
+ "tokens_seen": 1040515072
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000553307923771314,
+ "loss": 1.2857,
+ "theoretical_loss": 3.663918072317448,
+ "tokens_seen": 1040580608
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005532918756268806,
+ "loss": 1.2753,
+ "theoretical_loss": 3.663896443527251,
+ "tokens_seen": 1040646144
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005532758274824474,
+ "loss": 1.3353,
+ "theoretical_loss": 3.663874816480473,
+ "tokens_seen": 1040711680
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000553259779338014,
+ "loss": 1.3405,
+ "theoretical_loss": 3.6638531911768624,
+ "tokens_seen": 1040777216
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005532437311935808,
+ "loss": 1.3946,
+ "theoretical_loss": 3.6638315676161697,
+ "tokens_seen": 1040842752
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005532276830491474,
+ "loss": 1.3927,
+ "theoretical_loss": 3.6638099457981443,
+ "tokens_seen": 1040908288
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005532116349047142,
+ "loss": 1.3701,
+ "theoretical_loss": 3.6637883257225354,
+ "tokens_seen": 1040973824
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005531955867602808,
+ "loss": 1.3353,
+ "theoretical_loss": 3.6637667073890943,
+ "tokens_seen": 1041039360
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005531795386158476,
+ "loss": 1.3539,
+ "theoretical_loss": 3.66374509079757,
+ "tokens_seen": 1041104896
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005531634904714142,
+ "loss": 1.3463,
+ "theoretical_loss": 3.6637234759477124,
+ "tokens_seen": 1041170432
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000553147442326981,
+ "loss": 1.4955,
+ "theoretical_loss": 3.6637018628392726,
+ "tokens_seen": 1041235968
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005531313941825476,
+ "loss": 1.2463,
+ "theoretical_loss": 3.663680251471999,
+ "tokens_seen": 1041301504
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005531153460381144,
+ "loss": 1.3755,
+ "theoretical_loss": 3.6636586418456436,
+ "tokens_seen": 1041367040
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000553099297893681,
+ "loss": 1.4584,
+ "theoretical_loss": 3.6636370339599553,
+ "tokens_seen": 1041432576
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530832497492478,
+ "loss": 1.3345,
+ "theoretical_loss": 3.663615427814685,
+ "tokens_seen": 1041498112
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530672016048144,
+ "loss": 1.3675,
+ "theoretical_loss": 3.663593823409583,
+ "tokens_seen": 1041563648
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530511534603812,
+ "loss": 1.327,
+ "theoretical_loss": 3.6635722207444,
+ "tokens_seen": 1041629184
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530351053159478,
+ "loss": 1.4008,
+ "theoretical_loss": 3.663550619818885,
+ "tokens_seen": 1041694720
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530190571715146,
+ "loss": 1.3456,
+ "theoretical_loss": 3.6635290206327897,
+ "tokens_seen": 1041760256
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005530030090270812,
+ "loss": 1.4695,
+ "theoretical_loss": 3.663507423185864,
+ "tokens_seen": 1041825792
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552986960882648,
+ "loss": 1.2368,
+ "theoretical_loss": 3.6634858274778597,
+ "tokens_seen": 1041891328
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005529709127382146,
+ "loss": 1.3337,
+ "theoretical_loss": 3.6634642335085257,
+ "tokens_seen": 1041956864
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 351434,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3981451988220215,
+ "objective/train/theoretical_loss": 3.663442641277614,
+ "objective/train/tokens_used": 1062482400,
+ "theoretical_loss": 3.663442641277614,
+ "tokens_seen": 1042022400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005529548645937814,
+ "loss": 1.404,
+ "theoretical_loss": 3.663442641277614,
+ "tokens_seen": 1042022400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552938816449348,
+ "loss": 1.2685,
+ "theoretical_loss": 3.6634210507848746,
+ "tokens_seen": 1042087936
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005529227683049148,
+ "loss": 1.2916,
+ "theoretical_loss": 3.6633994620300587,
+ "tokens_seen": 1042153472
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005529067201604814,
+ "loss": 1.3839,
+ "theoretical_loss": 3.663377875012917,
+ "tokens_seen": 1042219008
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005528906720160482,
+ "loss": 1.2788,
+ "theoretical_loss": 3.6633562897331995,
+ "tokens_seen": 1042284544
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005528746238716148,
+ "loss": 1.3901,
+ "theoretical_loss": 3.6633347061906587,
+ "tokens_seen": 1042350080
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005528585757271816,
+ "loss": 1.3639,
+ "theoretical_loss": 3.663313124385045,
+ "tokens_seen": 1042415616
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005528425275827482,
+ "loss": 1.4525,
+ "theoretical_loss": 3.6632915443161087,
+ "tokens_seen": 1042481152
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552826479438315,
+ "loss": 1.3389,
+ "theoretical_loss": 3.663269965983601,
+ "tokens_seen": 1042546688
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005528104312938816,
+ "loss": 1.2627,
+ "theoretical_loss": 3.6632483893872747,
+ "tokens_seen": 1042612224
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005527943831494484,
+ "loss": 1.267,
+ "theoretical_loss": 3.6632268145268796,
+ "tokens_seen": 1042677760
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552778335005015,
+ "loss": 1.3791,
+ "theoretical_loss": 3.6632052414021663,
+ "tokens_seen": 1042743296
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005527622868605818,
+ "loss": 1.2854,
+ "theoretical_loss": 3.6631836700128875,
+ "tokens_seen": 1042808832
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005527462387161484,
+ "loss": 1.3592,
+ "theoretical_loss": 3.663162100358794,
+ "tokens_seen": 1042874368
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005527301905717152,
+ "loss": 1.5055,
+ "theoretical_loss": 3.6631405324396367,
+ "tokens_seen": 1042939904
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005527141424272818,
+ "loss": 1.3553,
+ "theoretical_loss": 3.663118966255168,
+ "tokens_seen": 1043005440
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005526980942828486,
+ "loss": 1.4006,
+ "theoretical_loss": 3.6630974018051385,
+ "tokens_seen": 1043070976
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005526820461384152,
+ "loss": 1.3105,
+ "theoretical_loss": 3.6630758390893003,
+ "tokens_seen": 1043136512
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552665997993982,
+ "loss": 1.2154,
+ "theoretical_loss": 3.663054278107405,
+ "tokens_seen": 1043202048
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005526499498495486,
+ "loss": 1.3248,
+ "theoretical_loss": 3.6630327188592036,
+ "tokens_seen": 1043267584
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005526339017051154,
+ "loss": 1.2735,
+ "theoretical_loss": 3.663011161344449,
+ "tokens_seen": 1043333120
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552617853560682,
+ "loss": 1.3892,
+ "theoretical_loss": 3.662989605562892,
+ "tokens_seen": 1043398656
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005526018054162488,
+ "loss": 1.3775,
+ "theoretical_loss": 3.6629680515142846,
+ "tokens_seen": 1043464192
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005525857572718154,
+ "loss": 1.4082,
+ "theoretical_loss": 3.6629464991983784,
+ "tokens_seen": 1043529728
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005525697091273822,
+ "loss": 1.413,
+ "theoretical_loss": 3.6629249486149256,
+ "tokens_seen": 1043595264
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005525536609829488,
+ "loss": 1.49,
+ "theoretical_loss": 3.662903399763678,
+ "tokens_seen": 1043660800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005525376128385156,
+ "loss": 1.2857,
+ "theoretical_loss": 3.6628818526443885,
+ "tokens_seen": 1043726336
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005525215646940823,
+ "loss": 1.3294,
+ "theoretical_loss": 3.6628603072568082,
+ "tokens_seen": 1043791872
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552505516549649,
+ "loss": 1.3684,
+ "theoretical_loss": 3.662838763600689,
+ "tokens_seen": 1043857408
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524894684052157,
+ "loss": 1.4218,
+ "theoretical_loss": 3.6628172216757844,
+ "tokens_seen": 1043922944
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524734202607824,
+ "loss": 1.4024,
+ "theoretical_loss": 3.662795681481845,
+ "tokens_seen": 1043988480
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524573721163491,
+ "loss": 1.5027,
+ "theoretical_loss": 3.662774143018624,
+ "tokens_seen": 1044054016
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524413239719158,
+ "loss": 1.3652,
+ "theoretical_loss": 3.6627526062858733,
+ "tokens_seen": 1044119552
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524252758274825,
+ "loss": 1.3955,
+ "theoretical_loss": 3.6627310712833454,
+ "tokens_seen": 1044185088
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005524092276830492,
+ "loss": 1.2968,
+ "theoretical_loss": 3.6627095380107932,
+ "tokens_seen": 1044250624
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005523931795386159,
+ "loss": 1.385,
+ "theoretical_loss": 3.6626880064679685,
+ "tokens_seen": 1044316160
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005523771313941826,
+ "loss": 1.424,
+ "theoretical_loss": 3.662666476654624,
+ "tokens_seen": 1044381696
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005523610832497493,
+ "loss": 1.3254,
+ "theoretical_loss": 3.6626449485705126,
+ "tokens_seen": 1044447232
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552345035105316,
+ "loss": 1.3258,
+ "theoretical_loss": 3.662623422215386,
+ "tokens_seen": 1044512768
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005523289869608827,
+ "loss": 1.3716,
+ "theoretical_loss": 3.6626018975889982,
+ "tokens_seen": 1044578304
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005523129388164494,
+ "loss": 1.2937,
+ "theoretical_loss": 3.6625803746911014,
+ "tokens_seen": 1044643840
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522968906720161,
+ "loss": 1.3599,
+ "theoretical_loss": 3.6625588535214475,
+ "tokens_seen": 1044709376
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522808425275828,
+ "loss": 1.4456,
+ "theoretical_loss": 3.6625373340797904,
+ "tokens_seen": 1044774912
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522647943831495,
+ "loss": 1.3919,
+ "theoretical_loss": 3.6625158163658824,
+ "tokens_seen": 1044840448
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522487462387162,
+ "loss": 1.3677,
+ "theoretical_loss": 3.6624943003794765,
+ "tokens_seen": 1044905984
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552232698094283,
+ "loss": 1.3915,
+ "theoretical_loss": 3.6624727861203263,
+ "tokens_seen": 1044971520
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522166499498496,
+ "loss": 1.3365,
+ "theoretical_loss": 3.6624512735881845,
+ "tokens_seen": 1045037056
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005522006018054164,
+ "loss": 1.3189,
+ "theoretical_loss": 3.6624297627828035,
+ "tokens_seen": 1045102592
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000552184553660983,
+ "loss": 1.4014,
+ "theoretical_loss": 3.662408253703937,
+ "tokens_seen": 1045168128
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005521685055165498,
+ "loss": 1.3043,
+ "theoretical_loss": 3.6623867463513387,
+ "tokens_seen": 1045233664
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 352278,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2615886926651,
+ "objective/train/theoretical_loss": 3.662365240724761,
+ "objective/train/tokens_used": 1065759200,
+ "theoretical_loss": 3.662365240724761,
+ "tokens_seen": 1045299200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005521524573721164,
+ "loss": 1.3378,
+ "theoretical_loss": 3.662365240724761,
+ "tokens_seen": 1045299200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005521364092276832,
+ "loss": 1.4455,
+ "theoretical_loss": 3.6623437368239573,
+ "tokens_seen": 1045364736
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005521203610832498,
+ "loss": 1.3465,
+ "theoretical_loss": 3.6623222346486806,
+ "tokens_seen": 1045430272
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005521043129388166,
+ "loss": 1.4097,
+ "theoretical_loss": 3.662300734198685,
+ "tokens_seen": 1045495808
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005520882647943832,
+ "loss": 1.4539,
+ "theoretical_loss": 3.6622792354737244,
+ "tokens_seen": 1045561344
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00055207221664995,
+ "loss": 1.3983,
+ "theoretical_loss": 3.662257738473551,
+ "tokens_seen": 1045626880
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005520561685055166,
+ "loss": 1.4315,
+ "theoretical_loss": 3.6622362431979187,
+ "tokens_seen": 1045692416
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005520401203610832,
+ "loss": 1.4431,
+ "theoretical_loss": 3.662214749646582,
+ "tokens_seen": 1045757952
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00055202407221665,
+ "loss": 1.3975,
+ "theoretical_loss": 3.6621932578192933,
+ "tokens_seen": 1045823488
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005520080240722167,
+ "loss": 1.4283,
+ "theoretical_loss": 3.6621717677158068,
+ "tokens_seen": 1045889024
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005519919759277834,
+ "loss": 1.2036,
+ "theoretical_loss": 3.6621502793358762,
+ "tokens_seen": 1045954560
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00055197592778335,
+ "loss": 1.2878,
+ "theoretical_loss": 3.6621287926792556,
+ "tokens_seen": 1046020096
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005519598796389168,
+ "loss": 1.4843,
+ "theoretical_loss": 3.6621073077456985,
+ "tokens_seen": 1046085632
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005519438314944835,
+ "loss": 1.3248,
+ "theoretical_loss": 3.662085824534959,
+ "tokens_seen": 1046151168
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005519277833500502,
+ "loss": 1.2949,
+ "theoretical_loss": 3.6620643430467905,
+ "tokens_seen": 1046216704
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005519117352056169,
+ "loss": 1.3293,
+ "theoretical_loss": 3.6620428632809476,
+ "tokens_seen": 1046282240
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518956870611835,
+ "loss": 1.3417,
+ "theoretical_loss": 3.6620213852371846,
+ "tokens_seen": 1046347776
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518796389167503,
+ "loss": 1.1959,
+ "theoretical_loss": 3.661999908915254,
+ "tokens_seen": 1046413312
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518635907723169,
+ "loss": 1.2977,
+ "theoretical_loss": 3.6619784343149124,
+ "tokens_seen": 1046478848
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518475426278837,
+ "loss": 1.2672,
+ "theoretical_loss": 3.6619569614359118,
+ "tokens_seen": 1046544384
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518314944834503,
+ "loss": 1.3203,
+ "theoretical_loss": 3.6619354902780072,
+ "tokens_seen": 1046609920
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005518154463390171,
+ "loss": 1.3206,
+ "theoretical_loss": 3.6619140208409533,
+ "tokens_seen": 1046675456
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517993981945837,
+ "loss": 1.3258,
+ "theoretical_loss": 3.6618925531245043,
+ "tokens_seen": 1046740992
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517833500501505,
+ "loss": 1.2913,
+ "theoretical_loss": 3.6618710871284144,
+ "tokens_seen": 1046806528
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517673019057171,
+ "loss": 1.3044,
+ "theoretical_loss": 3.661849622852438,
+ "tokens_seen": 1046872064
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517512537612839,
+ "loss": 1.3536,
+ "theoretical_loss": 3.6618281602963294,
+ "tokens_seen": 1046937600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517352056168505,
+ "loss": 1.3636,
+ "theoretical_loss": 3.6618066994598437,
+ "tokens_seen": 1047003136
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517191574724173,
+ "loss": 1.204,
+ "theoretical_loss": 3.661785240342735,
+ "tokens_seen": 1047068672
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005517031093279839,
+ "loss": 1.4292,
+ "theoretical_loss": 3.6617637829447585,
+ "tokens_seen": 1047134208
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516870611835507,
+ "loss": 1.4248,
+ "theoretical_loss": 3.661742327265668,
+ "tokens_seen": 1047199744
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516710130391173,
+ "loss": 1.2418,
+ "theoretical_loss": 3.6617208733052196,
+ "tokens_seen": 1047265280
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516549648946841,
+ "loss": 1.3601,
+ "theoretical_loss": 3.6616994210631666,
+ "tokens_seen": 1047330816
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516389167502507,
+ "loss": 1.3372,
+ "theoretical_loss": 3.661677970539265,
+ "tokens_seen": 1047396352
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516228686058175,
+ "loss": 1.2883,
+ "theoretical_loss": 3.661656521733269,
+ "tokens_seen": 1047461888
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005516068204613841,
+ "loss": 1.3536,
+ "theoretical_loss": 3.6616350746449338,
+ "tokens_seen": 1047527424
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515907723169509,
+ "loss": 1.4302,
+ "theoretical_loss": 3.6616136292740142,
+ "tokens_seen": 1047592960
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515747241725175,
+ "loss": 1.3126,
+ "theoretical_loss": 3.6615921856202656,
+ "tokens_seen": 1047658496
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515586760280843,
+ "loss": 1.371,
+ "theoretical_loss": 3.661570743683443,
+ "tokens_seen": 1047724032
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515426278836509,
+ "loss": 1.3739,
+ "theoretical_loss": 3.6615493034633015,
+ "tokens_seen": 1047789568
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515265797392177,
+ "loss": 1.3689,
+ "theoretical_loss": 3.6615278649595964,
+ "tokens_seen": 1047855104
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005515105315947843,
+ "loss": 1.303,
+ "theoretical_loss": 3.661506428172083,
+ "tokens_seen": 1047920640
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514944834503511,
+ "loss": 1.3732,
+ "theoretical_loss": 3.661484993100516,
+ "tokens_seen": 1047986176
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514784353059177,
+ "loss": 1.3382,
+ "theoretical_loss": 3.661463559744651,
+ "tokens_seen": 1048051712
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514623871614845,
+ "loss": 1.2,
+ "theoretical_loss": 3.6614421281042437,
+ "tokens_seen": 1048117248
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514463390170511,
+ "loss": 1.2869,
+ "theoretical_loss": 3.66142069817905,
+ "tokens_seen": 1048182784
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514302908726179,
+ "loss": 1.3309,
+ "theoretical_loss": 3.6613992699688245,
+ "tokens_seen": 1048248320
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005514142427281845,
+ "loss": 1.3837,
+ "theoretical_loss": 3.6613778434733226,
+ "tokens_seen": 1048313856
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513981945837513,
+ "loss": 1.4049,
+ "theoretical_loss": 3.661356418692301,
+ "tokens_seen": 1048379392
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513821464393179,
+ "loss": 1.3956,
+ "theoretical_loss": 3.6613349956255146,
+ "tokens_seen": 1048444928
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513660982948847,
+ "loss": 1.4081,
+ "theoretical_loss": 3.661313574272719,
+ "tokens_seen": 1048510464
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 353441,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3471620082855225,
+ "objective/train/theoretical_loss": 3.6612921546336703,
+ "objective/train/tokens_used": 1069036000,
+ "theoretical_loss": 3.6612921546336703,
+ "tokens_seen": 1048576000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513500501504514,
+ "loss": 1.3995,
+ "theoretical_loss": 3.6612921546336703,
+ "tokens_seen": 1048576000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513340020060181,
+ "loss": 1.3516,
+ "theoretical_loss": 3.6612707367081243,
+ "tokens_seen": 1048641536
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513179538615848,
+ "loss": 1.3101,
+ "theoretical_loss": 3.6612493204958367,
+ "tokens_seen": 1048707072
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005513019057171515,
+ "loss": 1.305,
+ "theoretical_loss": 3.6612279059965633,
+ "tokens_seen": 1048772608
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005512858575727182,
+ "loss": 1.2781,
+ "theoretical_loss": 3.6612064932100603,
+ "tokens_seen": 1048838144
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005512698094282849,
+ "loss": 1.3912,
+ "theoretical_loss": 3.6611850821360834,
+ "tokens_seen": 1048903680
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005512537612838516,
+ "loss": 1.3579,
+ "theoretical_loss": 3.6611636727743893,
+ "tokens_seen": 1048969216
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005512377131394183,
+ "loss": 1.3985,
+ "theoretical_loss": 3.661142265124733,
+ "tokens_seen": 1049034752
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000551221664994985,
+ "loss": 1.2568,
+ "theoretical_loss": 3.661120859186872,
+ "tokens_seen": 1049100288
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005512056168505517,
+ "loss": 1.2912,
+ "theoretical_loss": 3.6610994549605613,
+ "tokens_seen": 1049165824
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511895687061184,
+ "loss": 1.3483,
+ "theoretical_loss": 3.661078052445558,
+ "tokens_seen": 1049231360
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511735205616851,
+ "loss": 1.4482,
+ "theoretical_loss": 3.6610566516416174,
+ "tokens_seen": 1049296896
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511574724172518,
+ "loss": 1.1986,
+ "theoretical_loss": 3.661035252548497,
+ "tokens_seen": 1049362432
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511414242728185,
+ "loss": 1.3118,
+ "theoretical_loss": 3.661013855165953,
+ "tokens_seen": 1049427968
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511253761283852,
+ "loss": 1.3358,
+ "theoretical_loss": 3.660992459493741,
+ "tokens_seen": 1049493504
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005511093279839519,
+ "loss": 1.4138,
+ "theoretical_loss": 3.6609710655316183,
+ "tokens_seen": 1049559040
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005510932798395186,
+ "loss": 1.2601,
+ "theoretical_loss": 3.660949673279341,
+ "tokens_seen": 1049624576
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005510772316950853,
+ "loss": 1.2612,
+ "theoretical_loss": 3.6609282827366663,
+ "tokens_seen": 1049690112
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000551061183550652,
+ "loss": 1.373,
+ "theoretical_loss": 3.66090689390335,
+ "tokens_seen": 1049755648
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005510451354062187,
+ "loss": 1.2842,
+ "theoretical_loss": 3.660885506779149,
+ "tokens_seen": 1049821184
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005510290872617854,
+ "loss": 1.3445,
+ "theoretical_loss": 3.660864121363821,
+ "tokens_seen": 1049886720
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005510130391173521,
+ "loss": 1.2281,
+ "theoretical_loss": 3.6608427376571218,
+ "tokens_seen": 1049952256
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509969909729188,
+ "loss": 1.2281,
+ "theoretical_loss": 3.6608213556588085,
+ "tokens_seen": 1050017792
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509809428284855,
+ "loss": 1.377,
+ "theoretical_loss": 3.660799975368638,
+ "tokens_seen": 1050083328
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509648946840522,
+ "loss": 1.4085,
+ "theoretical_loss": 3.6607785967863675,
+ "tokens_seen": 1050148864
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509488465396189,
+ "loss": 1.3558,
+ "theoretical_loss": 3.6607572199117535,
+ "tokens_seen": 1050214400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509327983951856,
+ "loss": 1.3707,
+ "theoretical_loss": 3.6607358447445533,
+ "tokens_seen": 1050279936
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005509167502507523,
+ "loss": 1.3686,
+ "theoretical_loss": 3.6607144712845243,
+ "tokens_seen": 1050345472
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000550900702106319,
+ "loss": 1.3544,
+ "theoretical_loss": 3.6606930995314233,
+ "tokens_seen": 1050411008
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508846539618857,
+ "loss": 1.2874,
+ "theoretical_loss": 3.6606717294850077,
+ "tokens_seen": 1050476544
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508686058174524,
+ "loss": 1.3522,
+ "theoretical_loss": 3.6606503611450343,
+ "tokens_seen": 1050542080
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508525576730191,
+ "loss": 1.4388,
+ "theoretical_loss": 3.6606289945112613,
+ "tokens_seen": 1050607616
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508365095285859,
+ "loss": 1.3087,
+ "theoretical_loss": 3.660607629583445,
+ "tokens_seen": 1050673152
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508204613841525,
+ "loss": 1.2838,
+ "theoretical_loss": 3.6605862663613435,
+ "tokens_seen": 1050738688
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005508044132397193,
+ "loss": 1.3144,
+ "theoretical_loss": 3.6605649048447138,
+ "tokens_seen": 1050804224
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507883650952859,
+ "loss": 1.3996,
+ "theoretical_loss": 3.660543545033314,
+ "tokens_seen": 1050869760
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507723169508527,
+ "loss": 1.2333,
+ "theoretical_loss": 3.660522186926901,
+ "tokens_seen": 1050935296
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507562688064193,
+ "loss": 1.3352,
+ "theoretical_loss": 3.6605008305252325,
+ "tokens_seen": 1051000832
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507402206619861,
+ "loss": 1.2704,
+ "theoretical_loss": 3.6604794758280663,
+ "tokens_seen": 1051066368
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507241725175527,
+ "loss": 1.3452,
+ "theoretical_loss": 3.66045812283516,
+ "tokens_seen": 1051131904
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005507081243731195,
+ "loss": 1.2633,
+ "theoretical_loss": 3.660436771546272,
+ "tokens_seen": 1051197440
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506920762286861,
+ "loss": 1.313,
+ "theoretical_loss": 3.660415421961159,
+ "tokens_seen": 1051262976
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506760280842529,
+ "loss": 1.2397,
+ "theoretical_loss": 3.6603940740795795,
+ "tokens_seen": 1051328512
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506599799398195,
+ "loss": 1.3666,
+ "theoretical_loss": 3.660372727901291,
+ "tokens_seen": 1051394048
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506439317953861,
+ "loss": 1.2966,
+ "theoretical_loss": 3.660351383426052,
+ "tokens_seen": 1051459584
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506278836509529,
+ "loss": 1.3406,
+ "theoretical_loss": 3.6603300406536206,
+ "tokens_seen": 1051525120
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005506118355065196,
+ "loss": 1.3207,
+ "theoretical_loss": 3.660308699583754,
+ "tokens_seen": 1051590656
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005505957873620863,
+ "loss": 1.243,
+ "theoretical_loss": 3.660287360216211,
+ "tokens_seen": 1051656192
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000550579739217653,
+ "loss": 1.2685,
+ "theoretical_loss": 3.660266022550749,
+ "tokens_seen": 1051721728
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005505636910732197,
+ "loss": 1.3218,
+ "theoretical_loss": 3.6602446865871263,
+ "tokens_seen": 1051787264
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 354381,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3571959733963013,
+ "objective/train/theoretical_loss": 3.660223352325102,
+ "objective/train/tokens_used": 1072312800,
+ "theoretical_loss": 3.660223352325102,
+ "tokens_seen": 1051852800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005505476429287864,
+ "loss": 1.4772,
+ "theoretical_loss": 3.660223352325102,
+ "tokens_seen": 1051852800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005505315947843531,
+ "loss": 1.4361,
+ "theoretical_loss": 3.660202019764434,
+ "tokens_seen": 1051918336
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005505155466399198,
+ "loss": 1.2858,
+ "theoretical_loss": 3.6601806889048802,
+ "tokens_seen": 1051983872
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504994984954864,
+ "loss": 1.3927,
+ "theoretical_loss": 3.6601593597461997,
+ "tokens_seen": 1052049408
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504834503510532,
+ "loss": 1.3095,
+ "theoretical_loss": 3.6601380322881503,
+ "tokens_seen": 1052114944
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504674022066198,
+ "loss": 1.3242,
+ "theoretical_loss": 3.6601167065304905,
+ "tokens_seen": 1052180480
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504513540621866,
+ "loss": 1.4475,
+ "theoretical_loss": 3.6600953824729796,
+ "tokens_seen": 1052246016
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504353059177532,
+ "loss": 1.3673,
+ "theoretical_loss": 3.660074060115375,
+ "tokens_seen": 1052311552
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00055041925777332,
+ "loss": 1.2225,
+ "theoretical_loss": 3.660052739457436,
+ "tokens_seen": 1052377088
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005504032096288866,
+ "loss": 1.2726,
+ "theoretical_loss": 3.660031420498922,
+ "tokens_seen": 1052442624
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005503871614844534,
+ "loss": 1.3033,
+ "theoretical_loss": 3.66001010323959,
+ "tokens_seen": 1052508160
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00055037111334002,
+ "loss": 1.3181,
+ "theoretical_loss": 3.6599887876792003,
+ "tokens_seen": 1052573696
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005503550651955868,
+ "loss": 1.3174,
+ "theoretical_loss": 3.6599674738175114,
+ "tokens_seen": 1052639232
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005503390170511534,
+ "loss": 1.2422,
+ "theoretical_loss": 3.6599461616542817,
+ "tokens_seen": 1052704768
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005503229689067202,
+ "loss": 1.2816,
+ "theoretical_loss": 3.6599248511892704,
+ "tokens_seen": 1052770304
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005503069207622868,
+ "loss": 1.4043,
+ "theoretical_loss": 3.659903542422237,
+ "tokens_seen": 1052835840
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005502908726178536,
+ "loss": 1.295,
+ "theoretical_loss": 3.6598822353529386,
+ "tokens_seen": 1052901376
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005502748244734202,
+ "loss": 1.3358,
+ "theoretical_loss": 3.6598609299811367,
+ "tokens_seen": 1052966912
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000550258776328987,
+ "loss": 1.3238,
+ "theoretical_loss": 3.6598396263065895,
+ "tokens_seen": 1053032448
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005502427281845536,
+ "loss": 1.2124,
+ "theoretical_loss": 3.6598183243290556,
+ "tokens_seen": 1053097984
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005502266800401204,
+ "loss": 1.3889,
+ "theoretical_loss": 3.659797024048295,
+ "tokens_seen": 1053163520
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000550210631895687,
+ "loss": 1.411,
+ "theoretical_loss": 3.6597757254640664,
+ "tokens_seen": 1053229056
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501945837512538,
+ "loss": 1.2472,
+ "theoretical_loss": 3.6597544285761296,
+ "tokens_seen": 1053294592
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501785356068204,
+ "loss": 1.3524,
+ "theoretical_loss": 3.6597331333842438,
+ "tokens_seen": 1053360128
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501624874623872,
+ "loss": 1.3964,
+ "theoretical_loss": 3.6597118398881685,
+ "tokens_seen": 1053425664
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501464393179539,
+ "loss": 1.3295,
+ "theoretical_loss": 3.6596905480876627,
+ "tokens_seen": 1053491200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501303911735206,
+ "loss": 1.2603,
+ "theoretical_loss": 3.6596692579824865,
+ "tokens_seen": 1053556736
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005501143430290873,
+ "loss": 1.2604,
+ "theoretical_loss": 3.659647969572399,
+ "tokens_seen": 1053622272
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000550098294884654,
+ "loss": 1.4479,
+ "theoretical_loss": 3.6596266828571604,
+ "tokens_seen": 1053687808
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500822467402207,
+ "loss": 1.3373,
+ "theoretical_loss": 3.65960539783653,
+ "tokens_seen": 1053753344
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500661985957874,
+ "loss": 1.4261,
+ "theoretical_loss": 3.659584114510267,
+ "tokens_seen": 1053818880
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500501504513541,
+ "loss": 1.3478,
+ "theoretical_loss": 3.6595628328781324,
+ "tokens_seen": 1053884416
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500341023069208,
+ "loss": 1.3394,
+ "theoretical_loss": 3.659541552939885,
+ "tokens_seen": 1053949952
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500180541624875,
+ "loss": 1.3559,
+ "theoretical_loss": 3.659520274695285,
+ "tokens_seen": 1054015488
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005500020060180542,
+ "loss": 1.5336,
+ "theoretical_loss": 3.659498998144092,
+ "tokens_seen": 1054081024
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005499859578736209,
+ "loss": 1.3408,
+ "theoretical_loss": 3.6594777232860665,
+ "tokens_seen": 1054146560
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005499699097291876,
+ "loss": 1.3231,
+ "theoretical_loss": 3.6594564501209685,
+ "tokens_seen": 1054212096
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005499538615847543,
+ "loss": 1.3286,
+ "theoretical_loss": 3.6594351786485575,
+ "tokens_seen": 1054277632
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549937813440321,
+ "loss": 1.2911,
+ "theoretical_loss": 3.659413908868594,
+ "tokens_seen": 1054343168
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005499217652958877,
+ "loss": 1.3796,
+ "theoretical_loss": 3.6593926407808377,
+ "tokens_seen": 1054408704
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005499057171514544,
+ "loss": 1.4812,
+ "theoretical_loss": 3.65937137438505,
+ "tokens_seen": 1054474240
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498896690070211,
+ "loss": 1.3864,
+ "theoretical_loss": 3.6593501096809895,
+ "tokens_seen": 1054539776
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498736208625878,
+ "loss": 1.3371,
+ "theoretical_loss": 3.6593288466684175,
+ "tokens_seen": 1054605312
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498575727181545,
+ "loss": 1.2628,
+ "theoretical_loss": 3.6593075853470944,
+ "tokens_seen": 1054670848
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498415245737212,
+ "loss": 1.3075,
+ "theoretical_loss": 3.6592863257167805,
+ "tokens_seen": 1054736384
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498254764292879,
+ "loss": 1.2905,
+ "theoretical_loss": 3.659265067777236,
+ "tokens_seen": 1054801920
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005498094282848546,
+ "loss": 1.3658,
+ "theoretical_loss": 3.6592438115282215,
+ "tokens_seen": 1054867456
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005497933801404213,
+ "loss": 1.4136,
+ "theoretical_loss": 3.6592225569694974,
+ "tokens_seen": 1054932992
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549777331995988,
+ "loss": 1.3177,
+ "theoretical_loss": 3.659201304100825,
+ "tokens_seen": 1054998528
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005497612838515547,
+ "loss": 1.3653,
+ "theoretical_loss": 3.659180052921964,
+ "tokens_seen": 1055064064
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 355696,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4703857898712158,
+ "objective/train/theoretical_loss": 3.659158803432675,
+ "objective/train/tokens_used": 1075589600,
+ "theoretical_loss": 3.659158803432675,
+ "tokens_seen": 1055129600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005497452357071214,
+ "loss": 1.3876,
+ "theoretical_loss": 3.659158803432675,
+ "tokens_seen": 1055129600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005497291875626881,
+ "loss": 1.3574,
+ "theoretical_loss": 3.6591375556327197,
+ "tokens_seen": 1055195136
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005497131394182548,
+ "loss": 1.3676,
+ "theoretical_loss": 3.659116309521859,
+ "tokens_seen": 1055260672
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496970912738215,
+ "loss": 1.4873,
+ "theoretical_loss": 3.6590950650998524,
+ "tokens_seen": 1055326208
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496810431293882,
+ "loss": 1.4917,
+ "theoretical_loss": 3.6590738223664614,
+ "tokens_seen": 1055391744
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496649949849549,
+ "loss": 1.4306,
+ "theoretical_loss": 3.659052581321448,
+ "tokens_seen": 1055457280
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496489468405216,
+ "loss": 1.3966,
+ "theoretical_loss": 3.6590313419645715,
+ "tokens_seen": 1055522816
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496328986960883,
+ "loss": 1.3749,
+ "theoretical_loss": 3.659010104295594,
+ "tokens_seen": 1055588352
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549616850551655,
+ "loss": 1.3479,
+ "theoretical_loss": 3.6589888683142764,
+ "tokens_seen": 1055653888
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005496008024072217,
+ "loss": 1.4363,
+ "theoretical_loss": 3.6589676340203794,
+ "tokens_seen": 1055719424
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495847542627884,
+ "loss": 1.3794,
+ "theoretical_loss": 3.6589464014136643,
+ "tokens_seen": 1055784960
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495687061183551,
+ "loss": 1.2471,
+ "theoretical_loss": 3.6589251704938928,
+ "tokens_seen": 1055850496
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495526579739218,
+ "loss": 1.3435,
+ "theoretical_loss": 3.658903941260826,
+ "tokens_seen": 1055916032
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495526579739218,
+ "loss": 1.4349,
+ "theoretical_loss": 3.6588827137142252,
+ "tokens_seen": 1055981568
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495526579739218,
+ "loss": 1.3724,
+ "theoretical_loss": 3.6588614878538515,
+ "tokens_seen": 1056047104
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495366098294885,
+ "loss": 1.2603,
+ "theoretical_loss": 3.658840263679467,
+ "tokens_seen": 1056112640
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005495205616850552,
+ "loss": 1.3944,
+ "theoretical_loss": 3.6588190411908323,
+ "tokens_seen": 1056178176
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549504513540622,
+ "loss": 1.5014,
+ "theoretical_loss": 3.658797820387709,
+ "tokens_seen": 1056243712
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005494884653961886,
+ "loss": 1.4993,
+ "theoretical_loss": 3.6587766012698593,
+ "tokens_seen": 1056309248
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005494724172517554,
+ "loss": 1.2745,
+ "theoretical_loss": 3.658755383837045,
+ "tokens_seen": 1056374784
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549456369107322,
+ "loss": 1.4437,
+ "theoretical_loss": 3.658734168089026,
+ "tokens_seen": 1056440320
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005494403209628888,
+ "loss": 1.4329,
+ "theoretical_loss": 3.6587129540255665,
+ "tokens_seen": 1056505856
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005494242728184554,
+ "loss": 1.399,
+ "theoretical_loss": 3.6586917416464266,
+ "tokens_seen": 1056571392
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005494082246740222,
+ "loss": 1.3012,
+ "theoretical_loss": 3.658670530951368,
+ "tokens_seen": 1056636928
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005493921765295888,
+ "loss": 1.3914,
+ "theoretical_loss": 3.658649321940154,
+ "tokens_seen": 1056702464
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005493761283851556,
+ "loss": 1.2222,
+ "theoretical_loss": 3.6586281146125454,
+ "tokens_seen": 1056768000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005493600802407222,
+ "loss": 1.2534,
+ "theoretical_loss": 3.658606908968304,
+ "tokens_seen": 1056833536
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549344032096289,
+ "loss": 1.3097,
+ "theoretical_loss": 3.6585857050071926,
+ "tokens_seen": 1056899072
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005493279839518556,
+ "loss": 1.3939,
+ "theoretical_loss": 3.6585645027289724,
+ "tokens_seen": 1056964608
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005493119358074224,
+ "loss": 1.36,
+ "theoretical_loss": 3.658543302133406,
+ "tokens_seen": 1057030144
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549295887662989,
+ "loss": 1.3646,
+ "theoretical_loss": 3.658522103220256,
+ "tokens_seen": 1057095680
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005492798395185558,
+ "loss": 1.4481,
+ "theoretical_loss": 3.6585009059892832,
+ "tokens_seen": 1057161216
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005492637913741224,
+ "loss": 1.3917,
+ "theoretical_loss": 3.6584797104402513,
+ "tokens_seen": 1057226752
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549247743229689,
+ "loss": 1.4084,
+ "theoretical_loss": 3.6584585165729218,
+ "tokens_seen": 1057292288
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005492316950852558,
+ "loss": 1.4677,
+ "theoretical_loss": 3.6584373243870574,
+ "tokens_seen": 1057357824
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005492156469408225,
+ "loss": 1.3319,
+ "theoretical_loss": 3.6584161338824206,
+ "tokens_seen": 1057423360
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491995987963892,
+ "loss": 1.3983,
+ "theoretical_loss": 3.658394945058773,
+ "tokens_seen": 1057488896
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491835506519559,
+ "loss": 1.4069,
+ "theoretical_loss": 3.6583737579158777,
+ "tokens_seen": 1057554432
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491675025075226,
+ "loss": 1.348,
+ "theoretical_loss": 3.6583525724534978,
+ "tokens_seen": 1057619968
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491514543630893,
+ "loss": 1.3386,
+ "theoretical_loss": 3.6583313886713946,
+ "tokens_seen": 1057685504
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000549135406218656,
+ "loss": 1.2386,
+ "theoretical_loss": 3.6583102065693316,
+ "tokens_seen": 1057751040
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491193580742227,
+ "loss": 1.3445,
+ "theoretical_loss": 3.6582890261470715,
+ "tokens_seen": 1057816576
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005491033099297893,
+ "loss": 1.4633,
+ "theoretical_loss": 3.658267847404377,
+ "tokens_seen": 1057882112
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490872617853561,
+ "loss": 1.3658,
+ "theoretical_loss": 3.6582466703410104,
+ "tokens_seen": 1057947648
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490712136409227,
+ "loss": 1.3889,
+ "theoretical_loss": 3.658225494956735,
+ "tokens_seen": 1058013184
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490551654964895,
+ "loss": 1.3035,
+ "theoretical_loss": 3.658204321251313,
+ "tokens_seen": 1058078720
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490391173520561,
+ "loss": 1.3589,
+ "theoretical_loss": 3.6581831492245085,
+ "tokens_seen": 1058144256
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490230692076229,
+ "loss": 1.4039,
+ "theoretical_loss": 3.658161978876083,
+ "tokens_seen": 1058209792
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005490070210631895,
+ "loss": 1.2996,
+ "theoretical_loss": 3.6581408102058015,
+ "tokens_seen": 1058275328
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489909729187563,
+ "loss": 1.3559,
+ "theoretical_loss": 3.658119643213425,
+ "tokens_seen": 1058340864
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 356680,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2355961799621582,
+ "objective/train/theoretical_loss": 3.658098477898718,
+ "objective/train/tokens_used": 1078866400,
+ "theoretical_loss": 3.658098477898718,
+ "tokens_seen": 1058406400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489749247743229,
+ "loss": 1.2988,
+ "theoretical_loss": 3.658098477898718,
+ "tokens_seen": 1058406400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489588766298897,
+ "loss": 1.4177,
+ "theoretical_loss": 3.6580773142614422,
+ "tokens_seen": 1058471936
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489428284854564,
+ "loss": 1.3128,
+ "theoretical_loss": 3.658056152301363,
+ "tokens_seen": 1058537472
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489267803410231,
+ "loss": 1.3534,
+ "theoretical_loss": 3.6580349920182416,
+ "tokens_seen": 1058603008
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005489107321965898,
+ "loss": 1.3942,
+ "theoretical_loss": 3.658013833411842,
+ "tokens_seen": 1058668544
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005488946840521565,
+ "loss": 1.396,
+ "theoretical_loss": 3.6579926764819284,
+ "tokens_seen": 1058734080
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005488786359077232,
+ "loss": 1.4598,
+ "theoretical_loss": 3.657971521228263,
+ "tokens_seen": 1058799616
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005488625877632899,
+ "loss": 1.3345,
+ "theoretical_loss": 3.65795036765061,
+ "tokens_seen": 1058865152
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005488465396188566,
+ "loss": 1.5156,
+ "theoretical_loss": 3.6579292157487324,
+ "tokens_seen": 1058930688
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005488304914744233,
+ "loss": 1.3158,
+ "theoretical_loss": 3.6579080655223937,
+ "tokens_seen": 1058996224
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00054881444332999,
+ "loss": 1.3787,
+ "theoretical_loss": 3.657886916971359,
+ "tokens_seen": 1059061760
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487983951855567,
+ "loss": 1.4935,
+ "theoretical_loss": 3.6578657700953894,
+ "tokens_seen": 1059127296
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487823470411234,
+ "loss": 1.4037,
+ "theoretical_loss": 3.6578446248942504,
+ "tokens_seen": 1059192832
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487662988966901,
+ "loss": 1.2548,
+ "theoretical_loss": 3.6578234813677053,
+ "tokens_seen": 1059258368
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487502507522568,
+ "loss": 1.3403,
+ "theoretical_loss": 3.6578023395155173,
+ "tokens_seen": 1059323904
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487342026078235,
+ "loss": 1.3848,
+ "theoretical_loss": 3.6577811993374514,
+ "tokens_seen": 1059389440
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487181544633902,
+ "loss": 1.4138,
+ "theoretical_loss": 3.6577600608332705,
+ "tokens_seen": 1059454976
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005487021063189569,
+ "loss": 1.2685,
+ "theoretical_loss": 3.6577389240027394,
+ "tokens_seen": 1059520512
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005486860581745236,
+ "loss": 1.4151,
+ "theoretical_loss": 3.657717788845621,
+ "tokens_seen": 1059586048
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005486700100300903,
+ "loss": 1.4643,
+ "theoretical_loss": 3.65769665536168,
+ "tokens_seen": 1059651584
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000548653961885657,
+ "loss": 1.2092,
+ "theoretical_loss": 3.6576755235506804,
+ "tokens_seen": 1059717120
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005486379137412237,
+ "loss": 1.2998,
+ "theoretical_loss": 3.6576543934123866,
+ "tokens_seen": 1059782656
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005486218655967904,
+ "loss": 1.3523,
+ "theoretical_loss": 3.657633264946562,
+ "tokens_seen": 1059848192
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005486058174523571,
+ "loss": 1.3499,
+ "theoretical_loss": 3.6576121381529716,
+ "tokens_seen": 1059913728
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485897693079238,
+ "loss": 1.2299,
+ "theoretical_loss": 3.657591013031379,
+ "tokens_seen": 1059979264
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485737211634905,
+ "loss": 1.3489,
+ "theoretical_loss": 3.6575698895815494,
+ "tokens_seen": 1060044800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485576730190572,
+ "loss": 1.437,
+ "theoretical_loss": 3.657548767803246,
+ "tokens_seen": 1060110336
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485416248746239,
+ "loss": 1.3298,
+ "theoretical_loss": 3.6575276476962335,
+ "tokens_seen": 1060175872
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485255767301906,
+ "loss": 1.3867,
+ "theoretical_loss": 3.6575065292602775,
+ "tokens_seen": 1060241408
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005485095285857573,
+ "loss": 1.3321,
+ "theoretical_loss": 3.6574854124951415,
+ "tokens_seen": 1060306944
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000548493480441324,
+ "loss": 1.2883,
+ "theoretical_loss": 3.65746429740059,
+ "tokens_seen": 1060372480
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005484774322968907,
+ "loss": 1.2418,
+ "theoretical_loss": 3.6574431839763877,
+ "tokens_seen": 1060438016
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005484613841524574,
+ "loss": 1.3314,
+ "theoretical_loss": 3.6574220722222996,
+ "tokens_seen": 1060503552
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005484453360080241,
+ "loss": 1.4457,
+ "theoretical_loss": 3.65740096213809,
+ "tokens_seen": 1060569088
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005484292878635908,
+ "loss": 1.4252,
+ "theoretical_loss": 3.657379853723524,
+ "tokens_seen": 1060634624
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005484132397191575,
+ "loss": 1.2749,
+ "theoretical_loss": 3.657358746978366,
+ "tokens_seen": 1060700160
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483971915747242,
+ "loss": 1.3305,
+ "theoretical_loss": 3.6573376419023815,
+ "tokens_seen": 1060765696
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483811434302909,
+ "loss": 1.4029,
+ "theoretical_loss": 3.6573165384953343,
+ "tokens_seen": 1060831232
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483650952858576,
+ "loss": 1.3269,
+ "theoretical_loss": 3.6572954367569905,
+ "tokens_seen": 1060896768
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483490471414243,
+ "loss": 1.3708,
+ "theoretical_loss": 3.657274336687114,
+ "tokens_seen": 1060962304
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000548332998996991,
+ "loss": 1.3537,
+ "theoretical_loss": 3.6572532382854708,
+ "tokens_seen": 1061027840
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483169508525577,
+ "loss": 1.4125,
+ "theoretical_loss": 3.6572321415518254,
+ "tokens_seen": 1061093376
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005483009027081244,
+ "loss": 1.3706,
+ "theoretical_loss": 3.657211046485943,
+ "tokens_seen": 1061158912
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482848545636911,
+ "loss": 1.2991,
+ "theoretical_loss": 3.6571899530875887,
+ "tokens_seen": 1061224448
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482688064192578,
+ "loss": 1.2968,
+ "theoretical_loss": 3.657168861356528,
+ "tokens_seen": 1061289984
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482527582748245,
+ "loss": 1.3236,
+ "theoretical_loss": 3.657147771292526,
+ "tokens_seen": 1061355520
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482367101303912,
+ "loss": 1.3278,
+ "theoretical_loss": 3.657126682895348,
+ "tokens_seen": 1061421056
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482206619859579,
+ "loss": 1.4331,
+ "theoretical_loss": 3.65710559616476,
+ "tokens_seen": 1061486592
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005482046138415246,
+ "loss": 1.3522,
+ "theoretical_loss": 3.6570845111005257,
+ "tokens_seen": 1061552128
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005481885656970913,
+ "loss": 1.4438,
+ "theoretical_loss": 3.657063427702412,
+ "tokens_seen": 1061617664
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 357916,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2271572351455688,
+ "objective/train/theoretical_loss": 3.6570423459701846,
+ "objective/train/tokens_used": 1082143200,
+ "theoretical_loss": 3.6570423459701846,
+ "tokens_seen": 1061683200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000548172517552658,
+ "loss": 1.37,
+ "theoretical_loss": 3.6570423459701846,
+ "tokens_seen": 1061683200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005481564694082247,
+ "loss": 1.4331,
+ "theoretical_loss": 3.657021265903608,
+ "tokens_seen": 1061748736
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005481404212637914,
+ "loss": 1.3292,
+ "theoretical_loss": 3.6570001875024483,
+ "tokens_seen": 1061814272
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005481243731193581,
+ "loss": 1.2978,
+ "theoretical_loss": 3.6569791107664713,
+ "tokens_seen": 1061879808
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005481083249749249,
+ "loss": 1.3009,
+ "theoretical_loss": 3.6569580356954425,
+ "tokens_seen": 1061945344
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005480922768304915,
+ "loss": 1.4697,
+ "theoretical_loss": 3.6569369622891283,
+ "tokens_seen": 1062010880
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005480762286860583,
+ "loss": 1.3022,
+ "theoretical_loss": 3.6569158905472934,
+ "tokens_seen": 1062076416
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005480601805416249,
+ "loss": 1.3102,
+ "theoretical_loss": 3.656894820469704,
+ "tokens_seen": 1062141952
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005480441323971917,
+ "loss": 1.3952,
+ "theoretical_loss": 3.656873752056126,
+ "tokens_seen": 1062207488
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005480280842527583,
+ "loss": 1.4186,
+ "theoretical_loss": 3.6568526853063252,
+ "tokens_seen": 1062273024
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000548012036108325,
+ "loss": 1.3164,
+ "theoretical_loss": 3.6568316202200686,
+ "tokens_seen": 1062338560
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479959879638917,
+ "loss": 1.3217,
+ "theoretical_loss": 3.656810556797121,
+ "tokens_seen": 1062404096
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479799398194585,
+ "loss": 1.2744,
+ "theoretical_loss": 3.6567894950372493,
+ "tokens_seen": 1062469632
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479638916750251,
+ "loss": 1.2897,
+ "theoretical_loss": 3.6567684349402185,
+ "tokens_seen": 1062535168
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479478435305919,
+ "loss": 1.2689,
+ "theoretical_loss": 3.6567473765057956,
+ "tokens_seen": 1062600704
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479317953861585,
+ "loss": 1.4276,
+ "theoretical_loss": 3.656726319733747,
+ "tokens_seen": 1062666240
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005479157472417253,
+ "loss": 1.2482,
+ "theoretical_loss": 3.656705264623839,
+ "tokens_seen": 1062731776
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478996990972919,
+ "loss": 1.401,
+ "theoretical_loss": 3.656684211175837,
+ "tokens_seen": 1062797312
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478836509528587,
+ "loss": 1.2575,
+ "theoretical_loss": 3.6566631593895083,
+ "tokens_seen": 1062862848
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478676028084253,
+ "loss": 1.4389,
+ "theoretical_loss": 3.656642109264619,
+ "tokens_seen": 1062928384
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000547851554663992,
+ "loss": 1.4014,
+ "theoretical_loss": 3.6566210608009353,
+ "tokens_seen": 1062993920
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478355065195587,
+ "loss": 1.3772,
+ "theoretical_loss": 3.6566000139982235,
+ "tokens_seen": 1063059456
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478194583751254,
+ "loss": 1.3588,
+ "theoretical_loss": 3.656578968856251,
+ "tokens_seen": 1063124992
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005478034102306921,
+ "loss": 1.4303,
+ "theoretical_loss": 3.656557925374784,
+ "tokens_seen": 1063190528
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477873620862588,
+ "loss": 1.4474,
+ "theoretical_loss": 3.6565368835535885,
+ "tokens_seen": 1063256064
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477713139418255,
+ "loss": 1.2801,
+ "theoretical_loss": 3.656515843392432,
+ "tokens_seen": 1063321600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477552657973922,
+ "loss": 1.4602,
+ "theoretical_loss": 3.6564948048910813,
+ "tokens_seen": 1063387136
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477392176529589,
+ "loss": 1.2746,
+ "theoretical_loss": 3.656473768049303,
+ "tokens_seen": 1063452672
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477231695085256,
+ "loss": 1.301,
+ "theoretical_loss": 3.6564527328668635,
+ "tokens_seen": 1063518208
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005477071213640923,
+ "loss": 1.3755,
+ "theoretical_loss": 3.6564316993435297,
+ "tokens_seen": 1063583744
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000547691073219659,
+ "loss": 1.399,
+ "theoretical_loss": 3.656410667479069,
+ "tokens_seen": 1063649280
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005476750250752257,
+ "loss": 1.3038,
+ "theoretical_loss": 3.6563896372732483,
+ "tokens_seen": 1063714816
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005476589769307924,
+ "loss": 1.3797,
+ "theoretical_loss": 3.6563686087258347,
+ "tokens_seen": 1063780352
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005476429287863591,
+ "loss": 1.3812,
+ "theoretical_loss": 3.6563475818365943,
+ "tokens_seen": 1063845888
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005476268806419258,
+ "loss": 1.4176,
+ "theoretical_loss": 3.6563265566052956,
+ "tokens_seen": 1063911424
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005476108324974925,
+ "loss": 1.3154,
+ "theoretical_loss": 3.6563055330317047,
+ "tokens_seen": 1063976960
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005475947843530592,
+ "loss": 1.3561,
+ "theoretical_loss": 3.6562845111155893,
+ "tokens_seen": 1064042496
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005475787362086259,
+ "loss": 1.2732,
+ "theoretical_loss": 3.656263490856717,
+ "tokens_seen": 1064108032
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005475626880641926,
+ "loss": 1.3845,
+ "theoretical_loss": 3.6562424722548545,
+ "tokens_seen": 1064173568
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005475466399197593,
+ "loss": 1.3015,
+ "theoretical_loss": 3.656221455309769,
+ "tokens_seen": 1064239104
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000547530591775326,
+ "loss": 1.28,
+ "theoretical_loss": 3.6562004400212285,
+ "tokens_seen": 1064304640
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005475145436308927,
+ "loss": 1.3322,
+ "theoretical_loss": 3.6561794263890004,
+ "tokens_seen": 1064370176
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474984954864594,
+ "loss": 1.2637,
+ "theoretical_loss": 3.656158414412851,
+ "tokens_seen": 1064435712
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474824473420261,
+ "loss": 1.3027,
+ "theoretical_loss": 3.65613740409255,
+ "tokens_seen": 1064501248
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474663991975928,
+ "loss": 1.3667,
+ "theoretical_loss": 3.656116395427863,
+ "tokens_seen": 1064566784
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474503510531595,
+ "loss": 1.3948,
+ "theoretical_loss": 3.6560953884185583,
+ "tokens_seen": 1064632320
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474343029087262,
+ "loss": 1.3762,
+ "theoretical_loss": 3.6560743830644036,
+ "tokens_seen": 1064697856
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474182547642929,
+ "loss": 1.3312,
+ "theoretical_loss": 3.656053379365167,
+ "tokens_seen": 1064763392
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005474022066198596,
+ "loss": 1.3581,
+ "theoretical_loss": 3.656032377320616,
+ "tokens_seen": 1064828928
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005473861584754263,
+ "loss": 1.353,
+ "theoretical_loss": 3.6560113769305183,
+ "tokens_seen": 1064894464
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 359144,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4166908264160156,
+ "objective/train/theoretical_loss": 3.655990378194642,
+ "objective/train/tokens_used": 1085420000,
+ "theoretical_loss": 3.655990378194642,
+ "tokens_seen": 1064960000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000547370110330993,
+ "loss": 1.3862,
+ "theoretical_loss": 3.655990378194642,
+ "tokens_seen": 1064960000
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005473540621865597,
+ "loss": 1.4587,
+ "theoretical_loss": 3.655969381112754,
+ "tokens_seen": 1065025536
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005473380140421264,
+ "loss": 1.4051,
+ "theoretical_loss": 3.655948385684624,
+ "tokens_seen": 1065091072
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005473219658976931,
+ "loss": 1.3726,
+ "theoretical_loss": 3.6559273919100184,
+ "tokens_seen": 1065156608
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005473059177532598,
+ "loss": 1.3313,
+ "theoretical_loss": 3.6559063997887065,
+ "tokens_seen": 1065222144
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005472898696088265,
+ "loss": 1.3552,
+ "theoretical_loss": 3.6558854093204554,
+ "tokens_seen": 1065287680
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005472738214643932,
+ "loss": 1.3286,
+ "theoretical_loss": 3.655864420505034,
+ "tokens_seen": 1065353216
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005472577733199599,
+ "loss": 1.32,
+ "theoretical_loss": 3.65584343334221,
+ "tokens_seen": 1065418752
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005472417251755266,
+ "loss": 1.3726,
+ "theoretical_loss": 3.655822447831752,
+ "tokens_seen": 1065484288
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005472256770310933,
+ "loss": 1.4471,
+ "theoretical_loss": 3.655801463973428,
+ "tokens_seen": 1065549824
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00054720962888666,
+ "loss": 1.4042,
+ "theoretical_loss": 3.6557804817670063,
+ "tokens_seen": 1065615360
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471935807422267,
+ "loss": 1.3077,
+ "theoretical_loss": 3.6557595012122555,
+ "tokens_seen": 1065680896
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471775325977934,
+ "loss": 1.295,
+ "theoretical_loss": 3.655738522308944,
+ "tokens_seen": 1065746432
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471614844533601,
+ "loss": 1.3996,
+ "theoretical_loss": 3.65571754505684,
+ "tokens_seen": 1065811968
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471454363089268,
+ "loss": 1.2835,
+ "theoretical_loss": 3.6556965694557126,
+ "tokens_seen": 1065877504
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471293881644935,
+ "loss": 1.3876,
+ "theoretical_loss": 3.6556755955053295,
+ "tokens_seen": 1065943040
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005471133400200602,
+ "loss": 1.3348,
+ "theoretical_loss": 3.65565462320546,
+ "tokens_seen": 1066008576
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470972918756269,
+ "loss": 1.3571,
+ "theoretical_loss": 3.6556336525558724,
+ "tokens_seen": 1066074112
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470812437311936,
+ "loss": 1.2934,
+ "theoretical_loss": 3.6556126835563356,
+ "tokens_seen": 1066139648
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470651955867603,
+ "loss": 1.3368,
+ "theoretical_loss": 3.6555917162066187,
+ "tokens_seen": 1066205184
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000547049147442327,
+ "loss": 1.3447,
+ "theoretical_loss": 3.65557075050649,
+ "tokens_seen": 1066270720
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470330992978937,
+ "loss": 1.332,
+ "theoretical_loss": 3.6555497864557176,
+ "tokens_seen": 1066336256
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470170511534604,
+ "loss": 1.3861,
+ "theoretical_loss": 3.655528824054072,
+ "tokens_seen": 1066401792
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005470010030090271,
+ "loss": 1.2749,
+ "theoretical_loss": 3.655507863301321,
+ "tokens_seen": 1066467328
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469849548645938,
+ "loss": 1.2881,
+ "theoretical_loss": 3.655486904197234,
+ "tokens_seen": 1066532864
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469689067201605,
+ "loss": 1.3396,
+ "theoretical_loss": 3.65546594674158,
+ "tokens_seen": 1066598400
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469528585757272,
+ "loss": 1.3189,
+ "theoretical_loss": 3.6554449909341287,
+ "tokens_seen": 1066663936
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469368104312939,
+ "loss": 1.4198,
+ "theoretical_loss": 3.6554240367746473,
+ "tokens_seen": 1066729472
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469207622868606,
+ "loss": 1.2967,
+ "theoretical_loss": 3.6554030842629075,
+ "tokens_seen": 1066795008
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005469047141424273,
+ "loss": 1.2496,
+ "theoretical_loss": 3.6553821333986765,
+ "tokens_seen": 1066860544
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546888665997994,
+ "loss": 1.2873,
+ "theoretical_loss": 3.6553611841817246,
+ "tokens_seen": 1066926080
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005468726178535607,
+ "loss": 1.3809,
+ "theoretical_loss": 3.6553402366118206,
+ "tokens_seen": 1066991616
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005468565697091274,
+ "loss": 1.3995,
+ "theoretical_loss": 3.6553192906887344,
+ "tokens_seen": 1067057152
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005468405215646941,
+ "loss": 1.359,
+ "theoretical_loss": 3.655298346412235,
+ "tokens_seen": 1067122688
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005468244734202608,
+ "loss": 1.3591,
+ "theoretical_loss": 3.655277403782092,
+ "tokens_seen": 1067188224
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005468084252758275,
+ "loss": 1.4017,
+ "theoretical_loss": 3.6552564627980746,
+ "tokens_seen": 1067253760
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005467923771313942,
+ "loss": 1.3742,
+ "theoretical_loss": 3.655235523459953,
+ "tokens_seen": 1067319296
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546776328986961,
+ "loss": 1.3002,
+ "theoretical_loss": 3.655214585767496,
+ "tokens_seen": 1067384832
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005467602808425276,
+ "loss": 1.2625,
+ "theoretical_loss": 3.6551936497204736,
+ "tokens_seen": 1067450368
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005467442326980943,
+ "loss": 1.2739,
+ "theoretical_loss": 3.6551727153186553,
+ "tokens_seen": 1067515904
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546728184553661,
+ "loss": 1.2936,
+ "theoretical_loss": 3.6551517825618114,
+ "tokens_seen": 1067581440
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005467121364092278,
+ "loss": 1.4605,
+ "theoretical_loss": 3.6551308514497114,
+ "tokens_seen": 1067646976
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005466960882647944,
+ "loss": 1.3799,
+ "theoretical_loss": 3.6551099219821244,
+ "tokens_seen": 1067712512
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005466800401203612,
+ "loss": 1.3642,
+ "theoretical_loss": 3.655088994158821,
+ "tokens_seen": 1067778048
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005466639919759278,
+ "loss": 1.3985,
+ "theoretical_loss": 3.6550680679795713,
+ "tokens_seen": 1067843584
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005466479438314946,
+ "loss": 1.4229,
+ "theoretical_loss": 3.655047143444145,
+ "tokens_seen": 1067909120
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005466318956870612,
+ "loss": 1.3782,
+ "theoretical_loss": 3.655026220552312,
+ "tokens_seen": 1067974656
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546615847542628,
+ "loss": 1.338,
+ "theoretical_loss": 3.655005299303842,
+ "tokens_seen": 1068040192
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465997993981946,
+ "loss": 1.2879,
+ "theoretical_loss": 3.6549843796985053,
+ "tokens_seen": 1068105728
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465837512537614,
+ "loss": 1.3942,
+ "theoretical_loss": 3.654963461736073,
+ "tokens_seen": 1068171264
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 360072,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2868728637695312,
+ "objective/train/theoretical_loss": 3.654942545416314,
+ "objective/train/tokens_used": 1088696800,
+ "theoretical_loss": 3.654942545416314,
+ "tokens_seen": 1068236800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546567703109328,
+ "loss": 1.3204,
+ "theoretical_loss": 3.654942545416314,
+ "tokens_seen": 1068236800
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465516549648948,
+ "loss": 1.3386,
+ "theoretical_loss": 3.6549216307389996,
+ "tokens_seen": 1068302336
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465356068204614,
+ "loss": 1.4649,
+ "theoretical_loss": 3.6549007177038986,
+ "tokens_seen": 1068367872
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465195586760282,
+ "loss": 1.3254,
+ "theoretical_loss": 3.654879806310783,
+ "tokens_seen": 1068433408
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005465035105315948,
+ "loss": 1.416,
+ "theoretical_loss": 3.654858896559422,
+ "tokens_seen": 1068498944
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005464874623871616,
+ "loss": 1.3876,
+ "theoretical_loss": 3.654837988449587,
+ "tokens_seen": 1068564480
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005464714142427282,
+ "loss": 1.3284,
+ "theoretical_loss": 3.6548170819810477,
+ "tokens_seen": 1068630016
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546455366098295,
+ "loss": 1.3933,
+ "theoretical_loss": 3.6547961771535746,
+ "tokens_seen": 1068695552
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005464393179538616,
+ "loss": 1.3981,
+ "theoretical_loss": 3.654775273966939,
+ "tokens_seen": 1068761088
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005464232698094283,
+ "loss": 1.2899,
+ "theoretical_loss": 3.6547543724209106,
+ "tokens_seen": 1068826624
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546407221664995,
+ "loss": 1.3313,
+ "theoretical_loss": 3.654733472515261,
+ "tokens_seen": 1068892160
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463911735205617,
+ "loss": 1.3814,
+ "theoretical_loss": 3.6547125742497597,
+ "tokens_seen": 1068957696
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463751253761284,
+ "loss": 1.3791,
+ "theoretical_loss": 3.6546916776241787,
+ "tokens_seen": 1069023232
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463590772316951,
+ "loss": 1.3717,
+ "theoretical_loss": 3.654670782638288,
+ "tokens_seen": 1069088768
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463430290872618,
+ "loss": 1.4064,
+ "theoretical_loss": 3.654649889291858,
+ "tokens_seen": 1069154304
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463269809428285,
+ "loss": 1.3779,
+ "theoretical_loss": 3.654628997584661,
+ "tokens_seen": 1069219840
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005463109327983952,
+ "loss": 1.4312,
+ "theoretical_loss": 3.6546081075164665,
+ "tokens_seen": 1069285376
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005462948846539619,
+ "loss": 1.3432,
+ "theoretical_loss": 3.6545872190870465,
+ "tokens_seen": 1069350912
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005462788365095286,
+ "loss": 1.3097,
+ "theoretical_loss": 3.6545663322961714,
+ "tokens_seen": 1069416448
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005462627883650953,
+ "loss": 1.5095,
+ "theoretical_loss": 3.6545454471436125,
+ "tokens_seen": 1069481984
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546246740220662,
+ "loss": 1.4114,
+ "theoretical_loss": 3.654524563629141,
+ "tokens_seen": 1069547520
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005462306920762287,
+ "loss": 1.4444,
+ "theoretical_loss": 3.6545036817525283,
+ "tokens_seen": 1069613056
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005462146439317954,
+ "loss": 1.3725,
+ "theoretical_loss": 3.654482801513545,
+ "tokens_seen": 1069678592
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461985957873621,
+ "loss": 1.3284,
+ "theoretical_loss": 3.654461922911963,
+ "tokens_seen": 1069744128
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461825476429288,
+ "loss": 1.4145,
+ "theoretical_loss": 3.6544410459475527,
+ "tokens_seen": 1069809664
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461664994984955,
+ "loss": 1.233,
+ "theoretical_loss": 3.654420170620086,
+ "tokens_seen": 1069875200
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461504513540622,
+ "loss": 1.242,
+ "theoretical_loss": 3.654399296929335,
+ "tokens_seen": 1069940736
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461344032096289,
+ "loss": 1.352,
+ "theoretical_loss": 3.6543784248750697,
+ "tokens_seen": 1070006272
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461183550651956,
+ "loss": 1.3607,
+ "theoretical_loss": 3.6543575544570626,
+ "tokens_seen": 1070071808
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005461023069207623,
+ "loss": 1.4777,
+ "theoretical_loss": 3.6543366856750845,
+ "tokens_seen": 1070137344
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000546086258776329,
+ "loss": 1.2618,
+ "theoretical_loss": 3.6543158185289077,
+ "tokens_seen": 1070202880
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005460702106318957,
+ "loss": 1.3509,
+ "theoretical_loss": 3.6542949530183035,
+ "tokens_seen": 1070268416
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005460541624874624,
+ "loss": 1.3639,
+ "theoretical_loss": 3.654274089143043,
+ "tokens_seen": 1070333952
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005460381143430291,
+ "loss": 1.4166,
+ "theoretical_loss": 3.654253226902899,
+ "tokens_seen": 1070399488
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005460220661985958,
+ "loss": 1.3486,
+ "theoretical_loss": 3.6542323662976424,
+ "tokens_seen": 1070465024
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005460060180541625,
+ "loss": 1.4473,
+ "theoretical_loss": 3.654211507327046,
+ "tokens_seen": 1070530560
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005459899699097292,
+ "loss": 1.2746,
+ "theoretical_loss": 3.6541906499908805,
+ "tokens_seen": 1070596096
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005459739217652959,
+ "loss": 1.3722,
+ "theoretical_loss": 3.6541697942889177,
+ "tokens_seen": 1070661632
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005459578736208626,
+ "loss": 1.4536,
+ "theoretical_loss": 3.654148940220931,
+ "tokens_seen": 1070727168
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005459418254764293,
+ "loss": 1.3258,
+ "theoretical_loss": 3.654128087786691,
+ "tokens_seen": 1070792704
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000545925777331996,
+ "loss": 1.3775,
+ "theoretical_loss": 3.65410723698597,
+ "tokens_seen": 1070858240
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005459097291875627,
+ "loss": 1.3203,
+ "theoretical_loss": 3.654086387818541,
+ "tokens_seen": 1070923776
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458936810431294,
+ "loss": 1.3357,
+ "theoretical_loss": 3.654065540284175,
+ "tokens_seen": 1070989312
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458776328986961,
+ "loss": 1.4537,
+ "theoretical_loss": 3.6540446943826446,
+ "tokens_seen": 1071054848
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458615847542628,
+ "loss": 1.3457,
+ "theoretical_loss": 3.654023850113722,
+ "tokens_seen": 1071120384
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458455366098295,
+ "loss": 1.4527,
+ "theoretical_loss": 3.654003007477179,
+ "tokens_seen": 1071185920
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458294884653962,
+ "loss": 1.3514,
+ "theoretical_loss": 3.6539821664727894,
+ "tokens_seen": 1071251456
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005458134403209629,
+ "loss": 1.3989,
+ "theoretical_loss": 3.6539613271003235,
+ "tokens_seen": 1071316992
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457973921765296,
+ "loss": 1.5363,
+ "theoretical_loss": 3.653940489359555,
+ "tokens_seen": 1071382528
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457813440320963,
+ "loss": 1.473,
+ "theoretical_loss": 3.6539196532502562,
+ "tokens_seen": 1071448064
+ },
+ {
+ "epoch": 0.32,
+ "objective/train/docs_used": 361363,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2865943908691406,
+ "objective/train/theoretical_loss": 3.6538988187721997,
+ "objective/train/tokens_used": 1091973600,
+ "theoretical_loss": 3.6538988187721997,
+ "tokens_seen": 1071513600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.000545765295887663,
+ "loss": 1.3613,
+ "theoretical_loss": 3.6538988187721997,
+ "tokens_seen": 1071513600
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457492477432297,
+ "loss": 1.326,
+ "theoretical_loss": 3.653877985925157,
+ "tokens_seen": 1071579136
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457331995987964,
+ "loss": 1.456,
+ "theoretical_loss": 3.653857154708902,
+ "tokens_seen": 1071644672
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457171514543631,
+ "loss": 1.3492,
+ "theoretical_loss": 3.6538363251232067,
+ "tokens_seen": 1071710208
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005457011033099298,
+ "loss": 1.3966,
+ "theoretical_loss": 3.6538154971678436,
+ "tokens_seen": 1071775744
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005456850551654965,
+ "loss": 1.2911,
+ "theoretical_loss": 3.6537946708425864,
+ "tokens_seen": 1071841280
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005456690070210632,
+ "loss": 1.1557,
+ "theoretical_loss": 3.653773846147206,
+ "tokens_seen": 1071906816
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005456529588766299,
+ "loss": 1.3184,
+ "theoretical_loss": 3.653753023081477,
+ "tokens_seen": 1071972352
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005456369107321966,
+ "loss": 1.3235,
+ "theoretical_loss": 3.6537322016451714,
+ "tokens_seen": 1072037888
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005456208625877633,
+ "loss": 1.3949,
+ "theoretical_loss": 3.653711381838062,
+ "tokens_seen": 1072103424
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.00054560481444333,
+ "loss": 1.3414,
+ "theoretical_loss": 3.6536905636599224,
+ "tokens_seen": 1072168960
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005455887662988967,
+ "loss": 1.3246,
+ "theoretical_loss": 3.6536697471105253,
+ "tokens_seen": 1072234496
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005455727181544634,
+ "loss": 1.4048,
+ "theoretical_loss": 3.6536489321896433,
+ "tokens_seen": 1072300032
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005455566700100301,
+ "loss": 1.2653,
+ "theoretical_loss": 3.65362811889705,
+ "tokens_seen": 1072365568
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005455406218655968,
+ "loss": 1.3582,
+ "theoretical_loss": 3.653607307232518,
+ "tokens_seen": 1072431104
+ },
+ {
+ "epoch": 0.32,
+ "learning_rate": 0.0005455245737211635,
+ "loss": 1.3284,
+ "theoretical_loss": 3.653586497195821,
+ "tokens_seen": 1072496640
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005455085255767302,
+ "loss": 1.4269,
+ "theoretical_loss": 3.653565688786732,
+ "tokens_seen": 1072562176
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005454924774322969,
+ "loss": 1.33,
+ "theoretical_loss": 3.6535448820050247,
+ "tokens_seen": 1072627712
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005454764292878636,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6535240768504718,
+ "tokens_seen": 1072693248
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005454603811434303,
+ "loss": 1.3163,
+ "theoretical_loss": 3.6535032733228467,
+ "tokens_seen": 1072758784
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000545444332998997,
+ "loss": 1.3202,
+ "theoretical_loss": 3.6534824714219227,
+ "tokens_seen": 1072824320
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005454282848545637,
+ "loss": 1.3219,
+ "theoretical_loss": 3.6534616711474737,
+ "tokens_seen": 1072889856
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005454122367101304,
+ "loss": 1.294,
+ "theoretical_loss": 3.653440872499273,
+ "tokens_seen": 1072955392
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005453961885656971,
+ "loss": 1.3378,
+ "theoretical_loss": 3.6534200754770945,
+ "tokens_seen": 1073020928
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005453801404212638,
+ "loss": 1.3848,
+ "theoretical_loss": 3.653399280080711,
+ "tokens_seen": 1073086464
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005453640922768305,
+ "loss": 1.4149,
+ "theoretical_loss": 3.6533784863098964,
+ "tokens_seen": 1073152000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005453480441323973,
+ "loss": 1.4493,
+ "theoretical_loss": 3.6533576941644244,
+ "tokens_seen": 1073217536
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000545331995987964,
+ "loss": 1.4422,
+ "theoretical_loss": 3.6533369036440693,
+ "tokens_seen": 1073283072
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005453159478435307,
+ "loss": 1.344,
+ "theoretical_loss": 3.6533161147486037,
+ "tokens_seen": 1073348608
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452998996990974,
+ "loss": 1.3856,
+ "theoretical_loss": 3.6532953274778026,
+ "tokens_seen": 1073414144
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000545283851554664,
+ "loss": 1.4246,
+ "theoretical_loss": 3.653274541831439,
+ "tokens_seen": 1073479680
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452678034102308,
+ "loss": 1.4139,
+ "theoretical_loss": 3.6532537578092876,
+ "tokens_seen": 1073545216
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452517552657975,
+ "loss": 1.2834,
+ "theoretical_loss": 3.6532329754111217,
+ "tokens_seen": 1073610752
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452357071213642,
+ "loss": 1.3038,
+ "theoretical_loss": 3.6532121946367146,
+ "tokens_seen": 1073676288
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452196589769309,
+ "loss": 1.4628,
+ "theoretical_loss": 3.653191415485842,
+ "tokens_seen": 1073741824
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005452036108324976,
+ "loss": 1.3071,
+ "theoretical_loss": 3.6531706379582776,
+ "tokens_seen": 1073807360
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451875626880643,
+ "loss": 1.3948,
+ "theoretical_loss": 3.653149862053794,
+ "tokens_seen": 1073872896
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451715145436309,
+ "loss": 1.2693,
+ "theoretical_loss": 3.653129087772167,
+ "tokens_seen": 1073938432
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451554663991977,
+ "loss": 1.3543,
+ "theoretical_loss": 3.65310831511317,
+ "tokens_seen": 1074003968
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451394182547643,
+ "loss": 1.3167,
+ "theoretical_loss": 3.6530875440765778,
+ "tokens_seen": 1074069504
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451233701103311,
+ "loss": 1.2411,
+ "theoretical_loss": 3.6530667746621646,
+ "tokens_seen": 1074135040
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005451073219658977,
+ "loss": 1.3381,
+ "theoretical_loss": 3.6530460068697037,
+ "tokens_seen": 1074200576
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450912738214645,
+ "loss": 1.3835,
+ "theoretical_loss": 3.6530252406989714,
+ "tokens_seen": 1074266112
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450752256770311,
+ "loss": 1.4331,
+ "theoretical_loss": 3.65300447614974,
+ "tokens_seen": 1074331648
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450591775325979,
+ "loss": 1.4747,
+ "theoretical_loss": 3.6529837132217864,
+ "tokens_seen": 1074397184
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450431293881645,
+ "loss": 1.3274,
+ "theoretical_loss": 3.6529629519148825,
+ "tokens_seen": 1074462720
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450270812437312,
+ "loss": 1.3775,
+ "theoretical_loss": 3.6529421922288052,
+ "tokens_seen": 1074528256
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005450110330992979,
+ "loss": 1.4209,
+ "theoretical_loss": 3.6529214341633276,
+ "tokens_seen": 1074593792
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005449949849548646,
+ "loss": 1.3885,
+ "theoretical_loss": 3.6529006777182254,
+ "tokens_seen": 1074659328
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005449789368104313,
+ "loss": 1.3243,
+ "theoretical_loss": 3.652879922893272,
+ "tokens_seen": 1074724864
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 362330,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3024756908416748,
+ "objective/train/theoretical_loss": 3.6528591696882433,
+ "objective/train/tokens_used": 1095250400,
+ "theoretical_loss": 3.6528591696882433,
+ "tokens_seen": 1074790400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544962888665998,
+ "loss": 1.3094,
+ "theoretical_loss": 3.6528591696882433,
+ "tokens_seen": 1074790400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005449468405215647,
+ "loss": 1.4161,
+ "theoretical_loss": 3.652838418102914,
+ "tokens_seen": 1074855936
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005449307923771314,
+ "loss": 1.3407,
+ "theoretical_loss": 3.6528176681370583,
+ "tokens_seen": 1074921472
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005449147442326981,
+ "loss": 1.3344,
+ "theoretical_loss": 3.6527969197904513,
+ "tokens_seen": 1074987008
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448986960882648,
+ "loss": 1.3573,
+ "theoretical_loss": 3.6527761730628687,
+ "tokens_seen": 1075052544
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448826479438315,
+ "loss": 1.3483,
+ "theoretical_loss": 3.6527554279540846,
+ "tokens_seen": 1075118080
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448665997993982,
+ "loss": 1.3747,
+ "theoretical_loss": 3.6527346844638746,
+ "tokens_seen": 1075183616
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448505516549649,
+ "loss": 1.3018,
+ "theoretical_loss": 3.6527139425920128,
+ "tokens_seen": 1075249152
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448345035105316,
+ "loss": 1.3957,
+ "theoretical_loss": 3.652693202338275,
+ "tokens_seen": 1075314688
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005448184553660983,
+ "loss": 1.3263,
+ "theoretical_loss": 3.6526724637024373,
+ "tokens_seen": 1075380224
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544802407221665,
+ "loss": 1.3125,
+ "theoretical_loss": 3.6526517266842733,
+ "tokens_seen": 1075445760
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447863590772317,
+ "loss": 1.3413,
+ "theoretical_loss": 3.652630991283559,
+ "tokens_seen": 1075511296
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447703109327984,
+ "loss": 1.2878,
+ "theoretical_loss": 3.6526102575000694,
+ "tokens_seen": 1075576832
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447542627883651,
+ "loss": 1.3885,
+ "theoretical_loss": 3.6525895253335805,
+ "tokens_seen": 1075642368
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447382146439318,
+ "loss": 1.2449,
+ "theoretical_loss": 3.6525687947838668,
+ "tokens_seen": 1075707904
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447221664994985,
+ "loss": 1.33,
+ "theoretical_loss": 3.652548065850705,
+ "tokens_seen": 1075773440
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005447061183550652,
+ "loss": 1.2818,
+ "theoretical_loss": 3.6525273385338686,
+ "tokens_seen": 1075838976
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005446900702106319,
+ "loss": 1.2698,
+ "theoretical_loss": 3.6525066128331347,
+ "tokens_seen": 1075904512
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005446740220661986,
+ "loss": 1.4027,
+ "theoretical_loss": 3.6524858887482785,
+ "tokens_seen": 1075970048
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005446579739217653,
+ "loss": 1.323,
+ "theoretical_loss": 3.6524651662790752,
+ "tokens_seen": 1076035584
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544641925777332,
+ "loss": 1.4093,
+ "theoretical_loss": 3.652444445425301,
+ "tokens_seen": 1076101120
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005446258776328987,
+ "loss": 1.2921,
+ "theoretical_loss": 3.652423726186732,
+ "tokens_seen": 1076166656
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005446098294884654,
+ "loss": 1.3405,
+ "theoretical_loss": 3.6524030085631427,
+ "tokens_seen": 1076232192
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445937813440321,
+ "loss": 1.4306,
+ "theoretical_loss": 3.652382292554309,
+ "tokens_seen": 1076297728
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445777331995988,
+ "loss": 1.4591,
+ "theoretical_loss": 3.652361578160008,
+ "tokens_seen": 1076363264
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445616850551655,
+ "loss": 1.3253,
+ "theoretical_loss": 3.6523408653800145,
+ "tokens_seen": 1076428800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445456369107322,
+ "loss": 1.2952,
+ "theoretical_loss": 3.652320154214105,
+ "tokens_seen": 1076494336
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445295887662989,
+ "loss": 1.3806,
+ "theoretical_loss": 3.652299444662055,
+ "tokens_seen": 1076559872
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005445135406218656,
+ "loss": 1.4121,
+ "theoretical_loss": 3.6522787367236407,
+ "tokens_seen": 1076625408
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444974924774323,
+ "loss": 1.4716,
+ "theoretical_loss": 3.652258030398638,
+ "tokens_seen": 1076690944
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544481444332999,
+ "loss": 1.2995,
+ "theoretical_loss": 3.6522373256868237,
+ "tokens_seen": 1076756480
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444653961885657,
+ "loss": 1.3696,
+ "theoretical_loss": 3.652216622587973,
+ "tokens_seen": 1076822016
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444493480441324,
+ "loss": 1.3486,
+ "theoretical_loss": 3.6521959211018626,
+ "tokens_seen": 1076887552
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444332998996991,
+ "loss": 1.322,
+ "theoretical_loss": 3.6521752212282688,
+ "tokens_seen": 1076953088
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444172517552658,
+ "loss": 1.3572,
+ "theoretical_loss": 3.652154522966968,
+ "tokens_seen": 1077018624
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005444012036108325,
+ "loss": 1.3745,
+ "theoretical_loss": 3.652133826317735,
+ "tokens_seen": 1077084160
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005443851554663992,
+ "loss": 1.5146,
+ "theoretical_loss": 3.6521131312803483,
+ "tokens_seen": 1077149696
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005443691073219659,
+ "loss": 1.4918,
+ "theoretical_loss": 3.652092437854584,
+ "tokens_seen": 1077215232
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005443530591775326,
+ "loss": 1.3465,
+ "theoretical_loss": 3.652071746040217,
+ "tokens_seen": 1077280768
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005443370110330993,
+ "loss": 1.2983,
+ "theoretical_loss": 3.652051055837025,
+ "tokens_seen": 1077346304
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544320962888666,
+ "loss": 1.2927,
+ "theoretical_loss": 3.6520303672447847,
+ "tokens_seen": 1077411840
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005443049147442327,
+ "loss": 1.4392,
+ "theoretical_loss": 3.652009680263272,
+ "tokens_seen": 1077477376
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442888665997994,
+ "loss": 1.4125,
+ "theoretical_loss": 3.651988994892264,
+ "tokens_seen": 1077542912
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442728184553661,
+ "loss": 1.3548,
+ "theoretical_loss": 3.6519683111315375,
+ "tokens_seen": 1077608448
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442567703109328,
+ "loss": 1.4242,
+ "theoretical_loss": 3.651947628980869,
+ "tokens_seen": 1077673984
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442407221664995,
+ "loss": 1.3011,
+ "theoretical_loss": 3.6519269484400345,
+ "tokens_seen": 1077739520
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442246740220662,
+ "loss": 1.2613,
+ "theoretical_loss": 3.651906269508812,
+ "tokens_seen": 1077805056
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005442086258776329,
+ "loss": 1.311,
+ "theoretical_loss": 3.651885592186978,
+ "tokens_seen": 1077870592
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005441925777331996,
+ "loss": 1.3905,
+ "theoretical_loss": 3.6518649164743096,
+ "tokens_seen": 1077936128
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005441765295887663,
+ "loss": 1.383,
+ "theoretical_loss": 3.651844242370583,
+ "tokens_seen": 1078001664
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 363687,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3248112201690674,
+ "objective/train/theoretical_loss": 3.651823569875576,
+ "objective/train/tokens_used": 1098527200,
+ "theoretical_loss": 3.651823569875576,
+ "tokens_seen": 1078067200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000544160481444333,
+ "loss": 1.3867,
+ "theoretical_loss": 3.651823569875576,
+ "tokens_seen": 1078067200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005441444332998997,
+ "loss": 1.3997,
+ "theoretical_loss": 3.6518028989890654,
+ "tokens_seen": 1078132736
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005441283851554665,
+ "loss": 1.3344,
+ "theoretical_loss": 3.651782229710828,
+ "tokens_seen": 1078198272
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005441123370110331,
+ "loss": 1.2955,
+ "theoretical_loss": 3.6517615620406407,
+ "tokens_seen": 1078263808
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440962888665999,
+ "loss": 1.4162,
+ "theoretical_loss": 3.6517408959782816,
+ "tokens_seen": 1078329344
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440802407221665,
+ "loss": 1.3732,
+ "theoretical_loss": 3.651720231523527,
+ "tokens_seen": 1078394880
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440641925777333,
+ "loss": 1.4056,
+ "theoretical_loss": 3.651699568676155,
+ "tokens_seen": 1078460416
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440481444333,
+ "loss": 1.3838,
+ "theoretical_loss": 3.651678907435942,
+ "tokens_seen": 1078525952
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440320962888667,
+ "loss": 1.3309,
+ "theoretical_loss": 3.6516582478026662,
+ "tokens_seen": 1078591488
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440160481444333,
+ "loss": 1.3331,
+ "theoretical_loss": 3.6516375897761044,
+ "tokens_seen": 1078657024
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005440000000000001,
+ "loss": 1.2792,
+ "theoretical_loss": 3.651616933356034,
+ "tokens_seen": 1078722560
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439839518555668,
+ "loss": 1.4044,
+ "theoretical_loss": 3.6515962785422333,
+ "tokens_seen": 1078788096
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439679037111335,
+ "loss": 1.4031,
+ "theoretical_loss": 3.6515756253344787,
+ "tokens_seen": 1078853632
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439518555667002,
+ "loss": 1.3347,
+ "theoretical_loss": 3.651554973732549,
+ "tokens_seen": 1078919168
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439358074222669,
+ "loss": 1.4163,
+ "theoretical_loss": 3.6515343237362203,
+ "tokens_seen": 1078984704
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439197592778336,
+ "loss": 1.39,
+ "theoretical_loss": 3.6515136753452717,
+ "tokens_seen": 1079050240
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005439037111334003,
+ "loss": 1.3692,
+ "theoretical_loss": 3.65149302855948,
+ "tokens_seen": 1079115776
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543887662988967,
+ "loss": 1.2648,
+ "theoretical_loss": 3.651472383378623,
+ "tokens_seen": 1079181312
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005438716148445337,
+ "loss": 1.4106,
+ "theoretical_loss": 3.651451739802479,
+ "tokens_seen": 1079246848
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005438555667001004,
+ "loss": 1.4743,
+ "theoretical_loss": 3.6514310978308258,
+ "tokens_seen": 1079312384
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005438395185556671,
+ "loss": 1.3613,
+ "theoretical_loss": 3.6514104574634407,
+ "tokens_seen": 1079377920
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005438234704112338,
+ "loss": 1.4059,
+ "theoretical_loss": 3.6513898187001024,
+ "tokens_seen": 1079443456
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005438074222668005,
+ "loss": 1.357,
+ "theoretical_loss": 3.6513691815405878,
+ "tokens_seen": 1079508992
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005437913741223672,
+ "loss": 1.3831,
+ "theoretical_loss": 3.6513485459846757,
+ "tokens_seen": 1079574528
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005437753259779338,
+ "loss": 1.2636,
+ "theoretical_loss": 3.6513279120321442,
+ "tokens_seen": 1079640064
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005437592778335006,
+ "loss": 1.3083,
+ "theoretical_loss": 3.6513072796827712,
+ "tokens_seen": 1079705600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005437432296890672,
+ "loss": 1.4416,
+ "theoretical_loss": 3.651286648936335,
+ "tokens_seen": 1079771136
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543727181544634,
+ "loss": 1.3512,
+ "theoretical_loss": 3.6512660197926134,
+ "tokens_seen": 1079836672
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005437111334002006,
+ "loss": 1.3073,
+ "theoretical_loss": 3.6512453922513854,
+ "tokens_seen": 1079902208
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005436950852557674,
+ "loss": 1.2654,
+ "theoretical_loss": 3.6512247663124278,
+ "tokens_seen": 1079967744
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543679037111334,
+ "loss": 1.3332,
+ "theoretical_loss": 3.6512041419755206,
+ "tokens_seen": 1080033280
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005436629889669008,
+ "loss": 1.374,
+ "theoretical_loss": 3.6511835192404414,
+ "tokens_seen": 1080098816
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005436469408224674,
+ "loss": 1.395,
+ "theoretical_loss": 3.6511628981069686,
+ "tokens_seen": 1080164352
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005436308926780341,
+ "loss": 1.4138,
+ "theoretical_loss": 3.6511422785748806,
+ "tokens_seen": 1080229888
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005436148445336008,
+ "loss": 1.4597,
+ "theoretical_loss": 3.651121660643956,
+ "tokens_seen": 1080295424
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435987963891675,
+ "loss": 1.4631,
+ "theoretical_loss": 3.651101044313973,
+ "tokens_seen": 1080360960
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435827482447342,
+ "loss": 1.4754,
+ "theoretical_loss": 3.651080429584711,
+ "tokens_seen": 1080426496
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435667001003009,
+ "loss": 1.3145,
+ "theoretical_loss": 3.651059816455948,
+ "tokens_seen": 1080492032
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435506519558676,
+ "loss": 1.4438,
+ "theoretical_loss": 3.6510392049274625,
+ "tokens_seen": 1080557568
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435346038114343,
+ "loss": 1.3457,
+ "theoretical_loss": 3.651018594999034,
+ "tokens_seen": 1080623104
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543518555667001,
+ "loss": 1.413,
+ "theoretical_loss": 3.6509979866704407,
+ "tokens_seen": 1080688640
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005435025075225677,
+ "loss": 1.4507,
+ "theoretical_loss": 3.6509773799414615,
+ "tokens_seen": 1080754176
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434864593781344,
+ "loss": 1.3434,
+ "theoretical_loss": 3.6509567748118754,
+ "tokens_seen": 1080819712
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434704112337011,
+ "loss": 1.2866,
+ "theoretical_loss": 3.650936171281461,
+ "tokens_seen": 1080885248
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434543630892678,
+ "loss": 1.324,
+ "theoretical_loss": 3.6509155693499977,
+ "tokens_seen": 1080950784
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434383149448345,
+ "loss": 1.3252,
+ "theoretical_loss": 3.650894969017264,
+ "tokens_seen": 1081016320
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434222668004012,
+ "loss": 1.4008,
+ "theoretical_loss": 3.650874370283039,
+ "tokens_seen": 1081081856
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005434062186559679,
+ "loss": 1.2546,
+ "theoretical_loss": 3.650853773147102,
+ "tokens_seen": 1081147392
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005433901705115346,
+ "loss": 1.3324,
+ "theoretical_loss": 3.6508331776092318,
+ "tokens_seen": 1081212928
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005433741223671013,
+ "loss": 1.3447,
+ "theoretical_loss": 3.6508125836692082,
+ "tokens_seen": 1081278464
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 364592,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2894819974899292,
+ "objective/train/theoretical_loss": 3.6507919913268094,
+ "objective/train/tokens_used": 1101804000,
+ "theoretical_loss": 3.6507919913268094,
+ "tokens_seen": 1081344000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543358074222668,
+ "loss": 1.316,
+ "theoretical_loss": 3.6507919913268094,
+ "tokens_seen": 1081344000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005433420260782347,
+ "loss": 1.2995,
+ "theoretical_loss": 3.6507714005818155,
+ "tokens_seen": 1081409536
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005433259779338014,
+ "loss": 1.418,
+ "theoretical_loss": 3.6507508114340057,
+ "tokens_seen": 1081475072
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005433099297893681,
+ "loss": 1.3716,
+ "theoretical_loss": 3.6507302238831585,
+ "tokens_seen": 1081540608
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432938816449348,
+ "loss": 1.4082,
+ "theoretical_loss": 3.6507096379290545,
+ "tokens_seen": 1081606144
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432778335005015,
+ "loss": 1.3643,
+ "theoretical_loss": 3.6506890535714724,
+ "tokens_seen": 1081671680
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432617853560682,
+ "loss": 1.2969,
+ "theoretical_loss": 3.6506684708101913,
+ "tokens_seen": 1081737216
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432457372116349,
+ "loss": 1.3222,
+ "theoretical_loss": 3.6506478896449917,
+ "tokens_seen": 1081802752
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432296890672016,
+ "loss": 1.3993,
+ "theoretical_loss": 3.6506273100756528,
+ "tokens_seen": 1081868288
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005432136409227683,
+ "loss": 1.387,
+ "theoretical_loss": 3.6506067321019535,
+ "tokens_seen": 1081933824
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543197592778335,
+ "loss": 1.3076,
+ "theoretical_loss": 3.6505861557236745,
+ "tokens_seen": 1081999360
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431815446339017,
+ "loss": 1.2477,
+ "theoretical_loss": 3.6505655809405946,
+ "tokens_seen": 1082064896
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431654964894684,
+ "loss": 1.2312,
+ "theoretical_loss": 3.6505450077524944,
+ "tokens_seen": 1082130432
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431494483450351,
+ "loss": 1.4247,
+ "theoretical_loss": 3.6505244361591527,
+ "tokens_seen": 1082195968
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431334002006018,
+ "loss": 1.3597,
+ "theoretical_loss": 3.65050386616035,
+ "tokens_seen": 1082261504
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431173520561685,
+ "loss": 1.2222,
+ "theoretical_loss": 3.6504832977558657,
+ "tokens_seen": 1082327040
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005431013039117352,
+ "loss": 1.407,
+ "theoretical_loss": 3.6504627309454802,
+ "tokens_seen": 1082392576
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005430852557673019,
+ "loss": 1.3942,
+ "theoretical_loss": 3.6504421657289727,
+ "tokens_seen": 1082458112
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005430692076228686,
+ "loss": 1.3663,
+ "theoretical_loss": 3.6504216021061247,
+ "tokens_seen": 1082523648
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005430531594784353,
+ "loss": 1.4238,
+ "theoretical_loss": 3.6504010400767144,
+ "tokens_seen": 1082589184
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000543037111334002,
+ "loss": 1.2975,
+ "theoretical_loss": 3.650380479640523,
+ "tokens_seen": 1082654720
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005430210631895687,
+ "loss": 1.3741,
+ "theoretical_loss": 3.65035992079733,
+ "tokens_seen": 1082720256
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005430050150451354,
+ "loss": 1.3612,
+ "theoretical_loss": 3.650339363546916,
+ "tokens_seen": 1082785792
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005429889669007021,
+ "loss": 1.3039,
+ "theoretical_loss": 3.6503188078890614,
+ "tokens_seen": 1082851328
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005429729187562688,
+ "loss": 1.4418,
+ "theoretical_loss": 3.650298253823546,
+ "tokens_seen": 1082916864
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005429568706118355,
+ "loss": 1.4219,
+ "theoretical_loss": 3.6502777013501504,
+ "tokens_seen": 1082982400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005429408224674022,
+ "loss": 1.3086,
+ "theoretical_loss": 3.6502571504686543,
+ "tokens_seen": 1083047936
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542924774322969,
+ "loss": 1.3986,
+ "theoretical_loss": 3.6502366011788387,
+ "tokens_seen": 1083113472
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005429087261785356,
+ "loss": 1.4054,
+ "theoretical_loss": 3.650216053480484,
+ "tokens_seen": 1083179008
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005428926780341024,
+ "loss": 1.3911,
+ "theoretical_loss": 3.650195507373371,
+ "tokens_seen": 1083244544
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542876629889669,
+ "loss": 1.3547,
+ "theoretical_loss": 3.650174962857279,
+ "tokens_seen": 1083310080
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005428605817452358,
+ "loss": 1.37,
+ "theoretical_loss": 3.6501544199319897,
+ "tokens_seen": 1083375616
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005428445336008024,
+ "loss": 1.3649,
+ "theoretical_loss": 3.650133878597283,
+ "tokens_seen": 1083441152
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005428284854563692,
+ "loss": 1.3181,
+ "theoretical_loss": 3.6501133388529405,
+ "tokens_seen": 1083506688
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005428124373119358,
+ "loss": 1.3656,
+ "theoretical_loss": 3.650092800698742,
+ "tokens_seen": 1083572224
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005427963891675026,
+ "loss": 1.4077,
+ "theoretical_loss": 3.6500722641344683,
+ "tokens_seen": 1083637760
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005427803410230692,
+ "loss": 1.3426,
+ "theoretical_loss": 3.6500517291599004,
+ "tokens_seen": 1083703296
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542764292878636,
+ "loss": 1.3751,
+ "theoretical_loss": 3.6500311957748193,
+ "tokens_seen": 1083768832
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005427482447342026,
+ "loss": 1.335,
+ "theoretical_loss": 3.6500106639790055,
+ "tokens_seen": 1083834368
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005427321965897694,
+ "loss": 1.5347,
+ "theoretical_loss": 3.6499901337722402,
+ "tokens_seen": 1083899904
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542716148445336,
+ "loss": 1.3838,
+ "theoretical_loss": 3.649969605154304,
+ "tokens_seen": 1083965440
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005427001003009028,
+ "loss": 1.2416,
+ "theoretical_loss": 3.649949078124978,
+ "tokens_seen": 1084030976
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005426840521564694,
+ "loss": 1.3768,
+ "theoretical_loss": 3.6499285526840435,
+ "tokens_seen": 1084096512
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005426680040120362,
+ "loss": 1.3936,
+ "theoretical_loss": 3.6499080288312813,
+ "tokens_seen": 1084162048
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005426519558676028,
+ "loss": 1.4992,
+ "theoretical_loss": 3.649887506566473,
+ "tokens_seen": 1084227584
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005426359077231696,
+ "loss": 1.4021,
+ "theoretical_loss": 3.6498669858893993,
+ "tokens_seen": 1084293120
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005426198595787363,
+ "loss": 1.2887,
+ "theoretical_loss": 3.6498464667998416,
+ "tokens_seen": 1084358656
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542603811434303,
+ "loss": 1.4855,
+ "theoretical_loss": 3.6498259492975804,
+ "tokens_seen": 1084424192
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005425877632898697,
+ "loss": 1.3107,
+ "theoretical_loss": 3.649805433382398,
+ "tokens_seen": 1084489728
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005425717151454364,
+ "loss": 1.3343,
+ "theoretical_loss": 3.649784919054076,
+ "tokens_seen": 1084555264
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 365785,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3670212030410767,
+ "objective/train/theoretical_loss": 3.649764406312394,
+ "objective/train/tokens_used": 1105080800,
+ "theoretical_loss": 3.649764406312394,
+ "tokens_seen": 1084620800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542555667001003,
+ "loss": 1.4863,
+ "theoretical_loss": 3.649764406312394,
+ "tokens_seen": 1084620800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005425396188565698,
+ "loss": 1.4798,
+ "theoretical_loss": 3.649743895157135,
+ "tokens_seen": 1084686336
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005425235707121365,
+ "loss": 1.2752,
+ "theoretical_loss": 3.6497233855880804,
+ "tokens_seen": 1084751872
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005425075225677032,
+ "loss": 1.4032,
+ "theoretical_loss": 3.6497028776050113,
+ "tokens_seen": 1084817408
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005424914744232699,
+ "loss": 1.3299,
+ "theoretical_loss": 3.6496823712077093,
+ "tokens_seen": 1084882944
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005424754262788366,
+ "loss": 1.339,
+ "theoretical_loss": 3.649661866395957,
+ "tokens_seen": 1084948480
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005424593781344033,
+ "loss": 1.394,
+ "theoretical_loss": 3.6496413631695335,
+ "tokens_seen": 1085014016
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.00054244332998997,
+ "loss": 1.3726,
+ "theoretical_loss": 3.6496208615282226,
+ "tokens_seen": 1085079552
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005424272818455367,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6496003614718058,
+ "tokens_seen": 1085145088
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005424112337011034,
+ "loss": 1.3156,
+ "theoretical_loss": 3.6495798630000644,
+ "tokens_seen": 1085210624
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423951855566701,
+ "loss": 1.4943,
+ "theoretical_loss": 3.64955936611278,
+ "tokens_seen": 1085276160
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423791374122367,
+ "loss": 1.3143,
+ "theoretical_loss": 3.649538870809735,
+ "tokens_seen": 1085341696
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423630892678035,
+ "loss": 1.3866,
+ "theoretical_loss": 3.649518377090712,
+ "tokens_seen": 1085407232
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423470411233701,
+ "loss": 1.2915,
+ "theoretical_loss": 3.6494978849554913,
+ "tokens_seen": 1085472768
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423309929789369,
+ "loss": 1.2183,
+ "theoretical_loss": 3.6494773944038554,
+ "tokens_seen": 1085538304
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005423149448345035,
+ "loss": 1.4131,
+ "theoretical_loss": 3.6494569054355877,
+ "tokens_seen": 1085603840
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422988966900703,
+ "loss": 1.3191,
+ "theoretical_loss": 3.649436418050468,
+ "tokens_seen": 1085669376
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422828485456369,
+ "loss": 1.468,
+ "theoretical_loss": 3.64941593224828,
+ "tokens_seen": 1085734912
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422668004012037,
+ "loss": 1.3757,
+ "theoretical_loss": 3.6493954480288058,
+ "tokens_seen": 1085800448
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422507522567703,
+ "loss": 1.3375,
+ "theoretical_loss": 3.649374965391827,
+ "tokens_seen": 1085865984
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542234704112337,
+ "loss": 1.3805,
+ "theoretical_loss": 3.649354484337126,
+ "tokens_seen": 1085931520
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422186559679037,
+ "loss": 1.397,
+ "theoretical_loss": 3.6493340048644853,
+ "tokens_seen": 1085997056
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005422026078234704,
+ "loss": 1.321,
+ "theoretical_loss": 3.649313526973687,
+ "tokens_seen": 1086062592
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421865596790371,
+ "loss": 1.3058,
+ "theoretical_loss": 3.649293050664514,
+ "tokens_seen": 1086128128
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421705115346038,
+ "loss": 1.3293,
+ "theoretical_loss": 3.649272575936748,
+ "tokens_seen": 1086193664
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421544633901705,
+ "loss": 1.2778,
+ "theoretical_loss": 3.6492521027901716,
+ "tokens_seen": 1086259200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421384152457372,
+ "loss": 1.3685,
+ "theoretical_loss": 3.649231631224568,
+ "tokens_seen": 1086324736
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421223671013039,
+ "loss": 1.3574,
+ "theoretical_loss": 3.649211161239719,
+ "tokens_seen": 1086390272
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005421063189568706,
+ "loss": 1.3703,
+ "theoretical_loss": 3.6491906928354076,
+ "tokens_seen": 1086455808
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005420902708124373,
+ "loss": 1.3048,
+ "theoretical_loss": 3.649170226011416,
+ "tokens_seen": 1086521344
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000542074222668004,
+ "loss": 1.3437,
+ "theoretical_loss": 3.6491497607675276,
+ "tokens_seen": 1086586880
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005420581745235707,
+ "loss": 1.3581,
+ "theoretical_loss": 3.6491292971035243,
+ "tokens_seen": 1086652416
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005420421263791374,
+ "loss": 1.2778,
+ "theoretical_loss": 3.649108835019189,
+ "tokens_seen": 1086717952
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005420260782347041,
+ "loss": 1.366,
+ "theoretical_loss": 3.6490883745143052,
+ "tokens_seen": 1086783488
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005420100300902708,
+ "loss": 1.4187,
+ "theoretical_loss": 3.649067915588655,
+ "tokens_seen": 1086849024
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005419939819458375,
+ "loss": 1.3827,
+ "theoretical_loss": 3.6490474582420216,
+ "tokens_seen": 1086914560
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005419779338014042,
+ "loss": 1.4968,
+ "theoretical_loss": 3.649027002474188,
+ "tokens_seen": 1086980096
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005419618856569709,
+ "loss": 1.368,
+ "theoretical_loss": 3.6490065482849374,
+ "tokens_seen": 1087045632
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005419458375125376,
+ "loss": 1.4541,
+ "theoretical_loss": 3.648986095674052,
+ "tokens_seen": 1087111168
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005419297893681043,
+ "loss": 1.2993,
+ "theoretical_loss": 3.6489656446413155,
+ "tokens_seen": 1087176704
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000541913741223671,
+ "loss": 1.3196,
+ "theoretical_loss": 3.648945195186511,
+ "tokens_seen": 1087242240
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418976930792377,
+ "loss": 1.3327,
+ "theoretical_loss": 3.6489247473094215,
+ "tokens_seen": 1087307776
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418816449348044,
+ "loss": 1.417,
+ "theoretical_loss": 3.6489043010098303,
+ "tokens_seen": 1087373312
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418655967903711,
+ "loss": 1.333,
+ "theoretical_loss": 3.6488838562875205,
+ "tokens_seen": 1087438848
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418495486459378,
+ "loss": 1.3239,
+ "theoretical_loss": 3.6488634131422755,
+ "tokens_seen": 1087504384
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418335005015045,
+ "loss": 1.3577,
+ "theoretical_loss": 3.6488429715738784,
+ "tokens_seen": 1087569920
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418174523570712,
+ "loss": 1.313,
+ "theoretical_loss": 3.6488225315821126,
+ "tokens_seen": 1087635456
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005418014042126379,
+ "loss": 1.4152,
+ "theoretical_loss": 3.648802093166762,
+ "tokens_seen": 1087700992
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005417853560682046,
+ "loss": 1.5566,
+ "theoretical_loss": 3.6487816563276096,
+ "tokens_seen": 1087766528
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005417693079237713,
+ "loss": 1.3243,
+ "theoretical_loss": 3.648761221064439,
+ "tokens_seen": 1087832064
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 366774,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3897125720977783,
+ "objective/train/theoretical_loss": 3.6487407873770334,
+ "objective/train/tokens_used": 1108357600,
+ "theoretical_loss": 3.6487407873770334,
+ "tokens_seen": 1087897600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000541753259779338,
+ "loss": 1.4037,
+ "theoretical_loss": 3.6487407873770334,
+ "tokens_seen": 1087897600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005417372116349047,
+ "loss": 1.3203,
+ "theoretical_loss": 3.6487203552651772,
+ "tokens_seen": 1087963136
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005417211634904715,
+ "loss": 1.4418,
+ "theoretical_loss": 3.648699924728653,
+ "tokens_seen": 1088028672
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005417051153460381,
+ "loss": 1.3909,
+ "theoretical_loss": 3.6486794957672455,
+ "tokens_seen": 1088094208
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416890672016049,
+ "loss": 1.3976,
+ "theoretical_loss": 3.648659068380737,
+ "tokens_seen": 1088159744
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416730190571715,
+ "loss": 1.374,
+ "theoretical_loss": 3.648638642568913,
+ "tokens_seen": 1088225280
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416569709127383,
+ "loss": 1.3495,
+ "theoretical_loss": 3.6486182183315563,
+ "tokens_seen": 1088290816
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416409227683049,
+ "loss": 1.3232,
+ "theoretical_loss": 3.6485977956684503,
+ "tokens_seen": 1088356352
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416248746238717,
+ "loss": 1.3278,
+ "theoretical_loss": 3.64857737457938,
+ "tokens_seen": 1088421888
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005416088264794383,
+ "loss": 1.2627,
+ "theoretical_loss": 3.6485569550641284,
+ "tokens_seen": 1088487424
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415927783350051,
+ "loss": 1.3575,
+ "theoretical_loss": 3.64853653712248,
+ "tokens_seen": 1088552960
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415767301905717,
+ "loss": 1.599,
+ "theoretical_loss": 3.6485161207542185,
+ "tokens_seen": 1088618496
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415606820461385,
+ "loss": 1.3982,
+ "theoretical_loss": 3.648495705959128,
+ "tokens_seen": 1088684032
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415446339017051,
+ "loss": 1.334,
+ "theoretical_loss": 3.648475292736993,
+ "tokens_seen": 1088749568
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415285857572719,
+ "loss": 1.407,
+ "theoretical_loss": 3.6484548810875967,
+ "tokens_seen": 1088815104
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005415125376128385,
+ "loss": 1.2509,
+ "theoretical_loss": 3.6484344710107246,
+ "tokens_seen": 1088880640
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414964894684053,
+ "loss": 1.387,
+ "theoretical_loss": 3.6484140625061596,
+ "tokens_seen": 1088946176
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414804413239719,
+ "loss": 1.3232,
+ "theoretical_loss": 3.648393655573687,
+ "tokens_seen": 1089011712
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414643931795387,
+ "loss": 1.4861,
+ "theoretical_loss": 3.6483732502130906,
+ "tokens_seen": 1089077248
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414483450351053,
+ "loss": 1.3551,
+ "theoretical_loss": 3.6483528464241544,
+ "tokens_seen": 1089142784
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414322968906721,
+ "loss": 1.3327,
+ "theoretical_loss": 3.6483324442066634,
+ "tokens_seen": 1089208320
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414162487462387,
+ "loss": 1.4498,
+ "theoretical_loss": 3.6483120435604017,
+ "tokens_seen": 1089273856
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005414002006018055,
+ "loss": 1.4741,
+ "theoretical_loss": 3.648291644485154,
+ "tokens_seen": 1089339392
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005413841524573721,
+ "loss": 1.3898,
+ "theoretical_loss": 3.6482712469807046,
+ "tokens_seen": 1089404928
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005413681043129389,
+ "loss": 1.3997,
+ "theoretical_loss": 3.6482508510468383,
+ "tokens_seen": 1089470464
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005413520561685055,
+ "loss": 1.4476,
+ "theoretical_loss": 3.6482304566833395,
+ "tokens_seen": 1089536000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005413360080240723,
+ "loss": 1.3293,
+ "theoretical_loss": 3.6482100638899926,
+ "tokens_seen": 1089601536
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000541319959879639,
+ "loss": 1.4213,
+ "theoretical_loss": 3.6481896726665832,
+ "tokens_seen": 1089667072
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005413039117352057,
+ "loss": 1.3317,
+ "theoretical_loss": 3.648169283012895,
+ "tokens_seen": 1089732608
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412878635907723,
+ "loss": 1.3798,
+ "theoretical_loss": 3.6481488949287133,
+ "tokens_seen": 1089798144
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412718154463391,
+ "loss": 1.3283,
+ "theoretical_loss": 3.648128508413823,
+ "tokens_seen": 1089863680
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412557673019058,
+ "loss": 1.3927,
+ "theoretical_loss": 3.648108123468009,
+ "tokens_seen": 1089929216
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412397191574725,
+ "loss": 1.4057,
+ "theoretical_loss": 3.648087740091055,
+ "tokens_seen": 1089994752
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412236710130392,
+ "loss": 1.3371,
+ "theoretical_loss": 3.6480673582827476,
+ "tokens_seen": 1090060288
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005412076228686059,
+ "loss": 1.2532,
+ "theoretical_loss": 3.6480469780428706,
+ "tokens_seen": 1090125824
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005411915747241726,
+ "loss": 1.3346,
+ "theoretical_loss": 3.64802659937121,
+ "tokens_seen": 1090191360
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005411755265797393,
+ "loss": 1.4336,
+ "theoretical_loss": 3.6480062222675502,
+ "tokens_seen": 1090256896
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000541159478435306,
+ "loss": 1.4444,
+ "theoretical_loss": 3.647985846731676,
+ "tokens_seen": 1090322432
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005411434302908727,
+ "loss": 1.3397,
+ "theoretical_loss": 3.647965472763374,
+ "tokens_seen": 1090387968
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005411273821464394,
+ "loss": 1.3925,
+ "theoretical_loss": 3.6479451003624277,
+ "tokens_seen": 1090453504
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005411113340020061,
+ "loss": 1.3722,
+ "theoretical_loss": 3.647924729528624,
+ "tokens_seen": 1090519040
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410952858575728,
+ "loss": 1.3825,
+ "theoretical_loss": 3.6479043602617462,
+ "tokens_seen": 1090584576
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410792377131395,
+ "loss": 1.2827,
+ "theoretical_loss": 3.6478839925615807,
+ "tokens_seen": 1090650112
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410631895687062,
+ "loss": 1.2401,
+ "theoretical_loss": 3.647863626427913,
+ "tokens_seen": 1090715648
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410471414242729,
+ "loss": 1.4024,
+ "theoretical_loss": 3.6478432618605288,
+ "tokens_seen": 1090781184
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410310932798396,
+ "loss": 1.4545,
+ "theoretical_loss": 3.647822898859213,
+ "tokens_seen": 1090846720
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005410150451354063,
+ "loss": 1.347,
+ "theoretical_loss": 3.647802537423751,
+ "tokens_seen": 1090912256
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540998996990973,
+ "loss": 1.3257,
+ "theoretical_loss": 3.6477821775539283,
+ "tokens_seen": 1090977792
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005409829488465397,
+ "loss": 1.3556,
+ "theoretical_loss": 3.647761819249531,
+ "tokens_seen": 1091043328
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005409669007021064,
+ "loss": 1.3661,
+ "theoretical_loss": 3.647741462510344,
+ "tokens_seen": 1091108864
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 368007,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3992815017700195,
+ "objective/train/theoretical_loss": 3.647721107336154,
+ "objective/train/tokens_used": 1111634400,
+ "theoretical_loss": 3.647721107336154,
+ "tokens_seen": 1091174400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540950852557673,
+ "loss": 1.5353,
+ "theoretical_loss": 3.647721107336154,
+ "tokens_seen": 1091174400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005409348044132398,
+ "loss": 1.3725,
+ "theoretical_loss": 3.647700753726746,
+ "tokens_seen": 1091239936
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005409187562688064,
+ "loss": 1.4593,
+ "theoretical_loss": 3.6476804016819053,
+ "tokens_seen": 1091305472
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005409027081243732,
+ "loss": 1.2989,
+ "theoretical_loss": 3.6476600512014183,
+ "tokens_seen": 1091371008
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408866599799398,
+ "loss": 1.2775,
+ "theoretical_loss": 3.647639702285071,
+ "tokens_seen": 1091436544
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408706118355066,
+ "loss": 1.4484,
+ "theoretical_loss": 3.647619354932649,
+ "tokens_seen": 1091502080
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408545636910732,
+ "loss": 1.3696,
+ "theoretical_loss": 3.647599009143938,
+ "tokens_seen": 1091567616
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408385155466399,
+ "loss": 1.3474,
+ "theoretical_loss": 3.6475786649187247,
+ "tokens_seen": 1091633152
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408224674022066,
+ "loss": 1.4259,
+ "theoretical_loss": 3.6475583222567938,
+ "tokens_seen": 1091698688
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005408064192577733,
+ "loss": 1.3465,
+ "theoretical_loss": 3.6475379811579325,
+ "tokens_seen": 1091764224
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.00054079037111334,
+ "loss": 1.5082,
+ "theoretical_loss": 3.647517641621927,
+ "tokens_seen": 1091829760
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005407743229689067,
+ "loss": 1.3053,
+ "theoretical_loss": 3.647497303648562,
+ "tokens_seen": 1091895296
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005407582748244734,
+ "loss": 1.3067,
+ "theoretical_loss": 3.647476967237625,
+ "tokens_seen": 1091960832
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005407422266800401,
+ "loss": 1.4078,
+ "theoretical_loss": 3.6474566323889017,
+ "tokens_seen": 1092026368
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005407261785356068,
+ "loss": 1.3382,
+ "theoretical_loss": 3.6474362991021785,
+ "tokens_seen": 1092091904
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005407101303911735,
+ "loss": 1.3884,
+ "theoretical_loss": 3.647415967377242,
+ "tokens_seen": 1092157440
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005406940822467402,
+ "loss": 1.3208,
+ "theoretical_loss": 3.6473956372138776,
+ "tokens_seen": 1092222976
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005406780341023069,
+ "loss": 1.2752,
+ "theoretical_loss": 3.6473753086118723,
+ "tokens_seen": 1092288512
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005406619859578736,
+ "loss": 1.2989,
+ "theoretical_loss": 3.647354981571013,
+ "tokens_seen": 1092354048
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005406459378134403,
+ "loss": 1.2932,
+ "theoretical_loss": 3.6473346560910853,
+ "tokens_seen": 1092419584
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540629889669007,
+ "loss": 1.3318,
+ "theoretical_loss": 3.6473143321718755,
+ "tokens_seen": 1092485120
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005406138415245737,
+ "loss": 1.37,
+ "theoretical_loss": 3.647294009813171,
+ "tokens_seen": 1092550656
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405977933801404,
+ "loss": 1.2931,
+ "theoretical_loss": 3.6472736890147583,
+ "tokens_seen": 1092616192
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405817452357071,
+ "loss": 1.3279,
+ "theoretical_loss": 3.647253369776424,
+ "tokens_seen": 1092681728
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405656970912738,
+ "loss": 1.4076,
+ "theoretical_loss": 3.6472330520979543,
+ "tokens_seen": 1092747264
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405496489468405,
+ "loss": 1.3562,
+ "theoretical_loss": 3.647212735979136,
+ "tokens_seen": 1092812800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405336008024072,
+ "loss": 1.3469,
+ "theoretical_loss": 3.647192421419756,
+ "tokens_seen": 1092878336
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540517552657974,
+ "loss": 1.4398,
+ "theoretical_loss": 3.6471721084196016,
+ "tokens_seen": 1092943872
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005405015045135406,
+ "loss": 1.2875,
+ "theoretical_loss": 3.6471517969784584,
+ "tokens_seen": 1093009408
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005404854563691074,
+ "loss": 1.2645,
+ "theoretical_loss": 3.6471314870961145,
+ "tokens_seen": 1093074944
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540469408224674,
+ "loss": 1.3876,
+ "theoretical_loss": 3.6471111787723567,
+ "tokens_seen": 1093140480
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005404533600802408,
+ "loss": 1.381,
+ "theoretical_loss": 3.647090872006971,
+ "tokens_seen": 1093206016
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005404373119358074,
+ "loss": 1.4007,
+ "theoretical_loss": 3.647070566799745,
+ "tokens_seen": 1093271552
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005404212637913742,
+ "loss": 1.3374,
+ "theoretical_loss": 3.6470502631504664,
+ "tokens_seen": 1093337088
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005404052156469408,
+ "loss": 1.3547,
+ "theoretical_loss": 3.647029961058921,
+ "tokens_seen": 1093402624
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005403891675025076,
+ "loss": 1.2061,
+ "theoretical_loss": 3.6470096605248976,
+ "tokens_seen": 1093468160
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005403731193580742,
+ "loss": 1.3681,
+ "theoretical_loss": 3.646989361548181,
+ "tokens_seen": 1093533696
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540357071213641,
+ "loss": 1.4661,
+ "theoretical_loss": 3.646969064128561,
+ "tokens_seen": 1093599232
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005403410230692076,
+ "loss": 1.251,
+ "theoretical_loss": 3.646948768265823,
+ "tokens_seen": 1093664768
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005403249749247744,
+ "loss": 1.3535,
+ "theoretical_loss": 3.646928473959755,
+ "tokens_seen": 1093730304
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540308926780341,
+ "loss": 1.4352,
+ "theoretical_loss": 3.6469081812101445,
+ "tokens_seen": 1093795840
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402928786359078,
+ "loss": 1.4161,
+ "theoretical_loss": 3.6468878900167785,
+ "tokens_seen": 1093861376
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402768304914744,
+ "loss": 1.4134,
+ "theoretical_loss": 3.6468676003794447,
+ "tokens_seen": 1093926912
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402607823470412,
+ "loss": 1.3992,
+ "theoretical_loss": 3.6468473122979304,
+ "tokens_seen": 1093992448
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402447342026078,
+ "loss": 1.3122,
+ "theoretical_loss": 3.6468270257720232,
+ "tokens_seen": 1094057984
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402286860581746,
+ "loss": 1.2352,
+ "theoretical_loss": 3.6468067408015106,
+ "tokens_seen": 1094123520
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005402126379137412,
+ "loss": 1.537,
+ "theoretical_loss": 3.64678645738618,
+ "tokens_seen": 1094189056
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540196589769308,
+ "loss": 1.4234,
+ "theoretical_loss": 3.6467661755258196,
+ "tokens_seen": 1094254592
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005401805416248746,
+ "loss": 1.3623,
+ "theoretical_loss": 3.6467458952202163,
+ "tokens_seen": 1094320128
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005401644934804414,
+ "loss": 1.2615,
+ "theoretical_loss": 3.6467256164691584,
+ "tokens_seen": 1094385664
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 368931,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4394853115081787,
+ "objective/train/theoretical_loss": 3.646705339272434,
+ "objective/train/tokens_used": 1114911200,
+ "theoretical_loss": 3.646705339272434,
+ "tokens_seen": 1094451200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540148445336008,
+ "loss": 1.3719,
+ "theoretical_loss": 3.646705339272434,
+ "tokens_seen": 1094451200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005401323971915748,
+ "loss": 1.42,
+ "theoretical_loss": 3.64668506362983,
+ "tokens_seen": 1094516736
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005401163490471414,
+ "loss": 1.4435,
+ "theoretical_loss": 3.646664789541134,
+ "tokens_seen": 1094582272
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005401003009027082,
+ "loss": 1.3787,
+ "theoretical_loss": 3.6466445170061355,
+ "tokens_seen": 1094647808
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005400842527582748,
+ "loss": 1.4719,
+ "theoretical_loss": 3.646624246024621,
+ "tokens_seen": 1094713344
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005400682046138416,
+ "loss": 1.3855,
+ "theoretical_loss": 3.6466039765963787,
+ "tokens_seen": 1094778880
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005400521564694082,
+ "loss": 1.3523,
+ "theoretical_loss": 3.646583708721197,
+ "tokens_seen": 1094844416
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000540036108324975,
+ "loss": 1.4296,
+ "theoretical_loss": 3.646563442398864,
+ "tokens_seen": 1094909952
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005400200601805416,
+ "loss": 1.4298,
+ "theoretical_loss": 3.6465431776291677,
+ "tokens_seen": 1094975488
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005400040120361084,
+ "loss": 1.4982,
+ "theoretical_loss": 3.6465229144118956,
+ "tokens_seen": 1095041024
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000539987963891675,
+ "loss": 1.4781,
+ "theoretical_loss": 3.6465026527468365,
+ "tokens_seen": 1095106560
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005399719157472418,
+ "loss": 1.4457,
+ "theoretical_loss": 3.6464823926337786,
+ "tokens_seen": 1095172096
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005399558676028084,
+ "loss": 1.2957,
+ "theoretical_loss": 3.64646213407251,
+ "tokens_seen": 1095237632
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005399398194583752,
+ "loss": 1.4198,
+ "theoretical_loss": 3.6464418770628186,
+ "tokens_seen": 1095303168
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005399237713139418,
+ "loss": 1.3637,
+ "theoretical_loss": 3.646421621604494,
+ "tokens_seen": 1095368704
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005399077231695086,
+ "loss": 1.3435,
+ "theoretical_loss": 3.6464013676973233,
+ "tokens_seen": 1095434240
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005398916750250752,
+ "loss": 1.374,
+ "theoretical_loss": 3.6463811153410948,
+ "tokens_seen": 1095499776
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000539875626880642,
+ "loss": 1.3455,
+ "theoretical_loss": 3.6463608645355983,
+ "tokens_seen": 1095565312
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005398595787362087,
+ "loss": 1.3548,
+ "theoretical_loss": 3.6463406152806215,
+ "tokens_seen": 1095630848
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005398435305917754,
+ "loss": 1.4417,
+ "theoretical_loss": 3.646320367575953,
+ "tokens_seen": 1095696384
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000539827482447342,
+ "loss": 1.2332,
+ "theoretical_loss": 3.6463001214213815,
+ "tokens_seen": 1095761920
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005398114343029088,
+ "loss": 1.359,
+ "theoretical_loss": 3.646279876816695,
+ "tokens_seen": 1095827456
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005397953861584755,
+ "loss": 1.2995,
+ "theoretical_loss": 3.646259633761683,
+ "tokens_seen": 1095892992
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005397793380140422,
+ "loss": 1.3454,
+ "theoretical_loss": 3.646239392256134,
+ "tokens_seen": 1095958528
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005397632898696089,
+ "loss": 1.3796,
+ "theoretical_loss": 3.646219152299836,
+ "tokens_seen": 1096024064
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005397472417251756,
+ "loss": 1.3976,
+ "theoretical_loss": 3.6461989138925794,
+ "tokens_seen": 1096089600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005397311935807423,
+ "loss": 1.4253,
+ "theoretical_loss": 3.6461786770341518,
+ "tokens_seen": 1096155136
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000539715145436309,
+ "loss": 1.3519,
+ "theoretical_loss": 3.6461584417243422,
+ "tokens_seen": 1096220672
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396990972918757,
+ "loss": 1.4295,
+ "theoretical_loss": 3.64613820796294,
+ "tokens_seen": 1096286208
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396830491474424,
+ "loss": 1.3605,
+ "theoretical_loss": 3.6461179757497337,
+ "tokens_seen": 1096351744
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396670010030091,
+ "loss": 1.4344,
+ "theoretical_loss": 3.6460977450845125,
+ "tokens_seen": 1096417280
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396509528585758,
+ "loss": 1.2537,
+ "theoretical_loss": 3.646077515967065,
+ "tokens_seen": 1096482816
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396349047141425,
+ "loss": 1.3941,
+ "theoretical_loss": 3.6460572883971816,
+ "tokens_seen": 1096548352
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396188565697092,
+ "loss": 1.4557,
+ "theoretical_loss": 3.6460370623746496,
+ "tokens_seen": 1096613888
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005396028084252759,
+ "loss": 1.3558,
+ "theoretical_loss": 3.6460168378992597,
+ "tokens_seen": 1096679424
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395867602808426,
+ "loss": 1.3014,
+ "theoretical_loss": 3.6459966149708,
+ "tokens_seen": 1096744960
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395707121364093,
+ "loss": 1.4605,
+ "theoretical_loss": 3.6459763935890606,
+ "tokens_seen": 1096810496
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395546639919759,
+ "loss": 1.3429,
+ "theoretical_loss": 3.6459561737538304,
+ "tokens_seen": 1096876032
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395386158475427,
+ "loss": 1.4886,
+ "theoretical_loss": 3.645935955464899,
+ "tokens_seen": 1096941568
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395225677031093,
+ "loss": 1.4814,
+ "theoretical_loss": 3.645915738722055,
+ "tokens_seen": 1097007104
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005395065195586761,
+ "loss": 1.3228,
+ "theoretical_loss": 3.645895523525089,
+ "tokens_seen": 1097072640
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394904714142427,
+ "loss": 1.2483,
+ "theoretical_loss": 3.6458753098737895,
+ "tokens_seen": 1097138176
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394744232698095,
+ "loss": 1.3837,
+ "theoretical_loss": 3.6458550977679467,
+ "tokens_seen": 1097203712
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394583751253761,
+ "loss": 1.3484,
+ "theoretical_loss": 3.645834887207349,
+ "tokens_seen": 1097269248
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394423269809429,
+ "loss": 1.3385,
+ "theoretical_loss": 3.6458146781917877,
+ "tokens_seen": 1097334784
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394262788365095,
+ "loss": 1.2497,
+ "theoretical_loss": 3.645794470721051,
+ "tokens_seen": 1097400320
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005394102306920762,
+ "loss": 1.3259,
+ "theoretical_loss": 3.6457742647949294,
+ "tokens_seen": 1097465856
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005393941825476429,
+ "loss": 1.407,
+ "theoretical_loss": 3.6457540604132115,
+ "tokens_seen": 1097531392
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005393781344032096,
+ "loss": 1.3469,
+ "theoretical_loss": 3.645733857575688,
+ "tokens_seen": 1097596928
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005393620862587763,
+ "loss": 1.3935,
+ "theoretical_loss": 3.6457136562821493,
+ "tokens_seen": 1097662464
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 370145,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3979780673980713,
+ "objective/train/theoretical_loss": 3.645693456532384,
+ "objective/train/tokens_used": 1118188000,
+ "theoretical_loss": 3.645693456532384,
+ "tokens_seen": 1097728000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000539346038114343,
+ "loss": 1.3858,
+ "theoretical_loss": 3.645693456532384,
+ "tokens_seen": 1097728000
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005393299899699097,
+ "loss": 1.3624,
+ "theoretical_loss": 3.645673258326182,
+ "tokens_seen": 1097793536
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005393139418254765,
+ "loss": 1.4369,
+ "theoretical_loss": 3.6456530616633334,
+ "tokens_seen": 1097859072
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392978936810431,
+ "loss": 1.479,
+ "theoretical_loss": 3.645632866543629,
+ "tokens_seen": 1097924608
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392818455366099,
+ "loss": 1.4302,
+ "theoretical_loss": 3.645612672966858,
+ "tokens_seen": 1097990144
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392657973921765,
+ "loss": 1.401,
+ "theoretical_loss": 3.6455924809328106,
+ "tokens_seen": 1098055680
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392497492477433,
+ "loss": 1.3846,
+ "theoretical_loss": 3.645572290441277,
+ "tokens_seen": 1098121216
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392337011033099,
+ "loss": 1.4016,
+ "theoretical_loss": 3.645552101492047,
+ "tokens_seen": 1098186752
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392176529588767,
+ "loss": 1.3517,
+ "theoretical_loss": 3.645531914084911,
+ "tokens_seen": 1098252288
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005392016048144433,
+ "loss": 1.4179,
+ "theoretical_loss": 3.6455117282196587,
+ "tokens_seen": 1098317824
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391855566700101,
+ "loss": 1.3205,
+ "theoretical_loss": 3.6454915438960813,
+ "tokens_seen": 1098383360
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391695085255767,
+ "loss": 1.3199,
+ "theoretical_loss": 3.6454713611139686,
+ "tokens_seen": 1098448896
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391534603811435,
+ "loss": 1.3495,
+ "theoretical_loss": 3.6454511798731106,
+ "tokens_seen": 1098514432
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391374122367101,
+ "loss": 1.2753,
+ "theoretical_loss": 3.645431000173298,
+ "tokens_seen": 1098579968
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391213640922769,
+ "loss": 1.4133,
+ "theoretical_loss": 3.6454108220143215,
+ "tokens_seen": 1098645504
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005391053159478435,
+ "loss": 1.388,
+ "theoretical_loss": 3.6453906453959704,
+ "tokens_seen": 1098711040
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390892678034103,
+ "loss": 1.4133,
+ "theoretical_loss": 3.6453704703180367,
+ "tokens_seen": 1098776576
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390732196589769,
+ "loss": 1.45,
+ "theoretical_loss": 3.6453502967803098,
+ "tokens_seen": 1098842112
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390571715145437,
+ "loss": 1.3928,
+ "theoretical_loss": 3.645330124782581,
+ "tokens_seen": 1098907648
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390411233701103,
+ "loss": 1.3588,
+ "theoretical_loss": 3.6453099543246403,
+ "tokens_seen": 1098973184
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390250752256771,
+ "loss": 1.4345,
+ "theoretical_loss": 3.6452897854062787,
+ "tokens_seen": 1099038720
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005390090270812437,
+ "loss": 1.3978,
+ "theoretical_loss": 3.645269618027287,
+ "tokens_seen": 1099104256
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389929789368105,
+ "loss": 1.3343,
+ "theoretical_loss": 3.6452494521874557,
+ "tokens_seen": 1099169792
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389769307923771,
+ "loss": 1.4268,
+ "theoretical_loss": 3.6452292878865755,
+ "tokens_seen": 1099235328
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389608826479439,
+ "loss": 1.3604,
+ "theoretical_loss": 3.6452091251244374,
+ "tokens_seen": 1099300864
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389448345035105,
+ "loss": 1.3272,
+ "theoretical_loss": 3.645188963900832,
+ "tokens_seen": 1099366400
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389287863590773,
+ "loss": 1.4702,
+ "theoretical_loss": 3.645168804215551,
+ "tokens_seen": 1099431936
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005389127382146439,
+ "loss": 1.3084,
+ "theoretical_loss": 3.6451486460683844,
+ "tokens_seen": 1099497472
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388966900702107,
+ "loss": 1.386,
+ "theoretical_loss": 3.645128489459123,
+ "tokens_seen": 1099563008
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388806419257773,
+ "loss": 1.2313,
+ "theoretical_loss": 3.645108334387559,
+ "tokens_seen": 1099628544
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388645937813441,
+ "loss": 1.4147,
+ "theoretical_loss": 3.6450881808534827,
+ "tokens_seen": 1099694080
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388485456369107,
+ "loss": 1.5094,
+ "theoretical_loss": 3.6450680288566852,
+ "tokens_seen": 1099759616
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388324974924775,
+ "loss": 1.2891,
+ "theoretical_loss": 3.6450478783969573,
+ "tokens_seen": 1099825152
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388164493480441,
+ "loss": 1.3042,
+ "theoretical_loss": 3.6450277294740916,
+ "tokens_seen": 1099890688
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005388004012036109,
+ "loss": 1.4374,
+ "theoretical_loss": 3.645007582087877,
+ "tokens_seen": 1099956224
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387843530591775,
+ "loss": 1.3827,
+ "theoretical_loss": 3.6449874362381074,
+ "tokens_seen": 1100021760
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387683049147443,
+ "loss": 1.3079,
+ "theoretical_loss": 3.644967291924572,
+ "tokens_seen": 1100087296
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387522567703109,
+ "loss": 1.3284,
+ "theoretical_loss": 3.6449471491470633,
+ "tokens_seen": 1100152832
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387362086258777,
+ "loss": 1.3271,
+ "theoretical_loss": 3.6449270079053724,
+ "tokens_seen": 1100218368
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387201604814443,
+ "loss": 1.4039,
+ "theoretical_loss": 3.64490686819929,
+ "tokens_seen": 1100283904
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005387041123370111,
+ "loss": 1.3387,
+ "theoretical_loss": 3.6448867300286087,
+ "tokens_seen": 1100349440
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386880641925777,
+ "loss": 1.3554,
+ "theoretical_loss": 3.64486659339312,
+ "tokens_seen": 1100414976
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386720160481445,
+ "loss": 1.4049,
+ "theoretical_loss": 3.6448464582926143,
+ "tokens_seen": 1100480512
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386559679037111,
+ "loss": 1.3117,
+ "theoretical_loss": 3.6448263247268846,
+ "tokens_seen": 1100546048
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386399197592779,
+ "loss": 1.3727,
+ "theoretical_loss": 3.6448061926957207,
+ "tokens_seen": 1100611584
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386238716148445,
+ "loss": 1.3975,
+ "theoretical_loss": 3.644786062198916,
+ "tokens_seen": 1100677120
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005386078234704113,
+ "loss": 1.3578,
+ "theoretical_loss": 3.6447659332362616,
+ "tokens_seen": 1100742656
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000538591775325978,
+ "loss": 1.3708,
+ "theoretical_loss": 3.6447458058075495,
+ "tokens_seen": 1100808192
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005385757271815447,
+ "loss": 1.4256,
+ "theoretical_loss": 3.644725679912571,
+ "tokens_seen": 1100873728
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005385596790371113,
+ "loss": 1.4461,
+ "theoretical_loss": 3.644705555551118,
+ "tokens_seen": 1100939264
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 370963,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3684258460998535,
+ "objective/train/theoretical_loss": 3.6446854327229827,
+ "objective/train/tokens_used": 1121464800,
+ "theoretical_loss": 3.6446854327229827,
+ "tokens_seen": 1101004800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005385436308926781,
+ "loss": 1.451,
+ "theoretical_loss": 3.6446854327229827,
+ "tokens_seen": 1101004800
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005385275827482447,
+ "loss": 1.3557,
+ "theoretical_loss": 3.6446653114279566,
+ "tokens_seen": 1101070336
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005385115346038115,
+ "loss": 1.3526,
+ "theoretical_loss": 3.6446451916658322,
+ "tokens_seen": 1101135872
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005384954864593782,
+ "loss": 1.4003,
+ "theoretical_loss": 3.6446250734364014,
+ "tokens_seen": 1101201408
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005384794383149449,
+ "loss": 1.4314,
+ "theoretical_loss": 3.644604956739456,
+ "tokens_seen": 1101266944
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005384633901705116,
+ "loss": 1.3957,
+ "theoretical_loss": 3.6445848415747877,
+ "tokens_seen": 1101332480
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005384473420260783,
+ "loss": 1.5251,
+ "theoretical_loss": 3.644564727942189,
+ "tokens_seen": 1101398016
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000538431293881645,
+ "loss": 1.4191,
+ "theoretical_loss": 3.6445446158414523,
+ "tokens_seen": 1101463552
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005384152457372117,
+ "loss": 1.3306,
+ "theoretical_loss": 3.6445245052723694,
+ "tokens_seen": 1101529088
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383991975927784,
+ "loss": 1.3011,
+ "theoretical_loss": 3.644504396234733,
+ "tokens_seen": 1101594624
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383831494483451,
+ "loss": 1.356,
+ "theoretical_loss": 3.6444842887283353,
+ "tokens_seen": 1101660160
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383671013039118,
+ "loss": 1.3998,
+ "theoretical_loss": 3.644464182752968,
+ "tokens_seen": 1101725696
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383510531594785,
+ "loss": 1.2681,
+ "theoretical_loss": 3.644444078308424,
+ "tokens_seen": 1101791232
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383350050150452,
+ "loss": 1.3577,
+ "theoretical_loss": 3.644423975394496,
+ "tokens_seen": 1101856768
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383189568706119,
+ "loss": 1.3414,
+ "theoretical_loss": 3.6444038740109757,
+ "tokens_seen": 1101922304
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005383029087261786,
+ "loss": 1.394,
+ "theoretical_loss": 3.6443837741576557,
+ "tokens_seen": 1101987840
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005382868605817453,
+ "loss": 1.3517,
+ "theoretical_loss": 3.644363675834329,
+ "tokens_seen": 1102053376
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000538270812437312,
+ "loss": 1.3735,
+ "theoretical_loss": 3.644343579040788,
+ "tokens_seen": 1102118912
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005382547642928787,
+ "loss": 1.2879,
+ "theoretical_loss": 3.6443234837768252,
+ "tokens_seen": 1102184448
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005382387161484454,
+ "loss": 1.2937,
+ "theoretical_loss": 3.644303390042233,
+ "tokens_seen": 1102249984
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005382226680040121,
+ "loss": 1.2894,
+ "theoretical_loss": 3.6442832978368047,
+ "tokens_seen": 1102315520
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005382066198595788,
+ "loss": 1.2497,
+ "theoretical_loss": 3.6442632071603325,
+ "tokens_seen": 1102381056
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005381905717151455,
+ "loss": 1.3784,
+ "theoretical_loss": 3.6442431180126094,
+ "tokens_seen": 1102446592
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005381745235707122,
+ "loss": 1.3875,
+ "theoretical_loss": 3.6442230303934284,
+ "tokens_seen": 1102512128
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005381584754262788,
+ "loss": 1.252,
+ "theoretical_loss": 3.6442029443025814,
+ "tokens_seen": 1102577664
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005381424272818456,
+ "loss": 1.3298,
+ "theoretical_loss": 3.6441828597398622,
+ "tokens_seen": 1102643200
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005381263791374122,
+ "loss": 1.3961,
+ "theoretical_loss": 3.6441627767050635,
+ "tokens_seen": 1102708736
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000538110330992979,
+ "loss": 1.3554,
+ "theoretical_loss": 3.6441426951979787,
+ "tokens_seen": 1102774272
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005380942828485456,
+ "loss": 1.3804,
+ "theoretical_loss": 3.6441226152184,
+ "tokens_seen": 1102839808
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005380782347041124,
+ "loss": 1.4734,
+ "theoretical_loss": 3.6441025367661206,
+ "tokens_seen": 1102905344
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000538062186559679,
+ "loss": 1.3292,
+ "theoretical_loss": 3.6440824598409347,
+ "tokens_seen": 1102970880
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005380461384152458,
+ "loss": 1.3139,
+ "theoretical_loss": 3.644062384442634,
+ "tokens_seen": 1103036416
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005380300902708124,
+ "loss": 1.2785,
+ "theoretical_loss": 3.6440423105710122,
+ "tokens_seen": 1103101952
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005380140421263792,
+ "loss": 1.4209,
+ "theoretical_loss": 3.6440222382258622,
+ "tokens_seen": 1103167488
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005379979939819458,
+ "loss": 1.4102,
+ "theoretical_loss": 3.644002167406978,
+ "tokens_seen": 1103233024
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005379819458375126,
+ "loss": 1.3151,
+ "theoretical_loss": 3.643982098114152,
+ "tokens_seen": 1103298560
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005379658976930792,
+ "loss": 1.4163,
+ "theoretical_loss": 3.6439620303471782,
+ "tokens_seen": 1103364096
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000537949849548646,
+ "loss": 1.3813,
+ "theoretical_loss": 3.64394196410585,
+ "tokens_seen": 1103429632
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005379338014042126,
+ "loss": 1.4278,
+ "theoretical_loss": 3.6439218993899605,
+ "tokens_seen": 1103495168
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005379177532597794,
+ "loss": 1.43,
+ "theoretical_loss": 3.643901836199303,
+ "tokens_seen": 1103560704
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000537901705115346,
+ "loss": 1.3511,
+ "theoretical_loss": 3.643881774533672,
+ "tokens_seen": 1103626240
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378856569709128,
+ "loss": 1.3673,
+ "theoretical_loss": 3.6438617143928598,
+ "tokens_seen": 1103691776
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378696088264794,
+ "loss": 1.4885,
+ "theoretical_loss": 3.6438416557766597,
+ "tokens_seen": 1103757312
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378535606820462,
+ "loss": 1.5448,
+ "theoretical_loss": 3.643821598684867,
+ "tokens_seen": 1103822848
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378375125376128,
+ "loss": 1.304,
+ "theoretical_loss": 3.6438015431172737,
+ "tokens_seen": 1103888384
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378214643931796,
+ "loss": 1.3643,
+ "theoretical_loss": 3.6437814890736746,
+ "tokens_seen": 1103953920
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005378054162487462,
+ "loss": 1.3302,
+ "theoretical_loss": 3.643761436553863,
+ "tokens_seen": 1104019456
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000537789368104313,
+ "loss": 1.3877,
+ "theoretical_loss": 3.6437413855576324,
+ "tokens_seen": 1104084992
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005377733199598796,
+ "loss": 1.3457,
+ "theoretical_loss": 3.643721336084777,
+ "tokens_seen": 1104150528
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005377572718154464,
+ "loss": 1.434,
+ "theoretical_loss": 3.6437012881350905,
+ "tokens_seen": 1104216064
+ },
+ {
+ "epoch": 0.33,
+ "objective/train/docs_used": 372202,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2808032035827637,
+ "objective/train/theoretical_loss": 3.643681241708367,
+ "objective/train/tokens_used": 1124741600,
+ "theoretical_loss": 3.643681241708367,
+ "tokens_seen": 1104281600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000537741223671013,
+ "loss": 1.3161,
+ "theoretical_loss": 3.643681241708367,
+ "tokens_seen": 1104281600
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005377251755265798,
+ "loss": 1.3353,
+ "theoretical_loss": 3.6436611968044,
+ "tokens_seen": 1104347136
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005377091273821464,
+ "loss": 1.4238,
+ "theoretical_loss": 3.6436411534229842,
+ "tokens_seen": 1104412672
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005376930792377132,
+ "loss": 1.2942,
+ "theoretical_loss": 3.6436211115639128,
+ "tokens_seen": 1104478208
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005376770310932798,
+ "loss": 1.2785,
+ "theoretical_loss": 3.64360107122698,
+ "tokens_seen": 1104543744
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005376609829488466,
+ "loss": 1.4836,
+ "theoretical_loss": 3.643581032411981,
+ "tokens_seen": 1104609280
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005376449348044132,
+ "loss": 1.3562,
+ "theoretical_loss": 3.6435609951187082,
+ "tokens_seen": 1104674816
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.00053762888665998,
+ "loss": 1.4225,
+ "theoretical_loss": 3.643540959346957,
+ "tokens_seen": 1104740352
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005376128385155466,
+ "loss": 1.3835,
+ "theoretical_loss": 3.643520925096521,
+ "tokens_seen": 1104805888
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375967903711134,
+ "loss": 1.4524,
+ "theoretical_loss": 3.6435008923671948,
+ "tokens_seen": 1104871424
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.00053758074222668,
+ "loss": 1.4,
+ "theoretical_loss": 3.6434808611587726,
+ "tokens_seen": 1104936960
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375646940822468,
+ "loss": 1.3913,
+ "theoretical_loss": 3.6434608314710486,
+ "tokens_seen": 1105002496
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375486459378134,
+ "loss": 1.3233,
+ "theoretical_loss": 3.6434408033038173,
+ "tokens_seen": 1105068032
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375325977933802,
+ "loss": 1.3725,
+ "theoretical_loss": 3.6434207766568734,
+ "tokens_seen": 1105133568
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375165496489468,
+ "loss": 1.4318,
+ "theoretical_loss": 3.643400751530011,
+ "tokens_seen": 1105199104
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005375005015045136,
+ "loss": 1.3133,
+ "theoretical_loss": 3.6433807279230246,
+ "tokens_seen": 1105264640
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005374844533600802,
+ "loss": 1.4323,
+ "theoretical_loss": 3.6433607058357085,
+ "tokens_seen": 1105330176
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.000537468405215647,
+ "loss": 1.3564,
+ "theoretical_loss": 3.643340685267858,
+ "tokens_seen": 1105395712
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 0.0005374523570712136,
+ "loss": 1.3261,
+ "theoretical_loss": 3.6433206662192665,
+ "tokens_seen": 1105461248
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005374363089267804,
+ "loss": 1.393,
+ "theoretical_loss": 3.6433006486897304,
+ "tokens_seen": 1105526784
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000537420260782347,
+ "loss": 1.2428,
+ "theoretical_loss": 3.6432806326790423,
+ "tokens_seen": 1105592320
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005374042126379138,
+ "loss": 1.407,
+ "theoretical_loss": 3.643260618186999,
+ "tokens_seen": 1105657856
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005373881644934804,
+ "loss": 1.3188,
+ "theoretical_loss": 3.643240605213394,
+ "tokens_seen": 1105723392
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005373721163490472,
+ "loss": 1.4036,
+ "theoretical_loss": 3.6432205937580227,
+ "tokens_seen": 1105788928
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005373560682046138,
+ "loss": 1.4072,
+ "theoretical_loss": 3.643200583820679,
+ "tokens_seen": 1105854464
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005373400200601806,
+ "loss": 1.3953,
+ "theoretical_loss": 3.6431805754011592,
+ "tokens_seen": 1105920000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005373239719157472,
+ "loss": 1.382,
+ "theoretical_loss": 3.6431605684992574,
+ "tokens_seen": 1105985536
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000537307923771314,
+ "loss": 1.4688,
+ "theoretical_loss": 3.6431405631147684,
+ "tokens_seen": 1106051072
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005372918756268806,
+ "loss": 1.3794,
+ "theoretical_loss": 3.6431205592474876,
+ "tokens_seen": 1106116608
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005372758274824474,
+ "loss": 1.361,
+ "theoretical_loss": 3.64310055689721,
+ "tokens_seen": 1106182144
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000537259779338014,
+ "loss": 1.3768,
+ "theoretical_loss": 3.6430805560637305,
+ "tokens_seen": 1106247680
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005372437311935808,
+ "loss": 1.3212,
+ "theoretical_loss": 3.643060556746845,
+ "tokens_seen": 1106313216
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005372276830491474,
+ "loss": 1.2949,
+ "theoretical_loss": 3.643040558946347,
+ "tokens_seen": 1106378752
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005372116349047142,
+ "loss": 1.3277,
+ "theoretical_loss": 3.643020562662034,
+ "tokens_seen": 1106444288
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005371955867602808,
+ "loss": 1.4497,
+ "theoretical_loss": 3.643000567893699,
+ "tokens_seen": 1106509824
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005371795386158476,
+ "loss": 1.3987,
+ "theoretical_loss": 3.642980574641139,
+ "tokens_seen": 1106575360
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005371634904714142,
+ "loss": 1.3366,
+ "theoretical_loss": 3.6429605829041476,
+ "tokens_seen": 1106640896
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000537147442326981,
+ "loss": 1.4619,
+ "theoretical_loss": 3.642940592682522,
+ "tokens_seen": 1106706432
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005371313941825477,
+ "loss": 1.4139,
+ "theoretical_loss": 3.642920603976057,
+ "tokens_seen": 1106771968
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005371153460381144,
+ "loss": 1.3719,
+ "theoretical_loss": 3.6429006167845475,
+ "tokens_seen": 1106837504
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000537099297893681,
+ "loss": 1.3862,
+ "theoretical_loss": 3.642880631107789,
+ "tokens_seen": 1106903040
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370832497492478,
+ "loss": 1.4111,
+ "theoretical_loss": 3.642860646945578,
+ "tokens_seen": 1106968576
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370672016048145,
+ "loss": 1.2819,
+ "theoretical_loss": 3.64284066429771,
+ "tokens_seen": 1107034112
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370511534603812,
+ "loss": 1.2746,
+ "theoretical_loss": 3.642820683163979,
+ "tokens_seen": 1107099648
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370351053159479,
+ "loss": 1.3164,
+ "theoretical_loss": 3.6428007035441823,
+ "tokens_seen": 1107165184
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370190571715146,
+ "loss": 1.3168,
+ "theoretical_loss": 3.6427807254381146,
+ "tokens_seen": 1107230720
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005370030090270813,
+ "loss": 1.3052,
+ "theoretical_loss": 3.642760748845572,
+ "tokens_seen": 1107296256
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000536986960882648,
+ "loss": 1.4019,
+ "theoretical_loss": 3.6427407737663504,
+ "tokens_seen": 1107361792
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005369709127382147,
+ "loss": 1.3857,
+ "theoretical_loss": 3.642720800200246,
+ "tokens_seen": 1107427328
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005369548645937814,
+ "loss": 1.3194,
+ "theoretical_loss": 3.6427008281470536,
+ "tokens_seen": 1107492864
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 373185,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4264893531799316,
+ "objective/train/theoretical_loss": 3.64268085760657,
+ "objective/train/tokens_used": 1128018400,
+ "theoretical_loss": 3.64268085760657,
+ "tokens_seen": 1107558400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005369388164493481,
+ "loss": 1.3387,
+ "theoretical_loss": 3.64268085760657,
+ "tokens_seen": 1107558400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005369227683049148,
+ "loss": 1.2995,
+ "theoretical_loss": 3.6426608885785905,
+ "tokens_seen": 1107623936
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005369067201604815,
+ "loss": 1.3299,
+ "theoretical_loss": 3.6426409210629114,
+ "tokens_seen": 1107689472
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005368906720160482,
+ "loss": 1.4579,
+ "theoretical_loss": 3.6426209550593285,
+ "tokens_seen": 1107755008
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005368746238716149,
+ "loss": 1.325,
+ "theoretical_loss": 3.6426009905676384,
+ "tokens_seen": 1107820544
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005368585757271816,
+ "loss": 1.4244,
+ "theoretical_loss": 3.642581027587636,
+ "tokens_seen": 1107886080
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005368425275827483,
+ "loss": 1.4572,
+ "theoretical_loss": 3.6425610661191192,
+ "tokens_seen": 1107951616
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000536826479438315,
+ "loss": 1.3712,
+ "theoretical_loss": 3.6425411061618824,
+ "tokens_seen": 1108017152
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005368104312938817,
+ "loss": 1.4417,
+ "theoretical_loss": 3.6425211477157227,
+ "tokens_seen": 1108082688
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367943831494484,
+ "loss": 1.4092,
+ "theoretical_loss": 3.6425011907804365,
+ "tokens_seen": 1108148224
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367783350050151,
+ "loss": 1.4101,
+ "theoretical_loss": 3.64248123535582,
+ "tokens_seen": 1108213760
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367622868605817,
+ "loss": 1.3931,
+ "theoretical_loss": 3.6424612814416686,
+ "tokens_seen": 1108279296
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367462387161485,
+ "loss": 1.3125,
+ "theoretical_loss": 3.64244132903778,
+ "tokens_seen": 1108344832
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367301905717151,
+ "loss": 1.3324,
+ "theoretical_loss": 3.6424213781439496,
+ "tokens_seen": 1108410368
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005367141424272819,
+ "loss": 1.3561,
+ "theoretical_loss": 3.642401428759974,
+ "tokens_seen": 1108475904
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366980942828485,
+ "loss": 1.3003,
+ "theoretical_loss": 3.6423814808856507,
+ "tokens_seen": 1108541440
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366820461384153,
+ "loss": 1.299,
+ "theoretical_loss": 3.6423615345207745,
+ "tokens_seen": 1108606976
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366659979939819,
+ "loss": 1.502,
+ "theoretical_loss": 3.6423415896651434,
+ "tokens_seen": 1108672512
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366499498495487,
+ "loss": 1.2873,
+ "theoretical_loss": 3.6423216463185533,
+ "tokens_seen": 1108738048
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366339017051153,
+ "loss": 1.2303,
+ "theoretical_loss": 3.642301704480801,
+ "tokens_seen": 1108803584
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366178535606821,
+ "loss": 1.4252,
+ "theoretical_loss": 3.642281764151683,
+ "tokens_seen": 1108869120
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005366018054162487,
+ "loss": 1.3477,
+ "theoretical_loss": 3.642261825330997,
+ "tokens_seen": 1108934656
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365857572718155,
+ "loss": 1.4279,
+ "theoretical_loss": 3.642241888018538,
+ "tokens_seen": 1109000192
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365697091273821,
+ "loss": 1.4184,
+ "theoretical_loss": 3.642221952214104,
+ "tokens_seen": 1109065728
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365536609829489,
+ "loss": 1.4077,
+ "theoretical_loss": 3.6422020179174917,
+ "tokens_seen": 1109131264
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365376128385155,
+ "loss": 1.333,
+ "theoretical_loss": 3.642182085128498,
+ "tokens_seen": 1109196800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365215646940823,
+ "loss": 1.3556,
+ "theoretical_loss": 3.642162153846919,
+ "tokens_seen": 1109262336
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005365055165496489,
+ "loss": 1.4242,
+ "theoretical_loss": 3.642142224072553,
+ "tokens_seen": 1109327872
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364894684052157,
+ "loss": 1.432,
+ "theoretical_loss": 3.642122295805197,
+ "tokens_seen": 1109393408
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364734202607823,
+ "loss": 1.3803,
+ "theoretical_loss": 3.642102369044646,
+ "tokens_seen": 1109458944
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364573721163491,
+ "loss": 1.3278,
+ "theoretical_loss": 3.642082443790699,
+ "tokens_seen": 1109524480
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364413239719157,
+ "loss": 1.3702,
+ "theoretical_loss": 3.6420625200431527,
+ "tokens_seen": 1109590016
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364252758274825,
+ "loss": 1.3172,
+ "theoretical_loss": 3.6420425978018036,
+ "tokens_seen": 1109655552
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005364092276830491,
+ "loss": 1.3497,
+ "theoretical_loss": 3.64202267706645,
+ "tokens_seen": 1109721088
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363931795386159,
+ "loss": 1.3812,
+ "theoretical_loss": 3.642002757836888,
+ "tokens_seen": 1109786624
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363771313941825,
+ "loss": 1.4259,
+ "theoretical_loss": 3.6419828401129153,
+ "tokens_seen": 1109852160
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363610832497493,
+ "loss": 1.367,
+ "theoretical_loss": 3.6419629238943294,
+ "tokens_seen": 1109917696
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363450351053159,
+ "loss": 1.3267,
+ "theoretical_loss": 3.6419430091809275,
+ "tokens_seen": 1109983232
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363289869608827,
+ "loss": 1.3759,
+ "theoretical_loss": 3.641923095972507,
+ "tokens_seen": 1110048768
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005363129388164493,
+ "loss": 1.4187,
+ "theoretical_loss": 3.641903184268865,
+ "tokens_seen": 1110114304
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362968906720161,
+ "loss": 1.3049,
+ "theoretical_loss": 3.6418832740698,
+ "tokens_seen": 1110179840
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362808425275827,
+ "loss": 1.3543,
+ "theoretical_loss": 3.6418633653751087,
+ "tokens_seen": 1110245376
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362647943831495,
+ "loss": 1.3798,
+ "theoretical_loss": 3.6418434581845887,
+ "tokens_seen": 1110310912
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362487462387161,
+ "loss": 1.3423,
+ "theoretical_loss": 3.641823552498037,
+ "tokens_seen": 1110376448
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362326980942829,
+ "loss": 1.3781,
+ "theoretical_loss": 3.6418036483152525,
+ "tokens_seen": 1110441984
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362166499498495,
+ "loss": 1.4235,
+ "theoretical_loss": 3.641783745636032,
+ "tokens_seen": 1110507520
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005362006018054163,
+ "loss": 1.3896,
+ "theoretical_loss": 3.641763844460173,
+ "tokens_seen": 1110573056
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361845536609829,
+ "loss": 1.4963,
+ "theoretical_loss": 3.641743944787474,
+ "tokens_seen": 1110638592
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361685055165497,
+ "loss": 1.4341,
+ "theoretical_loss": 3.6417240466177327,
+ "tokens_seen": 1110704128
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361524573721163,
+ "loss": 1.4107,
+ "theoretical_loss": 3.641704149950746,
+ "tokens_seen": 1110769664
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 374395,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.658529281616211,
+ "objective/train/theoretical_loss": 3.6416842547863126,
+ "objective/train/tokens_used": 1131295200,
+ "theoretical_loss": 3.6416842547863126,
+ "tokens_seen": 1110835200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361364092276831,
+ "loss": 1.5326,
+ "theoretical_loss": 3.6416842547863126,
+ "tokens_seen": 1110835200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361203610832497,
+ "loss": 1.4379,
+ "theoretical_loss": 3.6416643611242305,
+ "tokens_seen": 1110900736
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005361043129388165,
+ "loss": 1.4246,
+ "theoretical_loss": 3.641644468964297,
+ "tokens_seen": 1110966272
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360882647943831,
+ "loss": 1.3521,
+ "theoretical_loss": 3.6416245783063106,
+ "tokens_seen": 1111031808
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360722166499499,
+ "loss": 1.4709,
+ "theoretical_loss": 3.6416046891500686,
+ "tokens_seen": 1111097344
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360561685055165,
+ "loss": 1.2381,
+ "theoretical_loss": 3.64158480149537,
+ "tokens_seen": 1111162880
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360401203610833,
+ "loss": 1.4248,
+ "theoretical_loss": 3.641564915342012,
+ "tokens_seen": 1111228416
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360240722166499,
+ "loss": 1.377,
+ "theoretical_loss": 3.6415450306897936,
+ "tokens_seen": 1111293952
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005360080240722167,
+ "loss": 1.3799,
+ "theoretical_loss": 3.6415251475385126,
+ "tokens_seen": 1111359488
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359919759277833,
+ "loss": 1.4776,
+ "theoretical_loss": 3.6415052658879667,
+ "tokens_seen": 1111425024
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359759277833501,
+ "loss": 1.3046,
+ "theoretical_loss": 3.6414853857379548,
+ "tokens_seen": 1111490560
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359598796389167,
+ "loss": 1.3972,
+ "theoretical_loss": 3.641465507088275,
+ "tokens_seen": 1111556096
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359438314944835,
+ "loss": 1.3837,
+ "theoretical_loss": 3.6414456299387257,
+ "tokens_seen": 1111621632
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359277833500501,
+ "loss": 1.4221,
+ "theoretical_loss": 3.641425754289105,
+ "tokens_seen": 1111687168
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005359117352056169,
+ "loss": 1.348,
+ "theoretical_loss": 3.641405880139211,
+ "tokens_seen": 1111752704
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005358956870611835,
+ "loss": 1.3488,
+ "theoretical_loss": 3.641386007488843,
+ "tokens_seen": 1111818240
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005358796389167503,
+ "loss": 1.3655,
+ "theoretical_loss": 3.6413661363377994,
+ "tokens_seen": 1111883776
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535863590772317,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6413462666858782,
+ "tokens_seen": 1111949312
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005358475426278837,
+ "loss": 1.3838,
+ "theoretical_loss": 3.641326398532878,
+ "tokens_seen": 1112014848
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005358314944834503,
+ "loss": 1.3424,
+ "theoretical_loss": 3.6413065318785978,
+ "tokens_seen": 1112080384
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005358154463390171,
+ "loss": 1.3995,
+ "theoretical_loss": 3.6412866667228356,
+ "tokens_seen": 1112145920
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357993981945837,
+ "loss": 1.3884,
+ "theoretical_loss": 3.641266803065391,
+ "tokens_seen": 1112211456
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357833500501505,
+ "loss": 1.5106,
+ "theoretical_loss": 3.6412469409060617,
+ "tokens_seen": 1112276992
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357673019057172,
+ "loss": 1.2812,
+ "theoretical_loss": 3.641227080244647,
+ "tokens_seen": 1112342528
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357512537612839,
+ "loss": 1.3918,
+ "theoretical_loss": 3.6412072210809456,
+ "tokens_seen": 1112408064
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357352056168507,
+ "loss": 1.3502,
+ "theoretical_loss": 3.6411873634147565,
+ "tokens_seen": 1112473600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357191574724173,
+ "loss": 1.4278,
+ "theoretical_loss": 3.6411675072458785,
+ "tokens_seen": 1112539136
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005357031093279841,
+ "loss": 1.3277,
+ "theoretical_loss": 3.64114765257411,
+ "tokens_seen": 1112604672
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356870611835507,
+ "loss": 1.3654,
+ "theoretical_loss": 3.641127799399251,
+ "tokens_seen": 1112670208
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356710130391175,
+ "loss": 1.3517,
+ "theoretical_loss": 3.641107947721099,
+ "tokens_seen": 1112735744
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356549648946841,
+ "loss": 1.3816,
+ "theoretical_loss": 3.641088097539454,
+ "tokens_seen": 1112801280
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356389167502509,
+ "loss": 1.293,
+ "theoretical_loss": 3.6410682488541157,
+ "tokens_seen": 1112866816
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356228686058175,
+ "loss": 1.2678,
+ "theoretical_loss": 3.641048401664882,
+ "tokens_seen": 1112932352
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005356068204613843,
+ "loss": 1.4152,
+ "theoretical_loss": 3.6410285559715523,
+ "tokens_seen": 1112997888
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355907723169509,
+ "loss": 1.377,
+ "theoretical_loss": 3.6410087117739263,
+ "tokens_seen": 1113063424
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355747241725177,
+ "loss": 1.333,
+ "theoretical_loss": 3.6409888690718026,
+ "tokens_seen": 1113128960
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355586760280843,
+ "loss": 1.3061,
+ "theoretical_loss": 3.640969027864981,
+ "tokens_seen": 1113194496
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355426278836511,
+ "loss": 1.4545,
+ "theoretical_loss": 3.6409491881532605,
+ "tokens_seen": 1113260032
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355265797392177,
+ "loss": 1.356,
+ "theoretical_loss": 3.64092934993644,
+ "tokens_seen": 1113325568
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005355105315947845,
+ "loss": 1.3286,
+ "theoretical_loss": 3.64090951321432,
+ "tokens_seen": 1113391104
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354944834503511,
+ "loss": 1.4135,
+ "theoretical_loss": 3.6408896779866993,
+ "tokens_seen": 1113456640
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354784353059178,
+ "loss": 1.3987,
+ "theoretical_loss": 3.640869844253377,
+ "tokens_seen": 1113522176
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354623871614845,
+ "loss": 1.3355,
+ "theoretical_loss": 3.6408500120141527,
+ "tokens_seen": 1113587712
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354463390170512,
+ "loss": 1.4287,
+ "theoretical_loss": 3.640830181268826,
+ "tokens_seen": 1113653248
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354302908726179,
+ "loss": 1.465,
+ "theoretical_loss": 3.6408103520171973,
+ "tokens_seen": 1113718784
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005354142427281846,
+ "loss": 1.3813,
+ "theoretical_loss": 3.6407905242590646,
+ "tokens_seen": 1113784320
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005353981945837513,
+ "loss": 1.3069,
+ "theoretical_loss": 3.6407706979942294,
+ "tokens_seen": 1113849856
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535382146439318,
+ "loss": 1.3965,
+ "theoretical_loss": 3.6407508732224896,
+ "tokens_seen": 1113915392
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005353660982948846,
+ "loss": 1.4686,
+ "theoretical_loss": 3.6407310499436463,
+ "tokens_seen": 1113980928
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005353500501504514,
+ "loss": 1.4068,
+ "theoretical_loss": 3.6407112281574987,
+ "tokens_seen": 1114046464
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 375251,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4065958261489868,
+ "objective/train/theoretical_loss": 3.640691407863846,
+ "objective/train/tokens_used": 1134572000,
+ "theoretical_loss": 3.640691407863846,
+ "tokens_seen": 1114112000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535334002006018,
+ "loss": 1.3989,
+ "theoretical_loss": 3.640691407863846,
+ "tokens_seen": 1114112000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005353179538615848,
+ "loss": 1.4307,
+ "theoretical_loss": 3.6406715890624897,
+ "tokens_seen": 1114177536
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005353019057171514,
+ "loss": 1.4326,
+ "theoretical_loss": 3.6406517717532276,
+ "tokens_seen": 1114243072
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005352858575727182,
+ "loss": 1.2901,
+ "theoretical_loss": 3.640631955935861,
+ "tokens_seen": 1114308608
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005352698094282848,
+ "loss": 1.4398,
+ "theoretical_loss": 3.6406121416101893,
+ "tokens_seen": 1114374144
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005352537612838516,
+ "loss": 1.4284,
+ "theoretical_loss": 3.6405923287760134,
+ "tokens_seen": 1114439680
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005352377131394182,
+ "loss": 1.2453,
+ "theoretical_loss": 3.6405725174331316,
+ "tokens_seen": 1114505216
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535221664994985,
+ "loss": 1.2765,
+ "theoretical_loss": 3.640552707581346,
+ "tokens_seen": 1114570752
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005352056168505516,
+ "loss": 1.3611,
+ "theoretical_loss": 3.640532899220455,
+ "tokens_seen": 1114636288
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005351895687061184,
+ "loss": 1.3387,
+ "theoretical_loss": 3.6405130923502593,
+ "tokens_seen": 1114701824
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535173520561685,
+ "loss": 1.3431,
+ "theoretical_loss": 3.6404932869705595,
+ "tokens_seen": 1114767360
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005351574724172518,
+ "loss": 1.4032,
+ "theoretical_loss": 3.6404734830811556,
+ "tokens_seen": 1114832896
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005351414242728184,
+ "loss": 1.3052,
+ "theoretical_loss": 3.6404536806818477,
+ "tokens_seen": 1114898432
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005351253761283852,
+ "loss": 1.314,
+ "theoretical_loss": 3.640433879772436,
+ "tokens_seen": 1114963968
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005351093279839518,
+ "loss": 1.2596,
+ "theoretical_loss": 3.6404140803527216,
+ "tokens_seen": 1115029504
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005350932798395186,
+ "loss": 1.3923,
+ "theoretical_loss": 3.640394282422504,
+ "tokens_seen": 1115095040
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005350772316950852,
+ "loss": 1.3095,
+ "theoretical_loss": 3.6403744859815843,
+ "tokens_seen": 1115160576
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535061183550652,
+ "loss": 1.3673,
+ "theoretical_loss": 3.6403546910297617,
+ "tokens_seen": 1115226112
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005350451354062186,
+ "loss": 1.229,
+ "theoretical_loss": 3.6403348975668384,
+ "tokens_seen": 1115291648
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005350290872617854,
+ "loss": 1.4101,
+ "theoretical_loss": 3.640315105592614,
+ "tokens_seen": 1115357184
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000535013039117352,
+ "loss": 1.3442,
+ "theoretical_loss": 3.640295315106889,
+ "tokens_seen": 1115422720
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349969909729188,
+ "loss": 1.3302,
+ "theoretical_loss": 3.640275526109464,
+ "tokens_seen": 1115488256
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349809428284854,
+ "loss": 1.3207,
+ "theoretical_loss": 3.64025573860014,
+ "tokens_seen": 1115553792
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349648946840522,
+ "loss": 1.4234,
+ "theoretical_loss": 3.640235952578718,
+ "tokens_seen": 1115619328
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349488465396188,
+ "loss": 1.3565,
+ "theoretical_loss": 3.640216168044997,
+ "tokens_seen": 1115684864
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349327983951856,
+ "loss": 1.3293,
+ "theoretical_loss": 3.64019638499878,
+ "tokens_seen": 1115750400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005349167502507522,
+ "loss": 1.3408,
+ "theoretical_loss": 3.640176603439867,
+ "tokens_seen": 1115815936
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534900702106319,
+ "loss": 1.3538,
+ "theoretical_loss": 3.640156823368058,
+ "tokens_seen": 1115881472
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005348846539618856,
+ "loss": 1.3844,
+ "theoretical_loss": 3.6401370447831547,
+ "tokens_seen": 1115947008
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005348686058174524,
+ "loss": 1.3133,
+ "theoretical_loss": 3.640117267684958,
+ "tokens_seen": 1116012544
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534852557673019,
+ "loss": 1.4505,
+ "theoretical_loss": 3.6400974920732683,
+ "tokens_seen": 1116078080
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005348365095285858,
+ "loss": 1.381,
+ "theoretical_loss": 3.6400777179478876,
+ "tokens_seen": 1116143616
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005348204613841524,
+ "loss": 1.3998,
+ "theoretical_loss": 3.640057945308616,
+ "tokens_seen": 1116209152
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005348044132397192,
+ "loss": 1.3332,
+ "theoretical_loss": 3.6400381741552548,
+ "tokens_seen": 1116274688
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005347883650952858,
+ "loss": 1.3156,
+ "theoretical_loss": 3.6400184044876056,
+ "tokens_seen": 1116340224
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005347723169508526,
+ "loss": 1.3118,
+ "theoretical_loss": 3.6399986363054686,
+ "tokens_seen": 1116405760
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005347562688064192,
+ "loss": 1.3324,
+ "theoretical_loss": 3.6399788696086457,
+ "tokens_seen": 1116471296
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534740220661986,
+ "loss": 1.2942,
+ "theoretical_loss": 3.639959104396938,
+ "tokens_seen": 1116536832
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005347241725175526,
+ "loss": 1.4229,
+ "theoretical_loss": 3.6399393406701464,
+ "tokens_seen": 1116602368
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005347081243731194,
+ "loss": 1.3852,
+ "theoretical_loss": 3.639919578428073,
+ "tokens_seen": 1116667904
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534692076228686,
+ "loss": 1.3425,
+ "theoretical_loss": 3.639899817670518,
+ "tokens_seen": 1116733440
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005346760280842528,
+ "loss": 1.4459,
+ "theoretical_loss": 3.639880058397284,
+ "tokens_seen": 1116798976
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005346599799398194,
+ "loss": 1.2834,
+ "theoretical_loss": 3.639860300608172,
+ "tokens_seen": 1116864512
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005346439317953862,
+ "loss": 1.2596,
+ "theoretical_loss": 3.6398405443029827,
+ "tokens_seen": 1116930048
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005346278836509528,
+ "loss": 1.3818,
+ "theoretical_loss": 3.6398207894815178,
+ "tokens_seen": 1116995584
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005346118355065196,
+ "loss": 1.3449,
+ "theoretical_loss": 3.63980103614358,
+ "tokens_seen": 1117061120
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005345957873620862,
+ "loss": 1.4162,
+ "theoretical_loss": 3.6397812842889694,
+ "tokens_seen": 1117126656
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534579739217653,
+ "loss": 1.3391,
+ "theoretical_loss": 3.639761533917489,
+ "tokens_seen": 1117192192
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005345636910732196,
+ "loss": 1.3396,
+ "theoretical_loss": 3.639741785028939,
+ "tokens_seen": 1117257728
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005345476429287864,
+ "loss": 1.3862,
+ "theoretical_loss": 3.6397220376231223,
+ "tokens_seen": 1117323264
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 376505,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3129369020462036,
+ "objective/train/theoretical_loss": 3.63970229169984,
+ "objective/train/tokens_used": 1137848800,
+ "theoretical_loss": 3.63970229169984,
+ "tokens_seen": 1117388800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005345315947843532,
+ "loss": 1.3009,
+ "theoretical_loss": 3.63970229169984,
+ "tokens_seen": 1117388800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005345155466399198,
+ "loss": 1.4542,
+ "theoretical_loss": 3.639682547258894,
+ "tokens_seen": 1117454336
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005344994984954866,
+ "loss": 1.4217,
+ "theoretical_loss": 3.6396628043000856,
+ "tokens_seen": 1117519872
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005344834503510532,
+ "loss": 1.3604,
+ "theoretical_loss": 3.6396430628232173,
+ "tokens_seen": 1117585408
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.00053446740220662,
+ "loss": 1.3649,
+ "theoretical_loss": 3.6396233228280908,
+ "tokens_seen": 1117650944
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005344513540621866,
+ "loss": 1.4095,
+ "theoretical_loss": 3.639603584314508,
+ "tokens_seen": 1117716480
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005344353059177534,
+ "loss": 1.3334,
+ "theoretical_loss": 3.639583847282271,
+ "tokens_seen": 1117782016
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.00053441925777332,
+ "loss": 1.3851,
+ "theoretical_loss": 3.6395641117311817,
+ "tokens_seen": 1117847552
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005344032096288868,
+ "loss": 1.2414,
+ "theoretical_loss": 3.6395443776610414,
+ "tokens_seen": 1117913088
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005343871614844534,
+ "loss": 1.3993,
+ "theoretical_loss": 3.6395246450716527,
+ "tokens_seen": 1117978624
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005343711133400202,
+ "loss": 1.3598,
+ "theoretical_loss": 3.6395049139628184,
+ "tokens_seen": 1118044160
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005343550651955868,
+ "loss": 1.415,
+ "theoretical_loss": 3.63948518433434,
+ "tokens_seen": 1118109696
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005343390170511536,
+ "loss": 1.4148,
+ "theoretical_loss": 3.639465456186019,
+ "tokens_seen": 1118175232
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005343229689067202,
+ "loss": 1.3158,
+ "theoretical_loss": 3.639445729517659,
+ "tokens_seen": 1118240768
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534306920762287,
+ "loss": 1.2983,
+ "theoretical_loss": 3.6394260043290614,
+ "tokens_seen": 1118306304
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005342908726178536,
+ "loss": 1.33,
+ "theoretical_loss": 3.6394062806200287,
+ "tokens_seen": 1118371840
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005342748244734204,
+ "loss": 1.4273,
+ "theoretical_loss": 3.639386558390363,
+ "tokens_seen": 1118437376
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534258776328987,
+ "loss": 1.343,
+ "theoretical_loss": 3.6393668376398667,
+ "tokens_seen": 1118502912
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005342427281845538,
+ "loss": 1.3797,
+ "theoretical_loss": 3.6393471183683426,
+ "tokens_seen": 1118568448
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005342266800401204,
+ "loss": 1.3075,
+ "theoretical_loss": 3.639327400575593,
+ "tokens_seen": 1118633984
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005342106318956872,
+ "loss": 1.3961,
+ "theoretical_loss": 3.6393076842614196,
+ "tokens_seen": 1118699520
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005341945837512538,
+ "loss": 1.2817,
+ "theoretical_loss": 3.6392879694256264,
+ "tokens_seen": 1118765056
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005341785356068206,
+ "loss": 1.3376,
+ "theoretical_loss": 3.6392682560680143,
+ "tokens_seen": 1118830592
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005341624874623872,
+ "loss": 1.2789,
+ "theoretical_loss": 3.6392485441883875,
+ "tokens_seen": 1118896128
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534146439317954,
+ "loss": 1.3712,
+ "theoretical_loss": 3.6392288337865475,
+ "tokens_seen": 1118961664
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005341303911735206,
+ "loss": 1.3982,
+ "theoretical_loss": 3.6392091248622975,
+ "tokens_seen": 1119027200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005341143430290874,
+ "loss": 1.4322,
+ "theoretical_loss": 3.6391894174154396,
+ "tokens_seen": 1119092736
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000534098294884654,
+ "loss": 1.4059,
+ "theoretical_loss": 3.6391697114457773,
+ "tokens_seen": 1119158272
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340822467402207,
+ "loss": 1.338,
+ "theoretical_loss": 3.639150006953113,
+ "tokens_seen": 1119223808
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340661985957874,
+ "loss": 1.2945,
+ "theoretical_loss": 3.63913030393725,
+ "tokens_seen": 1119289344
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340501504513541,
+ "loss": 1.2693,
+ "theoretical_loss": 3.6391106023979902,
+ "tokens_seen": 1119354880
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340341023069208,
+ "loss": 1.4471,
+ "theoretical_loss": 3.6390909023351368,
+ "tokens_seen": 1119420416
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340180541624875,
+ "loss": 1.3414,
+ "theoretical_loss": 3.639071203748494,
+ "tokens_seen": 1119485952
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005340020060180542,
+ "loss": 1.243,
+ "theoretical_loss": 3.6390515066378626,
+ "tokens_seen": 1119551488
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339859578736209,
+ "loss": 1.2848,
+ "theoretical_loss": 3.6390318110030475,
+ "tokens_seen": 1119617024
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339699097291876,
+ "loss": 1.3206,
+ "theoretical_loss": 3.6390121168438507,
+ "tokens_seen": 1119682560
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339538615847543,
+ "loss": 1.3479,
+ "theoretical_loss": 3.638992424160076,
+ "tokens_seen": 1119748096
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339378134403209,
+ "loss": 1.4657,
+ "theoretical_loss": 3.6389727329515256,
+ "tokens_seen": 1119813632
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339217652958877,
+ "loss": 1.3867,
+ "theoretical_loss": 3.6389530432180033,
+ "tokens_seen": 1119879168
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005339057171514543,
+ "loss": 1.3862,
+ "theoretical_loss": 3.6389333549593124,
+ "tokens_seen": 1119944704
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338896690070211,
+ "loss": 1.3789,
+ "theoretical_loss": 3.638913668175255,
+ "tokens_seen": 1120010240
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338736208625877,
+ "loss": 1.3899,
+ "theoretical_loss": 3.6388939828656364,
+ "tokens_seen": 1120075776
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338575727181545,
+ "loss": 1.245,
+ "theoretical_loss": 3.6388742990302587,
+ "tokens_seen": 1120141312
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338415245737211,
+ "loss": 1.3722,
+ "theoretical_loss": 3.638854616668925,
+ "tokens_seen": 1120206848
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338254764292879,
+ "loss": 1.413,
+ "theoretical_loss": 3.638834935781439,
+ "tokens_seen": 1120272384
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005338094282848545,
+ "loss": 1.3437,
+ "theoretical_loss": 3.6388152563676037,
+ "tokens_seen": 1120337920
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337933801404213,
+ "loss": 1.361,
+ "theoretical_loss": 3.638795578427224,
+ "tokens_seen": 1120403456
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337773319959879,
+ "loss": 1.2868,
+ "theoretical_loss": 3.638775901960102,
+ "tokens_seen": 1120468992
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337612838515547,
+ "loss": 1.3439,
+ "theoretical_loss": 3.6387562269660414,
+ "tokens_seen": 1120534528
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337452357071213,
+ "loss": 1.3982,
+ "theoretical_loss": 3.638736553444846,
+ "tokens_seen": 1120600064
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 377426,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3275361061096191,
+ "objective/train/theoretical_loss": 3.63871688139632,
+ "objective/train/tokens_used": 1141125600,
+ "theoretical_loss": 3.63871688139632,
+ "tokens_seen": 1120665600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337291875626881,
+ "loss": 1.3741,
+ "theoretical_loss": 3.63871688139632,
+ "tokens_seen": 1120665600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005337131394182547,
+ "loss": 1.4376,
+ "theoretical_loss": 3.638697210820266,
+ "tokens_seen": 1120731136
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336970912738215,
+ "loss": 1.4034,
+ "theoretical_loss": 3.6386775417164885,
+ "tokens_seen": 1120796672
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336810431293881,
+ "loss": 1.3859,
+ "theoretical_loss": 3.638657874084791,
+ "tokens_seen": 1120862208
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336649949849549,
+ "loss": 1.3789,
+ "theoretical_loss": 3.638638207924977,
+ "tokens_seen": 1120927744
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336489468405215,
+ "loss": 1.2965,
+ "theoretical_loss": 3.6386185432368503,
+ "tokens_seen": 1120993280
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336328986960883,
+ "loss": 1.3204,
+ "theoretical_loss": 3.638598880020215,
+ "tokens_seen": 1121058816
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336168505516549,
+ "loss": 1.3042,
+ "theoretical_loss": 3.638579218274875,
+ "tokens_seen": 1121124352
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005336008024072217,
+ "loss": 1.368,
+ "theoretical_loss": 3.638559558000634,
+ "tokens_seen": 1121189888
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335847542627883,
+ "loss": 1.4172,
+ "theoretical_loss": 3.6385398991972964,
+ "tokens_seen": 1121255424
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335687061183551,
+ "loss": 1.4214,
+ "theoretical_loss": 3.638520241864666,
+ "tokens_seen": 1121320960
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335526579739217,
+ "loss": 1.3088,
+ "theoretical_loss": 3.6385005860025457,
+ "tokens_seen": 1121386496
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335366098294885,
+ "loss": 1.2974,
+ "theoretical_loss": 3.6384809316107414,
+ "tokens_seen": 1121452032
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335205616850551,
+ "loss": 1.4272,
+ "theoretical_loss": 3.638461278689056,
+ "tokens_seen": 1121517568
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005335045135406219,
+ "loss": 1.3855,
+ "theoretical_loss": 3.6384416272372944,
+ "tokens_seen": 1121583104
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334884653961885,
+ "loss": 1.441,
+ "theoretical_loss": 3.63842197725526,
+ "tokens_seen": 1121648640
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334724172517553,
+ "loss": 1.2518,
+ "theoretical_loss": 3.6384023287427576,
+ "tokens_seen": 1121714176
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334563691073219,
+ "loss": 1.2455,
+ "theoretical_loss": 3.638382681699591,
+ "tokens_seen": 1121779712
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334403209628887,
+ "loss": 1.3986,
+ "theoretical_loss": 3.6383630361255648,
+ "tokens_seen": 1121845248
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334242728184553,
+ "loss": 1.3738,
+ "theoretical_loss": 3.638343392020483,
+ "tokens_seen": 1121910784
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005334082246740221,
+ "loss": 1.3148,
+ "theoretical_loss": 3.638323749384151,
+ "tokens_seen": 1121976320
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333921765295887,
+ "loss": 1.4755,
+ "theoretical_loss": 3.6383041082163716,
+ "tokens_seen": 1122041856
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333761283851555,
+ "loss": 1.366,
+ "theoretical_loss": 3.63828446851695,
+ "tokens_seen": 1122107392
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333600802407221,
+ "loss": 1.3049,
+ "theoretical_loss": 3.6382648302856917,
+ "tokens_seen": 1122172928
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333440320962889,
+ "loss": 1.345,
+ "theoretical_loss": 3.6382451935223994,
+ "tokens_seen": 1122238464
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333279839518556,
+ "loss": 1.4058,
+ "theoretical_loss": 3.638225558226879,
+ "tokens_seen": 1122304000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005333119358074223,
+ "loss": 1.3659,
+ "theoretical_loss": 3.6382059243989344,
+ "tokens_seen": 1122369536
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000533295887662989,
+ "loss": 1.3639,
+ "theoretical_loss": 3.6381862920383705,
+ "tokens_seen": 1122435072
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005332798395185557,
+ "loss": 1.2969,
+ "theoretical_loss": 3.6381666611449917,
+ "tokens_seen": 1122500608
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005332637913741224,
+ "loss": 1.2742,
+ "theoretical_loss": 3.638147031718603,
+ "tokens_seen": 1122566144
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005332477432296891,
+ "loss": 1.4018,
+ "theoretical_loss": 3.6381274037590092,
+ "tokens_seen": 1122631680
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005332316950852559,
+ "loss": 1.4345,
+ "theoretical_loss": 3.638107777266015,
+ "tokens_seen": 1122697216
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005332156469408225,
+ "loss": 1.4199,
+ "theoretical_loss": 3.6380881522394244,
+ "tokens_seen": 1122762752
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331995987963893,
+ "loss": 1.35,
+ "theoretical_loss": 3.6380685286790437,
+ "tokens_seen": 1122828288
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331835506519559,
+ "loss": 1.44,
+ "theoretical_loss": 3.638048906584677,
+ "tokens_seen": 1122893824
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331675025075227,
+ "loss": 1.4196,
+ "theoretical_loss": 3.6380292859561294,
+ "tokens_seen": 1122959360
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331514543630893,
+ "loss": 1.326,
+ "theoretical_loss": 3.6380096667932054,
+ "tokens_seen": 1123024896
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331354062186561,
+ "loss": 1.3748,
+ "theoretical_loss": 3.63799004909571,
+ "tokens_seen": 1123090432
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331193580742227,
+ "loss": 1.2627,
+ "theoretical_loss": 3.6379704328634492,
+ "tokens_seen": 1123155968
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005331033099297895,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6379508180962272,
+ "tokens_seen": 1123221504
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330872617853561,
+ "loss": 1.2757,
+ "theoretical_loss": 3.63793120479385,
+ "tokens_seen": 1123287040
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330712136409229,
+ "loss": 1.2797,
+ "theoretical_loss": 3.6379115929561214,
+ "tokens_seen": 1123352576
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330551654964895,
+ "loss": 1.2881,
+ "theoretical_loss": 3.6378919825828477,
+ "tokens_seen": 1123418112
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330391173520563,
+ "loss": 1.3451,
+ "theoretical_loss": 3.637872373673834,
+ "tokens_seen": 1123483648
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330230692076229,
+ "loss": 1.4061,
+ "theoretical_loss": 3.637852766228885,
+ "tokens_seen": 1123549184
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005330070210631897,
+ "loss": 1.3824,
+ "theoretical_loss": 3.637833160247806,
+ "tokens_seen": 1123614720
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329909729187563,
+ "loss": 1.3463,
+ "theoretical_loss": 3.637813555730403,
+ "tokens_seen": 1123680256
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329749247743231,
+ "loss": 1.483,
+ "theoretical_loss": 3.637793952676481,
+ "tokens_seen": 1123745792
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329588766298897,
+ "loss": 1.401,
+ "theoretical_loss": 3.637774351085845,
+ "tokens_seen": 1123811328
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329428284854565,
+ "loss": 1.3526,
+ "theoretical_loss": 3.637754750958302,
+ "tokens_seen": 1123876864
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 378684,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2597837448120117,
+ "objective/train/theoretical_loss": 3.637735152293655,
+ "objective/train/tokens_used": 1144402400,
+ "theoretical_loss": 3.637735152293655,
+ "tokens_seen": 1123942400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329267803410231,
+ "loss": 1.3474,
+ "theoretical_loss": 3.637735152293655,
+ "tokens_seen": 1123942400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005329107321965899,
+ "loss": 1.4682,
+ "theoretical_loss": 3.637715555091712,
+ "tokens_seen": 1124007936
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328946840521565,
+ "loss": 1.3396,
+ "theoretical_loss": 3.6376959593522766,
+ "tokens_seen": 1124073472
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328786359077233,
+ "loss": 1.2955,
+ "theoretical_loss": 3.637676365075156,
+ "tokens_seen": 1124139008
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328625877632899,
+ "loss": 1.3791,
+ "theoretical_loss": 3.6376567722601547,
+ "tokens_seen": 1124204544
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328465396188567,
+ "loss": 1.4525,
+ "theoretical_loss": 3.637637180907079,
+ "tokens_seen": 1124270080
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328304914744233,
+ "loss": 1.294,
+ "theoretical_loss": 3.637617591015734,
+ "tokens_seen": 1124335616
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005328144433299901,
+ "loss": 1.3757,
+ "theoretical_loss": 3.637598002585926,
+ "tokens_seen": 1124401152
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327983951855567,
+ "loss": 1.3695,
+ "theoretical_loss": 3.6375784156174604,
+ "tokens_seen": 1124466688
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327823470411235,
+ "loss": 1.4115,
+ "theoretical_loss": 3.6375588301101436,
+ "tokens_seen": 1124532224
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327662988966901,
+ "loss": 1.3728,
+ "theoretical_loss": 3.6375392460637808,
+ "tokens_seen": 1124597760
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327502507522569,
+ "loss": 1.3182,
+ "theoretical_loss": 3.6375196634781783,
+ "tokens_seen": 1124663296
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327342026078235,
+ "loss": 1.375,
+ "theoretical_loss": 3.637500082353142,
+ "tokens_seen": 1124728832
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327181544633903,
+ "loss": 1.3206,
+ "theoretical_loss": 3.6374805026884776,
+ "tokens_seen": 1124794368
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005327021063189569,
+ "loss": 1.3184,
+ "theoretical_loss": 3.637460924483991,
+ "tokens_seen": 1124859904
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005326860581745236,
+ "loss": 1.3875,
+ "theoretical_loss": 3.6374413477394887,
+ "tokens_seen": 1124925440
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005326700100300903,
+ "loss": 1.2268,
+ "theoretical_loss": 3.637421772454777,
+ "tokens_seen": 1124990976
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532653961885657,
+ "loss": 1.331,
+ "theoretical_loss": 3.637402198629661,
+ "tokens_seen": 1125056512
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005326379137412237,
+ "loss": 1.3865,
+ "theoretical_loss": 3.637382626263948,
+ "tokens_seen": 1125122048
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005326218655967904,
+ "loss": 1.4205,
+ "theoretical_loss": 3.637363055357443,
+ "tokens_seen": 1125187584
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005326058174523571,
+ "loss": 1.4883,
+ "theoretical_loss": 3.6373434859099536,
+ "tokens_seen": 1125253120
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325897693079238,
+ "loss": 1.3364,
+ "theoretical_loss": 3.637323917921285,
+ "tokens_seen": 1125318656
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325737211634905,
+ "loss": 1.3323,
+ "theoretical_loss": 3.637304351391244,
+ "tokens_seen": 1125384192
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325576730190572,
+ "loss": 1.4029,
+ "theoretical_loss": 3.6372847863196363,
+ "tokens_seen": 1125449728
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325416248746238,
+ "loss": 1.356,
+ "theoretical_loss": 3.6372652227062696,
+ "tokens_seen": 1125515264
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325255767301906,
+ "loss": 1.3703,
+ "theoretical_loss": 3.637245660550949,
+ "tokens_seen": 1125580800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005325095285857572,
+ "loss": 1.3003,
+ "theoretical_loss": 3.6372260998534816,
+ "tokens_seen": 1125646336
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532493480441324,
+ "loss": 1.3414,
+ "theoretical_loss": 3.6372065406136733,
+ "tokens_seen": 1125711872
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005324774322968906,
+ "loss": 1.3245,
+ "theoretical_loss": 3.6371869828313317,
+ "tokens_seen": 1125777408
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005324613841524574,
+ "loss": 1.2384,
+ "theoretical_loss": 3.637167426506262,
+ "tokens_seen": 1125842944
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532445336008024,
+ "loss": 1.3696,
+ "theoretical_loss": 3.637147871638272,
+ "tokens_seen": 1125908480
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005324292878635908,
+ "loss": 1.2808,
+ "theoretical_loss": 3.637128318227168,
+ "tokens_seen": 1125974016
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005324132397191574,
+ "loss": 1.405,
+ "theoretical_loss": 3.637108766272756,
+ "tokens_seen": 1126039552
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323971915747242,
+ "loss": 1.2078,
+ "theoretical_loss": 3.637089215774843,
+ "tokens_seen": 1126105088
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323811434302908,
+ "loss": 1.4223,
+ "theoretical_loss": 3.6370696667332365,
+ "tokens_seen": 1126170624
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323650952858576,
+ "loss": 1.3003,
+ "theoretical_loss": 3.6370501191477427,
+ "tokens_seen": 1126236160
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323490471414242,
+ "loss": 1.3226,
+ "theoretical_loss": 3.6370305730181682,
+ "tokens_seen": 1126301696
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532332998996991,
+ "loss": 1.4025,
+ "theoretical_loss": 3.6370110283443204,
+ "tokens_seen": 1126367232
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323169508525576,
+ "loss": 1.3653,
+ "theoretical_loss": 3.6369914851260057,
+ "tokens_seen": 1126432768
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005323009027081244,
+ "loss": 1.326,
+ "theoretical_loss": 3.636971943363031,
+ "tokens_seen": 1126498304
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532284854563691,
+ "loss": 1.3969,
+ "theoretical_loss": 3.6369524030552043,
+ "tokens_seen": 1126563840
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005322688064192578,
+ "loss": 1.2623,
+ "theoretical_loss": 3.6369328642023313,
+ "tokens_seen": 1126629376
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005322527582748244,
+ "loss": 1.415,
+ "theoretical_loss": 3.6369133268042195,
+ "tokens_seen": 1126694912
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005322367101303912,
+ "loss": 1.3816,
+ "theoretical_loss": 3.636893790860676,
+ "tokens_seen": 1126760448
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005322206619859578,
+ "loss": 1.4577,
+ "theoretical_loss": 3.6368742563715077,
+ "tokens_seen": 1126825984
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005322046138415246,
+ "loss": 1.2977,
+ "theoretical_loss": 3.6368547233365227,
+ "tokens_seen": 1126891520
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005321885656970912,
+ "loss": 1.421,
+ "theoretical_loss": 3.6368351917555275,
+ "tokens_seen": 1126957056
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532172517552658,
+ "loss": 1.4054,
+ "theoretical_loss": 3.6368156616283285,
+ "tokens_seen": 1127022592
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005321564694082246,
+ "loss": 1.3986,
+ "theoretical_loss": 3.6367961329547343,
+ "tokens_seen": 1127088128
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005321404212637914,
+ "loss": 1.3883,
+ "theoretical_loss": 3.6367766057345516,
+ "tokens_seen": 1127153664
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 379502,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4635413885116577,
+ "objective/train/theoretical_loss": 3.6367570799675875,
+ "objective/train/tokens_used": 1147679200,
+ "theoretical_loss": 3.6367570799675875,
+ "tokens_seen": 1127219200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005321243731193581,
+ "loss": 1.4607,
+ "theoretical_loss": 3.6367570799675875,
+ "tokens_seen": 1127219200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005321083249749248,
+ "loss": 1.3679,
+ "theoretical_loss": 3.6367375556536503,
+ "tokens_seen": 1127284736
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005320922768304915,
+ "loss": 1.4228,
+ "theoretical_loss": 3.636718032792546,
+ "tokens_seen": 1127350272
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005320762286860582,
+ "loss": 1.3982,
+ "theoretical_loss": 3.6366985113840835,
+ "tokens_seen": 1127415808
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005320601805416249,
+ "loss": 1.3395,
+ "theoretical_loss": 3.6366789914280693,
+ "tokens_seen": 1127481344
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005320441323971916,
+ "loss": 1.3854,
+ "theoretical_loss": 3.636659472924311,
+ "tokens_seen": 1127546880
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005320280842527583,
+ "loss": 1.3441,
+ "theoretical_loss": 3.636639955872617,
+ "tokens_seen": 1127612416
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000532012036108325,
+ "loss": 1.2984,
+ "theoretical_loss": 3.636620440272794,
+ "tokens_seen": 1127677952
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319959879638917,
+ "loss": 1.3089,
+ "theoretical_loss": 3.6366009261246495,
+ "tokens_seen": 1127743488
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319799398194584,
+ "loss": 1.289,
+ "theoretical_loss": 3.636581413427992,
+ "tokens_seen": 1127809024
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319638916750251,
+ "loss": 1.3694,
+ "theoretical_loss": 3.6365619021826285,
+ "tokens_seen": 1127874560
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319478435305918,
+ "loss": 1.4387,
+ "theoretical_loss": 3.6365423923883666,
+ "tokens_seen": 1127940096
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319317953861585,
+ "loss": 1.2967,
+ "theoretical_loss": 3.6365228840450152,
+ "tokens_seen": 1128005632
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005319157472417252,
+ "loss": 1.2617,
+ "theoretical_loss": 3.636503377152381,
+ "tokens_seen": 1128071168
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531899699097292,
+ "loss": 1.2746,
+ "theoretical_loss": 3.6364838717102725,
+ "tokens_seen": 1128136704
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005318836509528586,
+ "loss": 1.2389,
+ "theoretical_loss": 3.636464367718497,
+ "tokens_seen": 1128202240
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005318676028084254,
+ "loss": 1.2898,
+ "theoretical_loss": 3.636444865176863,
+ "tokens_seen": 1128267776
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531851554663992,
+ "loss": 1.3015,
+ "theoretical_loss": 3.636425364085178,
+ "tokens_seen": 1128333312
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005318355065195588,
+ "loss": 1.3878,
+ "theoretical_loss": 3.63640586444325,
+ "tokens_seen": 1128398848
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005318194583751254,
+ "loss": 1.3115,
+ "theoretical_loss": 3.636386366250888,
+ "tokens_seen": 1128464384
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005318034102306922,
+ "loss": 1.3701,
+ "theoretical_loss": 3.636366869507899,
+ "tokens_seen": 1128529920
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005317873620862588,
+ "loss": 1.2944,
+ "theoretical_loss": 3.6363473742140906,
+ "tokens_seen": 1128595456
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005317713139418256,
+ "loss": 1.3582,
+ "theoretical_loss": 3.6363278803692727,
+ "tokens_seen": 1128660992
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005317552657973922,
+ "loss": 1.2594,
+ "theoretical_loss": 3.6363083879732523,
+ "tokens_seen": 1128726528
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531739217652959,
+ "loss": 1.3404,
+ "theoretical_loss": 3.6362888970258376,
+ "tokens_seen": 1128792064
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005317231695085256,
+ "loss": 1.4112,
+ "theoretical_loss": 3.6362694075268367,
+ "tokens_seen": 1128857600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005317071213640924,
+ "loss": 1.4157,
+ "theoretical_loss": 3.636249919476059,
+ "tokens_seen": 1128923136
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531691073219659,
+ "loss": 1.3547,
+ "theoretical_loss": 3.6362304328733117,
+ "tokens_seen": 1128988672
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005316750250752258,
+ "loss": 1.2918,
+ "theoretical_loss": 3.6362109477184035,
+ "tokens_seen": 1129054208
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005316589769307924,
+ "loss": 1.394,
+ "theoretical_loss": 3.6361914640111435,
+ "tokens_seen": 1129119744
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005316429287863592,
+ "loss": 1.4599,
+ "theoretical_loss": 3.6361719817513385,
+ "tokens_seen": 1129185280
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005316268806419258,
+ "loss": 1.3713,
+ "theoretical_loss": 3.6361525009387985,
+ "tokens_seen": 1129250816
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005316108324974926,
+ "loss": 1.2741,
+ "theoretical_loss": 3.6361330215733316,
+ "tokens_seen": 1129316352
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005315947843530592,
+ "loss": 1.3546,
+ "theoretical_loss": 3.6361135436547456,
+ "tokens_seen": 1129381888
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531578736208626,
+ "loss": 1.3232,
+ "theoretical_loss": 3.63609406718285,
+ "tokens_seen": 1129447424
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005315626880641926,
+ "loss": 1.329,
+ "theoretical_loss": 3.6360745921574527,
+ "tokens_seen": 1129512960
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005315466399197594,
+ "loss": 1.4433,
+ "theoretical_loss": 3.6360551185783634,
+ "tokens_seen": 1129578496
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531530591775326,
+ "loss": 1.2452,
+ "theoretical_loss": 3.63603564644539,
+ "tokens_seen": 1129644032
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005315145436308928,
+ "loss": 1.3164,
+ "theoretical_loss": 3.636016175758341,
+ "tokens_seen": 1129709568
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314984954864594,
+ "loss": 1.524,
+ "theoretical_loss": 3.635996706517026,
+ "tokens_seen": 1129775104
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314824473420262,
+ "loss": 1.4299,
+ "theoretical_loss": 3.6359772387212526,
+ "tokens_seen": 1129840640
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314663991975928,
+ "loss": 1.3257,
+ "theoretical_loss": 3.6359577723708307,
+ "tokens_seen": 1129906176
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314503510531596,
+ "loss": 1.396,
+ "theoretical_loss": 3.635938307465569,
+ "tokens_seen": 1129971712
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314343029087262,
+ "loss": 1.3114,
+ "theoretical_loss": 3.6359188440052757,
+ "tokens_seen": 1130037248
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531418254764293,
+ "loss": 1.3683,
+ "theoretical_loss": 3.6358993819897605,
+ "tokens_seen": 1130102784
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005314022066198596,
+ "loss": 1.4065,
+ "theoretical_loss": 3.635879921418832,
+ "tokens_seen": 1130168320
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005313861584754264,
+ "loss": 1.3705,
+ "theoretical_loss": 3.6358604622922996,
+ "tokens_seen": 1130233856
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000531370110330993,
+ "loss": 1.2837,
+ "theoretical_loss": 3.635841004609972,
+ "tokens_seen": 1130299392
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005313540621865598,
+ "loss": 1.3338,
+ "theoretical_loss": 3.6358215483716587,
+ "tokens_seen": 1130364928
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005313380140421264,
+ "loss": 1.3236,
+ "theoretical_loss": 3.635802093577168,
+ "tokens_seen": 1130430464
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 380591,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.375028133392334,
+ "objective/train/theoretical_loss": 3.63578264022631,
+ "objective/train/tokens_used": 1150956000,
+ "theoretical_loss": 3.63578264022631,
+ "tokens_seen": 1130496000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005313219658976932,
+ "loss": 1.2996,
+ "theoretical_loss": 3.63578264022631,
+ "tokens_seen": 1130496000
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005313059177532598,
+ "loss": 1.372,
+ "theoretical_loss": 3.6357631883188932,
+ "tokens_seen": 1130561536
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005312898696088265,
+ "loss": 1.3764,
+ "theoretical_loss": 3.6357437378547273,
+ "tokens_seen": 1130627072
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005312738214643932,
+ "loss": 1.2847,
+ "theoretical_loss": 3.635724288833621,
+ "tokens_seen": 1130692608
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005312577733199599,
+ "loss": 1.3036,
+ "theoretical_loss": 3.635704841255384,
+ "tokens_seen": 1130758144
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005312417251755266,
+ "loss": 1.3314,
+ "theoretical_loss": 3.6356853951198262,
+ "tokens_seen": 1130823680
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005312256770310933,
+ "loss": 1.36,
+ "theoretical_loss": 3.6356659504267563,
+ "tokens_seen": 1130889216
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.00053120962888666,
+ "loss": 1.2925,
+ "theoretical_loss": 3.6356465071759834,
+ "tokens_seen": 1130954752
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311935807422267,
+ "loss": 1.3217,
+ "theoretical_loss": 3.6356270653673173,
+ "tokens_seen": 1131020288
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311775325977934,
+ "loss": 1.2822,
+ "theoretical_loss": 3.635607625000568,
+ "tokens_seen": 1131085824
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311614844533601,
+ "loss": 1.2929,
+ "theoretical_loss": 3.6355881860755437,
+ "tokens_seen": 1131151360
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311454363089267,
+ "loss": 1.3268,
+ "theoretical_loss": 3.635568748592056,
+ "tokens_seen": 1131216896
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311293881644935,
+ "loss": 1.3252,
+ "theoretical_loss": 3.6355493125499123,
+ "tokens_seen": 1131282432
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005311133400200601,
+ "loss": 1.3213,
+ "theoretical_loss": 3.6355298779489242,
+ "tokens_seen": 1131347968
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310972918756269,
+ "loss": 1.3944,
+ "theoretical_loss": 3.6355104447889,
+ "tokens_seen": 1131413504
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310812437311935,
+ "loss": 1.2435,
+ "theoretical_loss": 3.6354910130696494,
+ "tokens_seen": 1131479040
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310651955867603,
+ "loss": 1.4415,
+ "theoretical_loss": 3.635471582790983,
+ "tokens_seen": 1131544576
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310491474423269,
+ "loss": 1.3897,
+ "theoretical_loss": 3.63545215395271,
+ "tokens_seen": 1131610112
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310330992978937,
+ "loss": 1.3398,
+ "theoretical_loss": 3.6354327265546402,
+ "tokens_seen": 1131675648
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310170511534603,
+ "loss": 1.3494,
+ "theoretical_loss": 3.635413300596584,
+ "tokens_seen": 1131741184
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005310010030090271,
+ "loss": 1.2869,
+ "theoretical_loss": 3.635393876078351,
+ "tokens_seen": 1131806720
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309849548645937,
+ "loss": 1.3088,
+ "theoretical_loss": 3.6353744529997503,
+ "tokens_seen": 1131872256
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309689067201605,
+ "loss": 1.3001,
+ "theoretical_loss": 3.635355031360593,
+ "tokens_seen": 1131937792
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309528585757271,
+ "loss": 1.2959,
+ "theoretical_loss": 3.6353356111606887,
+ "tokens_seen": 1132003328
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309368104312939,
+ "loss": 1.3803,
+ "theoretical_loss": 3.6353161923998476,
+ "tokens_seen": 1132068864
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309207622868606,
+ "loss": 1.2426,
+ "theoretical_loss": 3.6352967750778786,
+ "tokens_seen": 1132134400
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005309047141424273,
+ "loss": 1.3723,
+ "theoretical_loss": 3.6352773591945935,
+ "tokens_seen": 1132199936
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000530888665997994,
+ "loss": 1.2994,
+ "theoretical_loss": 3.6352579447498017,
+ "tokens_seen": 1132265472
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005308726178535607,
+ "loss": 1.2576,
+ "theoretical_loss": 3.6352385317433136,
+ "tokens_seen": 1132331008
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005308565697091274,
+ "loss": 1.3685,
+ "theoretical_loss": 3.6352191201749386,
+ "tokens_seen": 1132396544
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005308405215646941,
+ "loss": 1.2988,
+ "theoretical_loss": 3.6351997100444873,
+ "tokens_seen": 1132462080
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005308244734202608,
+ "loss": 1.2542,
+ "theoretical_loss": 3.6351803013517707,
+ "tokens_seen": 1132527616
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005308084252758275,
+ "loss": 1.3261,
+ "theoretical_loss": 3.6351608940965985,
+ "tokens_seen": 1132593152
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005307923771313942,
+ "loss": 1.242,
+ "theoretical_loss": 3.6351414882787814,
+ "tokens_seen": 1132658688
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005307763289869609,
+ "loss": 1.3133,
+ "theoretical_loss": 3.6351220838981293,
+ "tokens_seen": 1132724224
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005307602808425276,
+ "loss": 1.2503,
+ "theoretical_loss": 3.6351026809544527,
+ "tokens_seen": 1132789760
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005307442326980943,
+ "loss": 1.2665,
+ "theoretical_loss": 3.6350832794475627,
+ "tokens_seen": 1132855296
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000530728184553661,
+ "loss": 1.3992,
+ "theoretical_loss": 3.635063879377269,
+ "tokens_seen": 1132920832
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005307121364092277,
+ "loss": 1.3293,
+ "theoretical_loss": 3.6350444807433826,
+ "tokens_seen": 1132986368
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306960882647944,
+ "loss": 1.2076,
+ "theoretical_loss": 3.6350250835457136,
+ "tokens_seen": 1133051904
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306800401203611,
+ "loss": 1.3282,
+ "theoretical_loss": 3.6350056877840733,
+ "tokens_seen": 1133117440
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306639919759278,
+ "loss": 1.3262,
+ "theoretical_loss": 3.6349862934582715,
+ "tokens_seen": 1133182976
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306479438314945,
+ "loss": 1.3443,
+ "theoretical_loss": 3.63496690056812,
+ "tokens_seen": 1133248512
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306318956870612,
+ "loss": 1.3307,
+ "theoretical_loss": 3.6349475091134287,
+ "tokens_seen": 1133314048
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005306158475426279,
+ "loss": 1.2516,
+ "theoretical_loss": 3.6349281190940084,
+ "tokens_seen": 1133379584
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305997993981946,
+ "loss": 1.3576,
+ "theoretical_loss": 3.63490873050967,
+ "tokens_seen": 1133445120
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305837512537613,
+ "loss": 1.3443,
+ "theoretical_loss": 3.6348893433602245,
+ "tokens_seen": 1133510656
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000530567703109328,
+ "loss": 1.3465,
+ "theoretical_loss": 3.634869957645482,
+ "tokens_seen": 1133576192
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305516549648947,
+ "loss": 1.385,
+ "theoretical_loss": 3.634850573365255,
+ "tokens_seen": 1133641728
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305356068204614,
+ "loss": 1.3111,
+ "theoretical_loss": 3.6348311905193524,
+ "tokens_seen": 1133707264
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 381475,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2271829843521118,
+ "objective/train/theoretical_loss": 3.6348118091075867,
+ "objective/train/tokens_used": 1154232800,
+ "theoretical_loss": 3.6348118091075867,
+ "tokens_seen": 1133772800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305195586760281,
+ "loss": 1.191,
+ "theoretical_loss": 3.6348118091075867,
+ "tokens_seen": 1133772800
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005305035105315948,
+ "loss": 1.2194,
+ "theoretical_loss": 3.634792429129768,
+ "tokens_seen": 1133838336
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005304874623871615,
+ "loss": 1.2952,
+ "theoretical_loss": 3.634773050585708,
+ "tokens_seen": 1133903872
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005304714142427283,
+ "loss": 1.2634,
+ "theoretical_loss": 3.6347536734752177,
+ "tokens_seen": 1133969408
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005304553660982949,
+ "loss": 1.339,
+ "theoretical_loss": 3.6347342977981074,
+ "tokens_seen": 1134034944
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005304393179538617,
+ "loss": 1.3236,
+ "theoretical_loss": 3.6347149235541893,
+ "tokens_seen": 1134100480
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005304232698094283,
+ "loss": 1.3053,
+ "theoretical_loss": 3.634695550743274,
+ "tokens_seen": 1134166016
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000530407221664995,
+ "loss": 1.3035,
+ "theoretical_loss": 3.6346761793651723,
+ "tokens_seen": 1134231552
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303911735205617,
+ "loss": 1.3321,
+ "theoretical_loss": 3.6346568094196963,
+ "tokens_seen": 1134297088
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303751253761285,
+ "loss": 1.3015,
+ "theoretical_loss": 3.634637440906657,
+ "tokens_seen": 1134362624
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303590772316951,
+ "loss": 1.3557,
+ "theoretical_loss": 3.634618073825866,
+ "tokens_seen": 1134428160
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303430290872619,
+ "loss": 1.3101,
+ "theoretical_loss": 3.634598708177134,
+ "tokens_seen": 1134493696
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303269809428285,
+ "loss": 1.2787,
+ "theoretical_loss": 3.6345793439602723,
+ "tokens_seen": 1134559232
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005303109327983953,
+ "loss": 1.4081,
+ "theoretical_loss": 3.6345599811750935,
+ "tokens_seen": 1134624768
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302948846539619,
+ "loss": 1.3721,
+ "theoretical_loss": 3.634540619821408,
+ "tokens_seen": 1134690304
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302788365095287,
+ "loss": 1.3142,
+ "theoretical_loss": 3.634521259899027,
+ "tokens_seen": 1134755840
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302627883650953,
+ "loss": 1.4324,
+ "theoretical_loss": 3.6345019014077637,
+ "tokens_seen": 1134821376
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302467402206621,
+ "loss": 1.247,
+ "theoretical_loss": 3.634482544347428,
+ "tokens_seen": 1134886912
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302306920762287,
+ "loss": 1.3462,
+ "theoretical_loss": 3.634463188717832,
+ "tokens_seen": 1134952448
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005302146439317955,
+ "loss": 1.301,
+ "theoretical_loss": 3.634443834518788,
+ "tokens_seen": 1135017984
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301985957873621,
+ "loss": 1.3281,
+ "theoretical_loss": 3.6344244817501066,
+ "tokens_seen": 1135083520
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301825476429289,
+ "loss": 1.3277,
+ "theoretical_loss": 3.6344051304116007,
+ "tokens_seen": 1135149056
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301664994984955,
+ "loss": 1.4713,
+ "theoretical_loss": 3.634385780503081,
+ "tokens_seen": 1135214592
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301504513540623,
+ "loss": 1.3735,
+ "theoretical_loss": 3.6343664320243594,
+ "tokens_seen": 1135280128
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301344032096289,
+ "loss": 1.254,
+ "theoretical_loss": 3.6343470849752486,
+ "tokens_seen": 1135345664
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301183550651957,
+ "loss": 1.4025,
+ "theoretical_loss": 3.6343277393555597,
+ "tokens_seen": 1135411200
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005301023069207623,
+ "loss": 1.2399,
+ "theoretical_loss": 3.634308395165105,
+ "tokens_seen": 1135476736
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300862587763291,
+ "loss": 1.2324,
+ "theoretical_loss": 3.6342890524036955,
+ "tokens_seen": 1135542272
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300702106318957,
+ "loss": 1.3211,
+ "theoretical_loss": 3.6342697110711444,
+ "tokens_seen": 1135607808
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300541624874625,
+ "loss": 1.401,
+ "theoretical_loss": 3.634250371167263,
+ "tokens_seen": 1135673344
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300381143430291,
+ "loss": 1.3394,
+ "theoretical_loss": 3.634231032691863,
+ "tokens_seen": 1135738880
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300220661985959,
+ "loss": 1.3315,
+ "theoretical_loss": 3.634211695644758,
+ "tokens_seen": 1135804416
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005300060180541625,
+ "loss": 1.3305,
+ "theoretical_loss": 3.634192360025758,
+ "tokens_seen": 1135869952
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299899699097293,
+ "loss": 1.2994,
+ "theoretical_loss": 3.634173025834677,
+ "tokens_seen": 1135935488
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299739217652959,
+ "loss": 1.3139,
+ "theoretical_loss": 3.6341536930713265,
+ "tokens_seen": 1136001024
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299578736208627,
+ "loss": 1.3998,
+ "theoretical_loss": 3.6341343617355184,
+ "tokens_seen": 1136066560
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299418254764293,
+ "loss": 1.3566,
+ "theoretical_loss": 3.634115031827065,
+ "tokens_seen": 1136132096
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299257773319961,
+ "loss": 1.3159,
+ "theoretical_loss": 3.6340957033457784,
+ "tokens_seen": 1136197632
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005299097291875627,
+ "loss": 1.2776,
+ "theoretical_loss": 3.6340763762914716,
+ "tokens_seen": 1136263168
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298936810431294,
+ "loss": 1.3585,
+ "theoretical_loss": 3.634057050663957,
+ "tokens_seen": 1136328704
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298776328986961,
+ "loss": 1.473,
+ "theoretical_loss": 3.634037726463046,
+ "tokens_seen": 1136394240
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298615847542628,
+ "loss": 1.3581,
+ "theoretical_loss": 3.634018403688552,
+ "tokens_seen": 1136459776
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298455366098295,
+ "loss": 1.404,
+ "theoretical_loss": 3.633999082340287,
+ "tokens_seen": 1136525312
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298294884653962,
+ "loss": 1.3263,
+ "theoretical_loss": 3.633979762418063,
+ "tokens_seen": 1136590848
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005298134403209629,
+ "loss": 1.3379,
+ "theoretical_loss": 3.6339604439216937,
+ "tokens_seen": 1136656384
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297973921765296,
+ "loss": 1.4255,
+ "theoretical_loss": 3.633941126850991,
+ "tokens_seen": 1136721920
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297813440320963,
+ "loss": 1.4317,
+ "theoretical_loss": 3.6339218112057674,
+ "tokens_seen": 1136787456
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000529765295887663,
+ "loss": 1.4105,
+ "theoretical_loss": 3.633902496985836,
+ "tokens_seen": 1136852992
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297492477432296,
+ "loss": 1.2663,
+ "theoretical_loss": 3.633883184191009,
+ "tokens_seen": 1136918528
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297331995987964,
+ "loss": 1.2736,
+ "theoretical_loss": 3.6338638728210997,
+ "tokens_seen": 1136984064
+ },
+ {
+ "epoch": 0.34,
+ "objective/train/docs_used": 382718,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4597383737564087,
+ "objective/train/theoretical_loss": 3.63384456287592,
+ "objective/train/tokens_used": 1157509600,
+ "theoretical_loss": 3.63384456287592,
+ "tokens_seen": 1137049600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297171514543631,
+ "loss": 1.4389,
+ "theoretical_loss": 3.63384456287592,
+ "tokens_seen": 1137049600
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005297011033099298,
+ "loss": 1.3618,
+ "theoretical_loss": 3.6338252543552834,
+ "tokens_seen": 1137115136
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005296850551654965,
+ "loss": 1.2754,
+ "theoretical_loss": 3.6338059472590025,
+ "tokens_seen": 1137180672
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005296690070210632,
+ "loss": 1.3212,
+ "theoretical_loss": 3.63378664158689,
+ "tokens_seen": 1137246208
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005296529588766299,
+ "loss": 1.3643,
+ "theoretical_loss": 3.6337673373387585,
+ "tokens_seen": 1137311744
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005296369107321966,
+ "loss": 1.39,
+ "theoretical_loss": 3.6337480345144217,
+ "tokens_seen": 1137377280
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005296208625877633,
+ "loss": 1.2826,
+ "theoretical_loss": 3.633728733113692,
+ "tokens_seen": 1137442816
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.00052960481444333,
+ "loss": 1.4381,
+ "theoretical_loss": 3.6337094331363824,
+ "tokens_seen": 1137508352
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295887662988967,
+ "loss": 1.4166,
+ "theoretical_loss": 3.6336901345823067,
+ "tokens_seen": 1137573888
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295727181544634,
+ "loss": 1.3983,
+ "theoretical_loss": 3.633670837451277,
+ "tokens_seen": 1137639424
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295566700100301,
+ "loss": 1.3055,
+ "theoretical_loss": 3.633651541743107,
+ "tokens_seen": 1137704960
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295406218655968,
+ "loss": 1.3178,
+ "theoretical_loss": 3.633632247457609,
+ "tokens_seen": 1137770496
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295245737211635,
+ "loss": 1.3164,
+ "theoretical_loss": 3.633612954594597,
+ "tokens_seen": 1137836032
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005295085255767302,
+ "loss": 1.325,
+ "theoretical_loss": 3.633593663153884,
+ "tokens_seen": 1137901568
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005294924774322969,
+ "loss": 1.3331,
+ "theoretical_loss": 3.6335743731352834,
+ "tokens_seen": 1137967104
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005294764292878636,
+ "loss": 1.3119,
+ "theoretical_loss": 3.633555084538608,
+ "tokens_seen": 1138032640
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005294603811434303,
+ "loss": 1.4214,
+ "theoretical_loss": 3.6335357973636713,
+ "tokens_seen": 1138098176
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.000529444332998997,
+ "loss": 1.2846,
+ "theoretical_loss": 3.6335165116102868,
+ "tokens_seen": 1138163712
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005294282848545637,
+ "loss": 1.3522,
+ "theoretical_loss": 3.633497227278268,
+ "tokens_seen": 1138229248
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005294122367101304,
+ "loss": 1.4455,
+ "theoretical_loss": 3.633477944367428,
+ "tokens_seen": 1138294784
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005293961885656971,
+ "loss": 1.338,
+ "theoretical_loss": 3.6334586628775805,
+ "tokens_seen": 1138360320
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005293801404212638,
+ "loss": 1.2857,
+ "theoretical_loss": 3.6334393828085387,
+ "tokens_seen": 1138425856
+ },
+ {
+ "epoch": 0.34,
+ "learning_rate": 0.0005293640922768305,
+ "loss": 1.4045,
+ "theoretical_loss": 3.633420104160116,
+ "tokens_seen": 1138491392
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005293480441323972,
+ "loss": 1.2807,
+ "theoretical_loss": 3.6334008269321263,
+ "tokens_seen": 1138556928
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005293319959879639,
+ "loss": 1.3043,
+ "theoretical_loss": 3.6333815511243834,
+ "tokens_seen": 1138622464
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005293159478435306,
+ "loss": 1.2798,
+ "theoretical_loss": 3.633362276736701,
+ "tokens_seen": 1138688000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292998996990973,
+ "loss": 1.2801,
+ "theoretical_loss": 3.633343003768892,
+ "tokens_seen": 1138753536
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000529283851554664,
+ "loss": 1.3817,
+ "theoretical_loss": 3.63332373222077,
+ "tokens_seen": 1138819072
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292678034102307,
+ "loss": 1.3764,
+ "theoretical_loss": 3.63330446209215,
+ "tokens_seen": 1138884608
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292517552657974,
+ "loss": 1.4049,
+ "theoretical_loss": 3.6332851933828447,
+ "tokens_seen": 1138950144
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292357071213641,
+ "loss": 1.2247,
+ "theoretical_loss": 3.6332659260926685,
+ "tokens_seen": 1139015680
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292196589769308,
+ "loss": 1.4062,
+ "theoretical_loss": 3.6332466602214346,
+ "tokens_seen": 1139081216
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005292036108324975,
+ "loss": 1.3674,
+ "theoretical_loss": 3.6332273957689574,
+ "tokens_seen": 1139146752
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005291875626880642,
+ "loss": 1.4022,
+ "theoretical_loss": 3.6332081327350503,
+ "tokens_seen": 1139212288
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000529171514543631,
+ "loss": 1.3372,
+ "theoretical_loss": 3.633188871119528,
+ "tokens_seen": 1139277824
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005291554663991976,
+ "loss": 1.3026,
+ "theoretical_loss": 3.633169610922204,
+ "tokens_seen": 1139343360
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005291394182547643,
+ "loss": 1.2527,
+ "theoretical_loss": 3.6331503521428923,
+ "tokens_seen": 1139408896
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000529123370110331,
+ "loss": 1.4002,
+ "theoretical_loss": 3.6331310947814073,
+ "tokens_seen": 1139474432
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005291073219658978,
+ "loss": 1.2718,
+ "theoretical_loss": 3.6331118388375625,
+ "tokens_seen": 1139539968
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005290912738214644,
+ "loss": 1.4031,
+ "theoretical_loss": 3.6330925843111723,
+ "tokens_seen": 1139605504
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005290752256770312,
+ "loss": 1.4044,
+ "theoretical_loss": 3.633073331202051,
+ "tokens_seen": 1139671040
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005290591775325978,
+ "loss": 1.4823,
+ "theoretical_loss": 3.6330540795100124,
+ "tokens_seen": 1139736576
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005290431293881646,
+ "loss": 1.3078,
+ "theoretical_loss": 3.6330348292348713,
+ "tokens_seen": 1139802112
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005290270812437312,
+ "loss": 1.3579,
+ "theoretical_loss": 3.6330155803764415,
+ "tokens_seen": 1139867648
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000529011033099298,
+ "loss": 1.3369,
+ "theoretical_loss": 3.632996332934537,
+ "tokens_seen": 1139933184
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005289949849548646,
+ "loss": 1.2965,
+ "theoretical_loss": 3.6329770869089737,
+ "tokens_seen": 1139998720
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005289789368104314,
+ "loss": 1.3081,
+ "theoretical_loss": 3.6329578422995636,
+ "tokens_seen": 1140064256
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528962888665998,
+ "loss": 1.3686,
+ "theoretical_loss": 3.632938599106123,
+ "tokens_seen": 1140129792
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005289468405215648,
+ "loss": 1.3117,
+ "theoretical_loss": 3.6329193573284657,
+ "tokens_seen": 1140195328
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005289307923771314,
+ "loss": 1.3551,
+ "theoretical_loss": 3.6329001169664057,
+ "tokens_seen": 1140260864
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 383588,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.193879246711731,
+ "objective/train/theoretical_loss": 3.6328808780197583,
+ "objective/train/tokens_used": 1160786400,
+ "theoretical_loss": 3.6328808780197583,
+ "tokens_seen": 1140326400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005289147442326982,
+ "loss": 1.3129,
+ "theoretical_loss": 3.6328808780197583,
+ "tokens_seen": 1140326400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005288986960882648,
+ "loss": 1.3148,
+ "theoretical_loss": 3.632861640488338,
+ "tokens_seen": 1140391936
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005288826479438316,
+ "loss": 1.4849,
+ "theoretical_loss": 3.6328424043719587,
+ "tokens_seen": 1140457472
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005288665997993982,
+ "loss": 1.2353,
+ "theoretical_loss": 3.632823169670435,
+ "tokens_seen": 1140523008
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528850551654965,
+ "loss": 1.4659,
+ "theoretical_loss": 3.6328039363835822,
+ "tokens_seen": 1140588544
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005288345035105316,
+ "loss": 1.2511,
+ "theoretical_loss": 3.632784704511215,
+ "tokens_seen": 1140654080
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005288184553660984,
+ "loss": 1.3932,
+ "theoretical_loss": 3.632765474053148,
+ "tokens_seen": 1140719616
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528802407221665,
+ "loss": 1.2355,
+ "theoretical_loss": 3.632746245009195,
+ "tokens_seen": 1140785152
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287863590772318,
+ "loss": 1.418,
+ "theoretical_loss": 3.632727017379172,
+ "tokens_seen": 1140850688
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287703109327984,
+ "loss": 1.3035,
+ "theoretical_loss": 3.632707791162893,
+ "tokens_seen": 1140916224
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287542627883652,
+ "loss": 1.2606,
+ "theoretical_loss": 3.6326885663601733,
+ "tokens_seen": 1140981760
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287382146439318,
+ "loss": 1.3402,
+ "theoretical_loss": 3.6326693429708277,
+ "tokens_seen": 1141047296
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287221664994986,
+ "loss": 1.3486,
+ "theoretical_loss": 3.632650120994671,
+ "tokens_seen": 1141112832
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005287061183550652,
+ "loss": 1.3213,
+ "theoretical_loss": 3.6326309004315185,
+ "tokens_seen": 1141178368
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528690070210632,
+ "loss": 1.2077,
+ "theoretical_loss": 3.632611681281185,
+ "tokens_seen": 1141243904
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005286740220661986,
+ "loss": 1.3652,
+ "theoretical_loss": 3.632592463543485,
+ "tokens_seen": 1141309440
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005286579739217654,
+ "loss": 1.3507,
+ "theoretical_loss": 3.632573247218235,
+ "tokens_seen": 1141374976
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528641925777332,
+ "loss": 1.3774,
+ "theoretical_loss": 3.6325540323052485,
+ "tokens_seen": 1141440512
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005286258776328988,
+ "loss": 1.3804,
+ "theoretical_loss": 3.6325348188043414,
+ "tokens_seen": 1141506048
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005286098294884654,
+ "loss": 1.2376,
+ "theoretical_loss": 3.632515606715329,
+ "tokens_seen": 1141571584
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005285937813440322,
+ "loss": 1.1911,
+ "theoretical_loss": 3.6324963960380257,
+ "tokens_seen": 1141637120
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005285777331995988,
+ "loss": 1.3345,
+ "theoretical_loss": 3.632477186772248,
+ "tokens_seen": 1141702656
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005285616850551656,
+ "loss": 1.3125,
+ "theoretical_loss": 3.6324579789178095,
+ "tokens_seen": 1141768192
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005285456369107322,
+ "loss": 1.477,
+ "theoretical_loss": 3.6324387724745275,
+ "tokens_seen": 1141833728
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528529588766299,
+ "loss": 1.3276,
+ "theoretical_loss": 3.6324195674422155,
+ "tokens_seen": 1141899264
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005285135406218656,
+ "loss": 1.3045,
+ "theoretical_loss": 3.63240036382069,
+ "tokens_seen": 1141964800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284974924774324,
+ "loss": 1.2982,
+ "theoretical_loss": 3.632381161609766,
+ "tokens_seen": 1142030336
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528481444332999,
+ "loss": 1.178,
+ "theoretical_loss": 3.632361960809259,
+ "tokens_seen": 1142095872
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284653961885657,
+ "loss": 1.3646,
+ "theoretical_loss": 3.632342761418985,
+ "tokens_seen": 1142161408
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284493480441324,
+ "loss": 1.3485,
+ "theoretical_loss": 3.6323235634387587,
+ "tokens_seen": 1142226944
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284332998996991,
+ "loss": 1.462,
+ "theoretical_loss": 3.632304366868396,
+ "tokens_seen": 1142292480
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284172517552658,
+ "loss": 1.3052,
+ "theoretical_loss": 3.6322851717077125,
+ "tokens_seen": 1142358016
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005284012036108325,
+ "loss": 1.2837,
+ "theoretical_loss": 3.632265977956524,
+ "tokens_seen": 1142423552
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005283851554663992,
+ "loss": 1.3251,
+ "theoretical_loss": 3.6322467856146456,
+ "tokens_seen": 1142489088
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005283691073219659,
+ "loss": 1.3377,
+ "theoretical_loss": 3.6322275946818934,
+ "tokens_seen": 1142554624
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005283530591775326,
+ "loss": 1.2977,
+ "theoretical_loss": 3.632208405158083,
+ "tokens_seen": 1142620160
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005283370110330993,
+ "loss": 1.3095,
+ "theoretical_loss": 3.63218921704303,
+ "tokens_seen": 1142685696
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528320962888666,
+ "loss": 1.3895,
+ "theoretical_loss": 3.632170030336551,
+ "tokens_seen": 1142751232
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005283049147442327,
+ "loss": 1.2499,
+ "theoretical_loss": 3.6321508450384608,
+ "tokens_seen": 1142816768
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282888665997994,
+ "loss": 1.3816,
+ "theoretical_loss": 3.6321316611485757,
+ "tokens_seen": 1142882304
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282728184553661,
+ "loss": 1.2767,
+ "theoretical_loss": 3.632112478666712,
+ "tokens_seen": 1142947840
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282567703109328,
+ "loss": 1.4018,
+ "theoretical_loss": 3.6320932975926845,
+ "tokens_seen": 1143013376
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282407221664995,
+ "loss": 1.2862,
+ "theoretical_loss": 3.6320741179263103,
+ "tokens_seen": 1143078912
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282246740220662,
+ "loss": 1.2824,
+ "theoretical_loss": 3.632054939667405,
+ "tokens_seen": 1143144448
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005282086258776329,
+ "loss": 1.346,
+ "theoretical_loss": 3.6320357628157844,
+ "tokens_seen": 1143209984
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005281925777331996,
+ "loss": 1.4018,
+ "theoretical_loss": 3.6320165873712646,
+ "tokens_seen": 1143275520
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005281765295887663,
+ "loss": 1.3485,
+ "theoretical_loss": 3.6319974133336625,
+ "tokens_seen": 1143341056
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528160481444333,
+ "loss": 1.3261,
+ "theoretical_loss": 3.6319782407027934,
+ "tokens_seen": 1143406592
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005281444332998997,
+ "loss": 1.356,
+ "theoretical_loss": 3.631959069478474,
+ "tokens_seen": 1143472128
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005281283851554664,
+ "loss": 1.364,
+ "theoretical_loss": 3.6319398996605194,
+ "tokens_seen": 1143537664
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 384799,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5311000347137451,
+ "objective/train/theoretical_loss": 3.631920731248747,
+ "objective/train/tokens_used": 1164063200,
+ "theoretical_loss": 3.631920731248747,
+ "tokens_seen": 1143603200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005281123370110331,
+ "loss": 1.4664,
+ "theoretical_loss": 3.631920731248747,
+ "tokens_seen": 1143603200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280962888665998,
+ "loss": 1.3345,
+ "theoretical_loss": 3.6319015642429733,
+ "tokens_seen": 1143668736
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280802407221665,
+ "loss": 1.3935,
+ "theoretical_loss": 3.631882398643013,
+ "tokens_seen": 1143734272
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280641925777332,
+ "loss": 1.2661,
+ "theoretical_loss": 3.6318632344486845,
+ "tokens_seen": 1143799808
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280481444332999,
+ "loss": 1.3856,
+ "theoretical_loss": 3.6318440716598026,
+ "tokens_seen": 1143865344
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280320962888666,
+ "loss": 1.2881,
+ "theoretical_loss": 3.631824910276184,
+ "tokens_seen": 1143930880
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005280160481444333,
+ "loss": 1.3577,
+ "theoretical_loss": 3.631805750297646,
+ "tokens_seen": 1143996416
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000528,
+ "loss": 1.2978,
+ "theoretical_loss": 3.631786591724004,
+ "tokens_seen": 1144061952
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279839518555667,
+ "loss": 1.3174,
+ "theoretical_loss": 3.6317674345550754,
+ "tokens_seen": 1144127488
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279679037111334,
+ "loss": 1.3476,
+ "theoretical_loss": 3.6317482787906763,
+ "tokens_seen": 1144193024
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279518555667001,
+ "loss": 1.3092,
+ "theoretical_loss": 3.6317291244306236,
+ "tokens_seen": 1144258560
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279358074222668,
+ "loss": 1.2958,
+ "theoretical_loss": 3.6317099714747334,
+ "tokens_seen": 1144324096
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279197592778335,
+ "loss": 1.2297,
+ "theoretical_loss": 3.631690819922823,
+ "tokens_seen": 1144389632
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005279037111334002,
+ "loss": 1.345,
+ "theoretical_loss": 3.631671669774708,
+ "tokens_seen": 1144455168
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005278876629889669,
+ "loss": 1.3698,
+ "theoretical_loss": 3.6316525210302064,
+ "tokens_seen": 1144520704
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005278716148445336,
+ "loss": 1.4033,
+ "theoretical_loss": 3.6316333736891337,
+ "tokens_seen": 1144586240
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005278555667001003,
+ "loss": 1.3209,
+ "theoretical_loss": 3.6316142277513084,
+ "tokens_seen": 1144651776
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000527839518555667,
+ "loss": 1.2795,
+ "theoretical_loss": 3.6315950832165456,
+ "tokens_seen": 1144717312
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005278234704112337,
+ "loss": 1.3696,
+ "theoretical_loss": 3.6315759400846632,
+ "tokens_seen": 1144782848
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005278074222668004,
+ "loss": 1.3169,
+ "theoretical_loss": 3.631556798355478,
+ "tokens_seen": 1144848384
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277913741223671,
+ "loss": 1.3511,
+ "theoretical_loss": 3.6315376580288063,
+ "tokens_seen": 1144913920
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277753259779338,
+ "loss": 1.3794,
+ "theoretical_loss": 3.631518519104466,
+ "tokens_seen": 1144979456
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277592778335005,
+ "loss": 1.3197,
+ "theoretical_loss": 3.631499381582273,
+ "tokens_seen": 1145044992
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277432296890673,
+ "loss": 1.2908,
+ "theoretical_loss": 3.6314802454620456,
+ "tokens_seen": 1145110528
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277271815446339,
+ "loss": 1.3856,
+ "theoretical_loss": 3.6314611107436,
+ "tokens_seen": 1145176064
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005277111334002007,
+ "loss": 1.3486,
+ "theoretical_loss": 3.6314419774267535,
+ "tokens_seen": 1145241600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005276950852557673,
+ "loss": 1.2676,
+ "theoretical_loss": 3.631422845511323,
+ "tokens_seen": 1145307136
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000527679037111334,
+ "loss": 1.3219,
+ "theoretical_loss": 3.6314037149971257,
+ "tokens_seen": 1145372672
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005276629889669007,
+ "loss": 1.3939,
+ "theoretical_loss": 3.6313845858839793,
+ "tokens_seen": 1145438208
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005276469408224675,
+ "loss": 1.3421,
+ "theoretical_loss": 3.6313654581717008,
+ "tokens_seen": 1145503744
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005276308926780341,
+ "loss": 1.3355,
+ "theoretical_loss": 3.631346331860107,
+ "tokens_seen": 1145569280
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005276148445336009,
+ "loss": 1.3466,
+ "theoretical_loss": 3.631327206949016,
+ "tokens_seen": 1145634816
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275987963891675,
+ "loss": 1.3706,
+ "theoretical_loss": 3.6313080834382445,
+ "tokens_seen": 1145700352
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275827482447343,
+ "loss": 1.428,
+ "theoretical_loss": 3.631288961327611,
+ "tokens_seen": 1145765888
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275667001003009,
+ "loss": 1.4646,
+ "theoretical_loss": 3.6312698406169313,
+ "tokens_seen": 1145831424
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275506519558677,
+ "loss": 1.2793,
+ "theoretical_loss": 3.6312507213060234,
+ "tokens_seen": 1145896960
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275346038114343,
+ "loss": 1.3848,
+ "theoretical_loss": 3.6312316033947054,
+ "tokens_seen": 1145962496
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275185556670011,
+ "loss": 1.3406,
+ "theoretical_loss": 3.6312124868827946,
+ "tokens_seen": 1146028032
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005275025075225677,
+ "loss": 1.3627,
+ "theoretical_loss": 3.631193371770108,
+ "tokens_seen": 1146093568
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274864593781345,
+ "loss": 1.2938,
+ "theoretical_loss": 3.631174258056464,
+ "tokens_seen": 1146159104
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274704112337011,
+ "loss": 1.2908,
+ "theoretical_loss": 3.6311551457416793,
+ "tokens_seen": 1146224640
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274543630892679,
+ "loss": 1.2411,
+ "theoretical_loss": 3.6311360348255723,
+ "tokens_seen": 1146290176
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274383149448345,
+ "loss": 1.3654,
+ "theoretical_loss": 3.63111692530796,
+ "tokens_seen": 1146355712
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274222668004013,
+ "loss": 1.3042,
+ "theoretical_loss": 3.6310978171886608,
+ "tokens_seen": 1146421248
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005274062186559679,
+ "loss": 1.3152,
+ "theoretical_loss": 3.631078710467492,
+ "tokens_seen": 1146486784
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273901705115347,
+ "loss": 1.3923,
+ "theoretical_loss": 3.6310596051442716,
+ "tokens_seen": 1146552320
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273741223671013,
+ "loss": 1.3292,
+ "theoretical_loss": 3.6310405012188176,
+ "tokens_seen": 1146617856
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273580742226681,
+ "loss": 1.5038,
+ "theoretical_loss": 3.6310213986909474,
+ "tokens_seen": 1146683392
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273420260782347,
+ "loss": 1.3402,
+ "theoretical_loss": 3.631002297560479,
+ "tokens_seen": 1146748928
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273259779338015,
+ "loss": 1.3025,
+ "theoretical_loss": 3.6309831978272307,
+ "tokens_seen": 1146814464
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 386018,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2878888845443726,
+ "objective/train/theoretical_loss": 3.63096409949102,
+ "objective/train/tokens_used": 1167340000,
+ "theoretical_loss": 3.63096409949102,
+ "tokens_seen": 1146880000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005273099297893682,
+ "loss": 1.3574,
+ "theoretical_loss": 3.63096409949102,
+ "tokens_seen": 1146880000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005272938816449349,
+ "loss": 1.2588,
+ "theoretical_loss": 3.630945002551665,
+ "tokens_seen": 1146945536
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005272778335005016,
+ "loss": 1.3601,
+ "theoretical_loss": 3.630925907008984,
+ "tokens_seen": 1147011072
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005272617853560683,
+ "loss": 1.2096,
+ "theoretical_loss": 3.6309068128627953,
+ "tokens_seen": 1147076608
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000527245737211635,
+ "loss": 1.2368,
+ "theoretical_loss": 3.630887720112916,
+ "tokens_seen": 1147142144
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005272296890672017,
+ "loss": 1.2394,
+ "theoretical_loss": 3.630868628759165,
+ "tokens_seen": 1147207680
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005272136409227683,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6308495388013604,
+ "tokens_seen": 1147273216
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271975927783351,
+ "loss": 1.2866,
+ "theoretical_loss": 3.6308304502393196,
+ "tokens_seen": 1147338752
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271815446339017,
+ "loss": 1.2818,
+ "theoretical_loss": 3.630811363072862,
+ "tokens_seen": 1147404288
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271654964894685,
+ "loss": 1.2521,
+ "theoretical_loss": 3.6307922773018055,
+ "tokens_seen": 1147469824
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271494483450351,
+ "loss": 1.383,
+ "theoretical_loss": 3.630773192925968,
+ "tokens_seen": 1147535360
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271334002006019,
+ "loss": 1.292,
+ "theoretical_loss": 3.630754109945168,
+ "tokens_seen": 1147600896
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271173520561685,
+ "loss": 1.3332,
+ "theoretical_loss": 3.6307350283592243,
+ "tokens_seen": 1147666432
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005271013039117353,
+ "loss": 1.3052,
+ "theoretical_loss": 3.6307159481679547,
+ "tokens_seen": 1147731968
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005270852557673019,
+ "loss": 1.2793,
+ "theoretical_loss": 3.630696869371178,
+ "tokens_seen": 1147797504
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005270692076228686,
+ "loss": 1.2779,
+ "theoretical_loss": 3.6306777919687128,
+ "tokens_seen": 1147863040
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005270531594784353,
+ "loss": 1.3197,
+ "theoretical_loss": 3.630658715960377,
+ "tokens_seen": 1147928576
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000527037111334002,
+ "loss": 1.2939,
+ "theoretical_loss": 3.63063964134599,
+ "tokens_seen": 1147994112
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005270210631895687,
+ "loss": 1.3744,
+ "theoretical_loss": 3.6306205681253694,
+ "tokens_seen": 1148059648
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005270050150451354,
+ "loss": 1.4097,
+ "theoretical_loss": 3.6306014962983344,
+ "tokens_seen": 1148125184
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269889669007021,
+ "loss": 1.2644,
+ "theoretical_loss": 3.6305824258647035,
+ "tokens_seen": 1148190720
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269729187562688,
+ "loss": 1.3718,
+ "theoretical_loss": 3.6305633568242954,
+ "tokens_seen": 1148256256
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269568706118355,
+ "loss": 1.3295,
+ "theoretical_loss": 3.630544289176929,
+ "tokens_seen": 1148321792
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269408224674022,
+ "loss": 1.3056,
+ "theoretical_loss": 3.6305252229224223,
+ "tokens_seen": 1148387328
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269247743229689,
+ "loss": 1.2321,
+ "theoretical_loss": 3.6305061580605953,
+ "tokens_seen": 1148452864
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005269087261785356,
+ "loss": 1.3242,
+ "theoretical_loss": 3.630487094591266,
+ "tokens_seen": 1148518400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005268926780341023,
+ "loss": 1.3703,
+ "theoretical_loss": 3.630468032514253,
+ "tokens_seen": 1148583936
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526876629889669,
+ "loss": 1.2667,
+ "theoretical_loss": 3.6304489718293755,
+ "tokens_seen": 1148649472
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005268605817452357,
+ "loss": 1.3421,
+ "theoretical_loss": 3.630429912536453,
+ "tokens_seen": 1148715008
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005268445336008024,
+ "loss": 1.4368,
+ "theoretical_loss": 3.6304108546353033,
+ "tokens_seen": 1148780544
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005268284854563691,
+ "loss": 1.3279,
+ "theoretical_loss": 3.6303917981257467,
+ "tokens_seen": 1148846080
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005268124373119358,
+ "loss": 1.2554,
+ "theoretical_loss": 3.6303727430076007,
+ "tokens_seen": 1148911616
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267963891675025,
+ "loss": 1.2973,
+ "theoretical_loss": 3.630353689280686,
+ "tokens_seen": 1148977152
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267803410230692,
+ "loss": 1.3164,
+ "theoretical_loss": 3.63033463694482,
+ "tokens_seen": 1149042688
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267642928786359,
+ "loss": 1.241,
+ "theoretical_loss": 3.6303155859998233,
+ "tokens_seen": 1149108224
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267482447342026,
+ "loss": 1.2774,
+ "theoretical_loss": 3.630296536445514,
+ "tokens_seen": 1149173760
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267321965897693,
+ "loss": 1.4199,
+ "theoretical_loss": 3.630277488281712,
+ "tokens_seen": 1149239296
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526716148445336,
+ "loss": 1.3624,
+ "theoretical_loss": 3.6302584415082357,
+ "tokens_seen": 1149304832
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005267001003009027,
+ "loss": 1.2518,
+ "theoretical_loss": 3.630239396124905,
+ "tokens_seen": 1149370368
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266840521564694,
+ "loss": 1.3389,
+ "theoretical_loss": 3.630220352131539,
+ "tokens_seen": 1149435904
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266680040120361,
+ "loss": 1.3208,
+ "theoretical_loss": 3.6302013095279575,
+ "tokens_seen": 1149501440
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266519558676028,
+ "loss": 1.4276,
+ "theoretical_loss": 3.6301822683139786,
+ "tokens_seen": 1149566976
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266359077231695,
+ "loss": 1.2476,
+ "theoretical_loss": 3.6301632284894225,
+ "tokens_seen": 1149632512
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266198595787362,
+ "loss": 1.2424,
+ "theoretical_loss": 3.630144190054109,
+ "tokens_seen": 1149698048
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005266038114343029,
+ "loss": 1.3232,
+ "theoretical_loss": 3.6301251530078575,
+ "tokens_seen": 1149763584
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005265877632898696,
+ "loss": 1.2944,
+ "theoretical_loss": 3.6301061173504863,
+ "tokens_seen": 1149829120
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005265717151454363,
+ "loss": 1.4103,
+ "theoretical_loss": 3.6300870830818166,
+ "tokens_seen": 1149894656
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526555667001003,
+ "loss": 1.3234,
+ "theoretical_loss": 3.6300680502016665,
+ "tokens_seen": 1149960192
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005265396188565697,
+ "loss": 1.3174,
+ "theoretical_loss": 3.630049018709856,
+ "tokens_seen": 1150025728
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005265235707121364,
+ "loss": 1.3077,
+ "theoretical_loss": 3.6300299886062053,
+ "tokens_seen": 1150091264
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 386869,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3539170026779175,
+ "objective/train/theoretical_loss": 3.6300109598905332,
+ "objective/train/tokens_used": 1170616800,
+ "theoretical_loss": 3.6300109598905332,
+ "tokens_seen": 1150156800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005265075225677031,
+ "loss": 1.2592,
+ "theoretical_loss": 3.6300109598905332,
+ "tokens_seen": 1150156800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005264914744232698,
+ "loss": 1.4126,
+ "theoretical_loss": 3.6299919325626604,
+ "tokens_seen": 1150222336
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005264754262788365,
+ "loss": 1.2547,
+ "theoretical_loss": 3.629972906622406,
+ "tokens_seen": 1150287872
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005264593781344032,
+ "loss": 1.3082,
+ "theoretical_loss": 3.6299538820695894,
+ "tokens_seen": 1150353408
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.00052644332998997,
+ "loss": 1.3854,
+ "theoretical_loss": 3.629934858904031,
+ "tokens_seen": 1150418944
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005264272818455366,
+ "loss": 1.3252,
+ "theoretical_loss": 3.6299158371255507,
+ "tokens_seen": 1150484480
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005264112337011033,
+ "loss": 1.4755,
+ "theoretical_loss": 3.629896816733967,
+ "tokens_seen": 1150550016
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.00052639518555667,
+ "loss": 1.3732,
+ "theoretical_loss": 3.6298777977291024,
+ "tokens_seen": 1150615552
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005263791374122368,
+ "loss": 1.2985,
+ "theoretical_loss": 3.6298587801107747,
+ "tokens_seen": 1150681088
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005263630892678034,
+ "loss": 1.2555,
+ "theoretical_loss": 3.6298397638788042,
+ "tokens_seen": 1150746624
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005263470411233702,
+ "loss": 1.3708,
+ "theoretical_loss": 3.629820749033011,
+ "tokens_seen": 1150812160
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005263309929789368,
+ "loss": 1.2837,
+ "theoretical_loss": 3.6298017355732157,
+ "tokens_seen": 1150877696
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005263149448345036,
+ "loss": 1.3719,
+ "theoretical_loss": 3.629782723499238,
+ "tokens_seen": 1150943232
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005262988966900702,
+ "loss": 1.2352,
+ "theoretical_loss": 3.629763712810898,
+ "tokens_seen": 1151008768
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526282848545637,
+ "loss": 1.4079,
+ "theoretical_loss": 3.629744703508015,
+ "tokens_seen": 1151074304
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005262668004012036,
+ "loss": 1.3179,
+ "theoretical_loss": 3.6297256955904107,
+ "tokens_seen": 1151139840
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005262507522567704,
+ "loss": 1.2817,
+ "theoretical_loss": 3.6297066890579037,
+ "tokens_seen": 1151205376
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526234704112337,
+ "loss": 1.2657,
+ "theoretical_loss": 3.6296876839103156,
+ "tokens_seen": 1151270912
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005262186559679038,
+ "loss": 1.3571,
+ "theoretical_loss": 3.629668680147466,
+ "tokens_seen": 1151336448
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005262026078234704,
+ "loss": 1.4269,
+ "theoretical_loss": 3.6296496777691756,
+ "tokens_seen": 1151401984
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005261865596790372,
+ "loss": 1.3469,
+ "theoretical_loss": 3.629630676775264,
+ "tokens_seen": 1151467520
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005261705115346038,
+ "loss": 1.3293,
+ "theoretical_loss": 3.6296116771655518,
+ "tokens_seen": 1151533056
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005261544633901706,
+ "loss": 1.2247,
+ "theoretical_loss": 3.62959267893986,
+ "tokens_seen": 1151598592
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005261384152457372,
+ "loss": 1.327,
+ "theoretical_loss": 3.6295736820980085,
+ "tokens_seen": 1151664128
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000526122367101304,
+ "loss": 1.3716,
+ "theoretical_loss": 3.6295546866398176,
+ "tokens_seen": 1151729664
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005261063189568707,
+ "loss": 1.3355,
+ "theoretical_loss": 3.629535692565108,
+ "tokens_seen": 1151795200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260902708124374,
+ "loss": 1.3666,
+ "theoretical_loss": 3.6295166998737005,
+ "tokens_seen": 1151860736
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260742226680041,
+ "loss": 1.3776,
+ "theoretical_loss": 3.6294977085654154,
+ "tokens_seen": 1151926272
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260581745235708,
+ "loss": 1.3003,
+ "theoretical_loss": 3.6294787186400734,
+ "tokens_seen": 1151991808
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260421263791375,
+ "loss": 1.3106,
+ "theoretical_loss": 3.629459730097495,
+ "tokens_seen": 1152057344
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260260782347042,
+ "loss": 1.2752,
+ "theoretical_loss": 3.6294407429375006,
+ "tokens_seen": 1152122880
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005260100300902709,
+ "loss": 1.2717,
+ "theoretical_loss": 3.6294217571599114,
+ "tokens_seen": 1152188416
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005259939819458376,
+ "loss": 1.333,
+ "theoretical_loss": 3.629402772764548,
+ "tokens_seen": 1152253952
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005259779338014043,
+ "loss": 1.2802,
+ "theoretical_loss": 3.6293837897512313,
+ "tokens_seen": 1152319488
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525961885656971,
+ "loss": 1.3182,
+ "theoretical_loss": 3.6293648081197816,
+ "tokens_seen": 1152385024
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005259458375125377,
+ "loss": 1.319,
+ "theoretical_loss": 3.6293458278700204,
+ "tokens_seen": 1152450560
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005259297893681044,
+ "loss": 1.2196,
+ "theoretical_loss": 3.6293268490017674,
+ "tokens_seen": 1152516096
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005259137412236711,
+ "loss": 1.4032,
+ "theoretical_loss": 3.629307871514845,
+ "tokens_seen": 1152581632
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258976930792378,
+ "loss": 1.347,
+ "theoretical_loss": 3.6292888954090734,
+ "tokens_seen": 1152647168
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258816449348045,
+ "loss": 1.3088,
+ "theoretical_loss": 3.629269920684273,
+ "tokens_seen": 1152712704
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258655967903712,
+ "loss": 1.3199,
+ "theoretical_loss": 3.6292509473402657,
+ "tokens_seen": 1152778240
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258495486459379,
+ "loss": 1.4102,
+ "theoretical_loss": 3.6292319753768725,
+ "tokens_seen": 1152843776
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258335005015046,
+ "loss": 1.2946,
+ "theoretical_loss": 3.6292130047939137,
+ "tokens_seen": 1152909312
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005258174523570712,
+ "loss": 1.4109,
+ "theoretical_loss": 3.6291940355912113,
+ "tokens_seen": 1152974848
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525801404212638,
+ "loss": 1.3048,
+ "theoretical_loss": 3.6291750677685854,
+ "tokens_seen": 1153040384
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005257853560682046,
+ "loss": 1.3441,
+ "theoretical_loss": 3.6291561013258584,
+ "tokens_seen": 1153105920
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005257693079237714,
+ "loss": 1.2741,
+ "theoretical_loss": 3.629137136262851,
+ "tokens_seen": 1153171456
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525753259779338,
+ "loss": 1.3727,
+ "theoretical_loss": 3.6291181725793833,
+ "tokens_seen": 1153236992
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005257372116349048,
+ "loss": 1.2986,
+ "theoretical_loss": 3.629099210275278,
+ "tokens_seen": 1153302528
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005257211634904714,
+ "loss": 1.4147,
+ "theoretical_loss": 3.6290802493503564,
+ "tokens_seen": 1153368064
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 388139,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4212877750396729,
+ "objective/train/theoretical_loss": 3.6290612898044388,
+ "objective/train/tokens_used": 1173893600,
+ "theoretical_loss": 3.6290612898044388,
+ "tokens_seen": 1153433600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005257051153460382,
+ "loss": 1.3308,
+ "theoretical_loss": 3.6290612898044388,
+ "tokens_seen": 1153433600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256890672016048,
+ "loss": 1.3459,
+ "theoretical_loss": 3.6290423316373475,
+ "tokens_seen": 1153499136
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256730190571715,
+ "loss": 1.3637,
+ "theoretical_loss": 3.629023374848903,
+ "tokens_seen": 1153564672
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256569709127382,
+ "loss": 1.3223,
+ "theoretical_loss": 3.6290044194389277,
+ "tokens_seen": 1153630208
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256409227683049,
+ "loss": 1.3115,
+ "theoretical_loss": 3.6289854654072426,
+ "tokens_seen": 1153695744
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256248746238716,
+ "loss": 1.3293,
+ "theoretical_loss": 3.628966512753669,
+ "tokens_seen": 1153761280
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005256088264794383,
+ "loss": 1.3262,
+ "theoretical_loss": 3.6289475614780287,
+ "tokens_seen": 1153826816
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525592778335005,
+ "loss": 1.1924,
+ "theoretical_loss": 3.628928611580143,
+ "tokens_seen": 1153892352
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005255767301905717,
+ "loss": 1.2593,
+ "theoretical_loss": 3.628909663059834,
+ "tokens_seen": 1153957888
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005255606820461384,
+ "loss": 1.3586,
+ "theoretical_loss": 3.6288907159169232,
+ "tokens_seen": 1154023424
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005255446339017051,
+ "loss": 1.173,
+ "theoretical_loss": 3.628871770151232,
+ "tokens_seen": 1154088960
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005255285857572718,
+ "loss": 1.3188,
+ "theoretical_loss": 3.628852825762582,
+ "tokens_seen": 1154154496
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005255125376128385,
+ "loss": 1.3508,
+ "theoretical_loss": 3.6288338827507953,
+ "tokens_seen": 1154220032
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254964894684052,
+ "loss": 1.2634,
+ "theoretical_loss": 3.628814941115693,
+ "tokens_seen": 1154285568
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254804413239719,
+ "loss": 1.3406,
+ "theoretical_loss": 3.628796000857098,
+ "tokens_seen": 1154351104
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254643931795386,
+ "loss": 1.2375,
+ "theoretical_loss": 3.628777061974831,
+ "tokens_seen": 1154416640
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254483450351053,
+ "loss": 1.3291,
+ "theoretical_loss": 3.628758124468715,
+ "tokens_seen": 1154482176
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525432296890672,
+ "loss": 1.3058,
+ "theoretical_loss": 3.6287391883385705,
+ "tokens_seen": 1154547712
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254162487462387,
+ "loss": 1.3351,
+ "theoretical_loss": 3.6287202535842207,
+ "tokens_seen": 1154613248
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005254002006018054,
+ "loss": 1.2449,
+ "theoretical_loss": 3.6287013202054865,
+ "tokens_seen": 1154678784
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253841524573721,
+ "loss": 1.2602,
+ "theoretical_loss": 3.628682388202191,
+ "tokens_seen": 1154744320
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253681043129388,
+ "loss": 1.3141,
+ "theoretical_loss": 3.628663457574155,
+ "tokens_seen": 1154809856
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253520561685055,
+ "loss": 1.2782,
+ "theoretical_loss": 3.628644528321202,
+ "tokens_seen": 1154875392
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253360080240722,
+ "loss": 1.2779,
+ "theoretical_loss": 3.628625600443153,
+ "tokens_seen": 1154940928
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253199598796389,
+ "loss": 1.2693,
+ "theoretical_loss": 3.6286066739398306,
+ "tokens_seen": 1155006464
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005253039117352056,
+ "loss": 1.4455,
+ "theoretical_loss": 3.6285877488110563,
+ "tokens_seen": 1155072000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005252878635907723,
+ "loss": 1.3163,
+ "theoretical_loss": 3.6285688250566532,
+ "tokens_seen": 1155137536
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525271815446339,
+ "loss": 1.1737,
+ "theoretical_loss": 3.6285499026764434,
+ "tokens_seen": 1155203072
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005252557673019057,
+ "loss": 1.362,
+ "theoretical_loss": 3.6285309816702482,
+ "tokens_seen": 1155268608
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005252397191574724,
+ "loss": 1.2838,
+ "theoretical_loss": 3.628512062037891,
+ "tokens_seen": 1155334144
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005252236710130391,
+ "loss": 1.4258,
+ "theoretical_loss": 3.6284931437791936,
+ "tokens_seen": 1155399680
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005252076228686058,
+ "loss": 1.417,
+ "theoretical_loss": 3.628474226893978,
+ "tokens_seen": 1155465216
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005251915747241725,
+ "loss": 1.2895,
+ "theoretical_loss": 3.628455311382068,
+ "tokens_seen": 1155530752
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005251755265797392,
+ "loss": 1.3196,
+ "theoretical_loss": 3.6284363972432843,
+ "tokens_seen": 1155596288
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005251594784353059,
+ "loss": 1.3434,
+ "theoretical_loss": 3.628417484477451,
+ "tokens_seen": 1155661824
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005251434302908726,
+ "loss": 1.3113,
+ "theoretical_loss": 3.6283985730843886,
+ "tokens_seen": 1155727360
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005251273821464393,
+ "loss": 1.2642,
+ "theoretical_loss": 3.628379663063921,
+ "tokens_seen": 1155792896
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000525111334002006,
+ "loss": 1.2885,
+ "theoretical_loss": 3.628360754415871,
+ "tokens_seen": 1155858432
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250952858575727,
+ "loss": 1.3405,
+ "theoretical_loss": 3.62834184714006,
+ "tokens_seen": 1155923968
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250792377131394,
+ "loss": 1.4213,
+ "theoretical_loss": 3.628322941236312,
+ "tokens_seen": 1155989504
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250631895687061,
+ "loss": 1.3425,
+ "theoretical_loss": 3.6283040367044483,
+ "tokens_seen": 1156055040
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250471414242728,
+ "loss": 1.3402,
+ "theoretical_loss": 3.628285133544293,
+ "tokens_seen": 1156120576
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250310932798395,
+ "loss": 1.3353,
+ "theoretical_loss": 3.6282662317556675,
+ "tokens_seen": 1156186112
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005250150451354063,
+ "loss": 1.3254,
+ "theoretical_loss": 3.6282473313383954,
+ "tokens_seen": 1156251648
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249989969909729,
+ "loss": 1.3593,
+ "theoretical_loss": 3.628228432292299,
+ "tokens_seen": 1156317184
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249829488465397,
+ "loss": 1.3537,
+ "theoretical_loss": 3.628209534617201,
+ "tokens_seen": 1156382720
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249669007021063,
+ "loss": 1.3434,
+ "theoretical_loss": 3.628190638312925,
+ "tokens_seen": 1156448256
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000524950852557673,
+ "loss": 1.2878,
+ "theoretical_loss": 3.628171743379294,
+ "tokens_seen": 1156513792
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249348044132397,
+ "loss": 1.407,
+ "theoretical_loss": 3.62815284981613,
+ "tokens_seen": 1156579328
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249187562688065,
+ "loss": 1.35,
+ "theoretical_loss": 3.6281339576232563,
+ "tokens_seen": 1156644864
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 389093,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3961246013641357,
+ "objective/train/theoretical_loss": 3.628115066800496,
+ "objective/train/tokens_used": 1177170400,
+ "theoretical_loss": 3.628115066800496,
+ "tokens_seen": 1156710400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005249027081243732,
+ "loss": 1.4008,
+ "theoretical_loss": 3.628115066800496,
+ "tokens_seen": 1156710400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005248866599799399,
+ "loss": 1.3283,
+ "theoretical_loss": 3.628096177347672,
+ "tokens_seen": 1156775936
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005248706118355066,
+ "loss": 1.4352,
+ "theoretical_loss": 3.6280772892646067,
+ "tokens_seen": 1156841472
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005248545636910733,
+ "loss": 1.388,
+ "theoretical_loss": 3.6280584025511247,
+ "tokens_seen": 1156907008
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.00052483851554664,
+ "loss": 1.3747,
+ "theoretical_loss": 3.628039517207048,
+ "tokens_seen": 1156972544
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005248224674022067,
+ "loss": 1.3926,
+ "theoretical_loss": 3.6280206332322,
+ "tokens_seen": 1157038080
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005248064192577734,
+ "loss": 1.3103,
+ "theoretical_loss": 3.628001750626404,
+ "tokens_seen": 1157103616
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247903711133401,
+ "loss": 1.2729,
+ "theoretical_loss": 3.627982869389483,
+ "tokens_seen": 1157169152
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247743229689068,
+ "loss": 1.2684,
+ "theoretical_loss": 3.6279639895212608,
+ "tokens_seen": 1157234688
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247582748244735,
+ "loss": 1.2934,
+ "theoretical_loss": 3.6279451110215595,
+ "tokens_seen": 1157300224
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247422266800402,
+ "loss": 1.2891,
+ "theoretical_loss": 3.6279262338902036,
+ "tokens_seen": 1157365760
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247261785356069,
+ "loss": 1.4099,
+ "theoretical_loss": 3.6279073581270165,
+ "tokens_seen": 1157431296
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005247101303911736,
+ "loss": 1.3125,
+ "theoretical_loss": 3.6278884837318204,
+ "tokens_seen": 1157496832
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005246940822467403,
+ "loss": 1.2276,
+ "theoretical_loss": 3.6278696107044395,
+ "tokens_seen": 1157562368
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000524678034102307,
+ "loss": 1.3279,
+ "theoretical_loss": 3.6278507390446975,
+ "tokens_seen": 1157627904
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005246619859578737,
+ "loss": 1.3733,
+ "theoretical_loss": 3.627831868752417,
+ "tokens_seen": 1157693440
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005246459378134404,
+ "loss": 1.2596,
+ "theoretical_loss": 3.6278129998274227,
+ "tokens_seen": 1157758976
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005246298896690071,
+ "loss": 1.2671,
+ "theoretical_loss": 3.6277941322695373,
+ "tokens_seen": 1157824512
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005246138415245738,
+ "loss": 1.3346,
+ "theoretical_loss": 3.627775266078584,
+ "tokens_seen": 1157890048
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245977933801405,
+ "loss": 1.2842,
+ "theoretical_loss": 3.6277564012543877,
+ "tokens_seen": 1157955584
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245817452357072,
+ "loss": 1.27,
+ "theoretical_loss": 3.6277375377967704,
+ "tokens_seen": 1158021120
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245656970912739,
+ "loss": 1.3695,
+ "theoretical_loss": 3.627718675705558,
+ "tokens_seen": 1158086656
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245496489468406,
+ "loss": 1.2826,
+ "theoretical_loss": 3.6276998149805717,
+ "tokens_seen": 1158152192
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245336008024073,
+ "loss": 1.3259,
+ "theoretical_loss": 3.627680955621637,
+ "tokens_seen": 1158217728
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000524517552657974,
+ "loss": 1.2803,
+ "theoretical_loss": 3.627662097628577,
+ "tokens_seen": 1158283264
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005245015045135407,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6276432410012154,
+ "tokens_seen": 1158348800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244854563691074,
+ "loss": 1.3034,
+ "theoretical_loss": 3.6276243857393764,
+ "tokens_seen": 1158414336
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244694082246741,
+ "loss": 1.4376,
+ "theoretical_loss": 3.627605531842884,
+ "tokens_seen": 1158479872
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244533600802408,
+ "loss": 1.246,
+ "theoretical_loss": 3.6275866793115616,
+ "tokens_seen": 1158545408
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244373119358075,
+ "loss": 1.3351,
+ "theoretical_loss": 3.6275678281452333,
+ "tokens_seen": 1158610944
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244212637913741,
+ "loss": 1.3593,
+ "theoretical_loss": 3.627548978343723,
+ "tokens_seen": 1158676480
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005244052156469409,
+ "loss": 1.3827,
+ "theoretical_loss": 3.627530129906855,
+ "tokens_seen": 1158742016
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243891675025075,
+ "loss": 1.4156,
+ "theoretical_loss": 3.627511282834453,
+ "tokens_seen": 1158807552
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243731193580743,
+ "loss": 1.2261,
+ "theoretical_loss": 3.6274924371263415,
+ "tokens_seen": 1158873088
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243570712136409,
+ "loss": 1.3422,
+ "theoretical_loss": 3.627473592782344,
+ "tokens_seen": 1158938624
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243410230692077,
+ "loss": 1.4268,
+ "theoretical_loss": 3.627454749802285,
+ "tokens_seen": 1159004160
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243249749247743,
+ "loss": 1.3028,
+ "theoretical_loss": 3.6274359081859884,
+ "tokens_seen": 1159069696
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005243089267803411,
+ "loss": 1.3061,
+ "theoretical_loss": 3.627417067933279,
+ "tokens_seen": 1159135232
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242928786359077,
+ "loss": 1.326,
+ "theoretical_loss": 3.6273982290439806,
+ "tokens_seen": 1159200768
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242768304914744,
+ "loss": 1.4346,
+ "theoretical_loss": 3.627379391517917,
+ "tokens_seen": 1159266304
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242607823470411,
+ "loss": 1.2694,
+ "theoretical_loss": 3.6273605553549135,
+ "tokens_seen": 1159331840
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242447342026078,
+ "loss": 1.361,
+ "theoretical_loss": 3.6273417205547935,
+ "tokens_seen": 1159397376
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242286860581745,
+ "loss": 1.3586,
+ "theoretical_loss": 3.627322887117382,
+ "tokens_seen": 1159462912
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005242126379137412,
+ "loss": 1.2966,
+ "theoretical_loss": 3.6273040550425035,
+ "tokens_seen": 1159528448
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241965897693079,
+ "loss": 1.3173,
+ "theoretical_loss": 3.627285224329982,
+ "tokens_seen": 1159593984
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241805416248746,
+ "loss": 1.3016,
+ "theoretical_loss": 3.6272663949796415,
+ "tokens_seen": 1159659520
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241644934804413,
+ "loss": 1.2903,
+ "theoretical_loss": 3.6272475669913073,
+ "tokens_seen": 1159725056
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000524148445336008,
+ "loss": 1.2825,
+ "theoretical_loss": 3.627228740364804,
+ "tokens_seen": 1159790592
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241323971915747,
+ "loss": 1.33,
+ "theoretical_loss": 3.6272099150999555,
+ "tokens_seen": 1159856128
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241163490471414,
+ "loss": 1.3692,
+ "theoretical_loss": 3.627191091196586,
+ "tokens_seen": 1159921664
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 390304,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.382239580154419,
+ "objective/train/theoretical_loss": 3.6271722686545216,
+ "objective/train/tokens_used": 1180447200,
+ "theoretical_loss": 3.6271722686545216,
+ "tokens_seen": 1159987200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005241003009027081,
+ "loss": 1.3223,
+ "theoretical_loss": 3.6271722686545216,
+ "tokens_seen": 1159987200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240842527582748,
+ "loss": 1.323,
+ "theoretical_loss": 3.6271534474735865,
+ "tokens_seen": 1160052736
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240682046138415,
+ "loss": 1.3826,
+ "theoretical_loss": 3.6271346276536045,
+ "tokens_seen": 1160118272
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240521564694082,
+ "loss": 1.3941,
+ "theoretical_loss": 3.627115809194401,
+ "tokens_seen": 1160183808
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240361083249749,
+ "loss": 1.3702,
+ "theoretical_loss": 3.627096992095801,
+ "tokens_seen": 1160249344
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240200601805416,
+ "loss": 1.3475,
+ "theoretical_loss": 3.627078176357628,
+ "tokens_seen": 1160314880
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005240040120361083,
+ "loss": 1.3882,
+ "theoretical_loss": 3.6270593619797085,
+ "tokens_seen": 1160380416
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523987963891675,
+ "loss": 1.3651,
+ "theoretical_loss": 3.6270405489618662,
+ "tokens_seen": 1160445952
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005239719157472417,
+ "loss": 1.3723,
+ "theoretical_loss": 3.6270217373039264,
+ "tokens_seen": 1160511488
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005239558676028084,
+ "loss": 1.3694,
+ "theoretical_loss": 3.627002927005714,
+ "tokens_seen": 1160577024
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005239398194583751,
+ "loss": 1.3012,
+ "theoretical_loss": 3.626984118067054,
+ "tokens_seen": 1160642560
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005239237713139418,
+ "loss": 1.322,
+ "theoretical_loss": 3.6269653104877713,
+ "tokens_seen": 1160708096
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005239077231695085,
+ "loss": 1.3307,
+ "theoretical_loss": 3.626946504267691,
+ "tokens_seen": 1160773632
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005238916750250752,
+ "loss": 1.3745,
+ "theoretical_loss": 3.6269276994066377,
+ "tokens_seen": 1160839168
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005238756268806419,
+ "loss": 1.265,
+ "theoretical_loss": 3.6269088959044367,
+ "tokens_seen": 1160904704
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005238595787362086,
+ "loss": 1.3075,
+ "theoretical_loss": 3.626890093760913,
+ "tokens_seen": 1160970240
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005238435305917753,
+ "loss": 1.4269,
+ "theoretical_loss": 3.626871292975893,
+ "tokens_seen": 1161035776
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523827482447342,
+ "loss": 1.3602,
+ "theoretical_loss": 3.6268524935491997,
+ "tokens_seen": 1161101312
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005238114343029087,
+ "loss": 1.2614,
+ "theoretical_loss": 3.62683369548066,
+ "tokens_seen": 1161166848
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005237953861584754,
+ "loss": 1.4316,
+ "theoretical_loss": 3.6268148987700983,
+ "tokens_seen": 1161232384
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005237793380140421,
+ "loss": 1.3308,
+ "theoretical_loss": 3.62679610341734,
+ "tokens_seen": 1161297920
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005237632898696088,
+ "loss": 1.3643,
+ "theoretical_loss": 3.626777309422211,
+ "tokens_seen": 1161363456
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005237472417251755,
+ "loss": 1.3086,
+ "theoretical_loss": 3.6267585167845358,
+ "tokens_seen": 1161428992
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005237311935807422,
+ "loss": 1.3594,
+ "theoretical_loss": 3.62673972550414,
+ "tokens_seen": 1161494528
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523715145436309,
+ "loss": 1.4302,
+ "theoretical_loss": 3.6267209355808494,
+ "tokens_seen": 1161560064
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236990972918757,
+ "loss": 1.4588,
+ "theoretical_loss": 3.6267021470144893,
+ "tokens_seen": 1161625600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236830491474423,
+ "loss": 1.3555,
+ "theoretical_loss": 3.6266833598048844,
+ "tokens_seen": 1161691136
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236670010030091,
+ "loss": 1.4816,
+ "theoretical_loss": 3.626664573951861,
+ "tokens_seen": 1161756672
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236509528585757,
+ "loss": 1.3267,
+ "theoretical_loss": 3.626645789455245,
+ "tokens_seen": 1161822208
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236349047141425,
+ "loss": 1.442,
+ "theoretical_loss": 3.6266270063148607,
+ "tokens_seen": 1161887744
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236188565697092,
+ "loss": 1.2699,
+ "theoretical_loss": 3.6266082245305347,
+ "tokens_seen": 1161953280
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005236028084252759,
+ "loss": 1.4851,
+ "theoretical_loss": 3.626589444102092,
+ "tokens_seen": 1162018816
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005235867602808426,
+ "loss": 1.4912,
+ "theoretical_loss": 3.6265706650293588,
+ "tokens_seen": 1162084352
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005235707121364093,
+ "loss": 1.506,
+ "theoretical_loss": 3.6265518873121607,
+ "tokens_seen": 1162149888
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523554663991976,
+ "loss": 1.2892,
+ "theoretical_loss": 3.626533110950323,
+ "tokens_seen": 1162215424
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005235386158475427,
+ "loss": 1.3803,
+ "theoretical_loss": 3.6265143359436722,
+ "tokens_seen": 1162280960
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005235225677031094,
+ "loss": 1.3633,
+ "theoretical_loss": 3.6264955622920327,
+ "tokens_seen": 1162346496
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005235065195586761,
+ "loss": 1.2449,
+ "theoretical_loss": 3.626476789995232,
+ "tokens_seen": 1162412032
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234904714142428,
+ "loss": 1.373,
+ "theoretical_loss": 3.6264580190530946,
+ "tokens_seen": 1162477568
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234744232698095,
+ "loss": 1.3181,
+ "theoretical_loss": 3.626439249465448,
+ "tokens_seen": 1162543104
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234583751253762,
+ "loss": 1.2979,
+ "theoretical_loss": 3.626420481232116,
+ "tokens_seen": 1162608640
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234423269809429,
+ "loss": 1.2596,
+ "theoretical_loss": 3.6264017143529257,
+ "tokens_seen": 1162674176
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234262788365096,
+ "loss": 1.2634,
+ "theoretical_loss": 3.626382948827703,
+ "tokens_seen": 1162739712
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005234102306920763,
+ "loss": 1.3723,
+ "theoretical_loss": 3.626364184656274,
+ "tokens_seen": 1162805248
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523394182547643,
+ "loss": 1.3086,
+ "theoretical_loss": 3.6263454218384648,
+ "tokens_seen": 1162870784
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005233781344032097,
+ "loss": 1.3344,
+ "theoretical_loss": 3.626326660374101,
+ "tokens_seen": 1162936320
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005233620862587764,
+ "loss": 1.4669,
+ "theoretical_loss": 3.6263079002630096,
+ "tokens_seen": 1163001856
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005233460381143431,
+ "loss": 1.4078,
+ "theoretical_loss": 3.6262891415050156,
+ "tokens_seen": 1163067392
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005233299899699098,
+ "loss": 1.2619,
+ "theoretical_loss": 3.6262703840999455,
+ "tokens_seen": 1163132928
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005233139418254765,
+ "loss": 1.2902,
+ "theoretical_loss": 3.6262516280476262,
+ "tokens_seen": 1163198464
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 391310,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3289260864257812,
+ "objective/train/theoretical_loss": 3.6262328733478837,
+ "objective/train/tokens_used": 1183724000,
+ "theoretical_loss": 3.6262328733478837,
+ "tokens_seen": 1163264000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232978936810432,
+ "loss": 1.4095,
+ "theoretical_loss": 3.6262328733478837,
+ "tokens_seen": 1163264000
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232818455366099,
+ "loss": 1.3516,
+ "theoretical_loss": 3.6262141200005433,
+ "tokens_seen": 1163329536
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232657973921766,
+ "loss": 1.3389,
+ "theoretical_loss": 3.6261953680054324,
+ "tokens_seen": 1163395072
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232497492477433,
+ "loss": 1.3979,
+ "theoretical_loss": 3.6261766173623773,
+ "tokens_seen": 1163460608
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.00052323370110331,
+ "loss": 1.4256,
+ "theoretical_loss": 3.6261578680712034,
+ "tokens_seen": 1163526144
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232176529588767,
+ "loss": 1.3421,
+ "theoretical_loss": 3.6261391201317377,
+ "tokens_seen": 1163591680
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005232016048144434,
+ "loss": 1.3171,
+ "theoretical_loss": 3.6261203735438072,
+ "tokens_seen": 1163657216
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231855566700101,
+ "loss": 1.3133,
+ "theoretical_loss": 3.6261016283072376,
+ "tokens_seen": 1163722752
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231695085255768,
+ "loss": 1.3757,
+ "theoretical_loss": 3.626082884421855,
+ "tokens_seen": 1163788288
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231534603811435,
+ "loss": 1.3293,
+ "theoretical_loss": 3.6260641418874875,
+ "tokens_seen": 1163853824
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231374122367102,
+ "loss": 1.3661,
+ "theoretical_loss": 3.62604540070396,
+ "tokens_seen": 1163919360
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231213640922769,
+ "loss": 1.3974,
+ "theoretical_loss": 3.6260266608711005,
+ "tokens_seen": 1163984896
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005231053159478436,
+ "loss": 1.3269,
+ "theoretical_loss": 3.626007922388734,
+ "tokens_seen": 1164050432
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005230892678034103,
+ "loss": 1.3172,
+ "theoretical_loss": 3.6259891852566883,
+ "tokens_seen": 1164115968
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000523073219658977,
+ "loss": 1.2811,
+ "theoretical_loss": 3.62597044947479,
+ "tokens_seen": 1164181504
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005230571715145437,
+ "loss": 1.2927,
+ "theoretical_loss": 3.6259517150428655,
+ "tokens_seen": 1164247040
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005230411233701104,
+ "loss": 1.3156,
+ "theoretical_loss": 3.6259329819607418,
+ "tokens_seen": 1164312576
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005230250752256771,
+ "loss": 1.3063,
+ "theoretical_loss": 3.625914250228245,
+ "tokens_seen": 1164378112
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005230090270812438,
+ "loss": 1.4155,
+ "theoretical_loss": 3.625895519845203,
+ "tokens_seen": 1164443648
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005229929789368104,
+ "loss": 1.3127,
+ "theoretical_loss": 3.625876790811442,
+ "tokens_seen": 1164509184
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005229769307923772,
+ "loss": 1.4032,
+ "theoretical_loss": 3.625858063126789,
+ "tokens_seen": 1164574720
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005229608826479438,
+ "loss": 1.2937,
+ "theoretical_loss": 3.625839336791071,
+ "tokens_seen": 1164640256
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005229448345035106,
+ "loss": 1.3879,
+ "theoretical_loss": 3.625820611804115,
+ "tokens_seen": 1164705792
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005229287863590772,
+ "loss": 1.334,
+ "theoretical_loss": 3.6258018881657472,
+ "tokens_seen": 1164771328
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522912738214644,
+ "loss": 1.3847,
+ "theoretical_loss": 3.6257831658757955,
+ "tokens_seen": 1164836864
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228966900702106,
+ "loss": 1.3339,
+ "theoretical_loss": 3.6257644449340867,
+ "tokens_seen": 1164902400
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228806419257773,
+ "loss": 1.2804,
+ "theoretical_loss": 3.6257457253404475,
+ "tokens_seen": 1164967936
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522864593781344,
+ "loss": 1.212,
+ "theoretical_loss": 3.6257270070947056,
+ "tokens_seen": 1165033472
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228485456369107,
+ "loss": 1.3679,
+ "theoretical_loss": 3.6257082901966884,
+ "tokens_seen": 1165099008
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228324974924774,
+ "loss": 1.4327,
+ "theoretical_loss": 3.6256895746462217,
+ "tokens_seen": 1165164544
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228164493480441,
+ "loss": 1.3036,
+ "theoretical_loss": 3.625670860443134,
+ "tokens_seen": 1165230080
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005228004012036108,
+ "loss": 1.3006,
+ "theoretical_loss": 3.6256521475872514,
+ "tokens_seen": 1165295616
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005227843530591775,
+ "loss": 1.2842,
+ "theoretical_loss": 3.6256334360784024,
+ "tokens_seen": 1165361152
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005227683049147442,
+ "loss": 1.3809,
+ "theoretical_loss": 3.625614725916413,
+ "tokens_seen": 1165426688
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005227522567703109,
+ "loss": 1.2579,
+ "theoretical_loss": 3.6255960171011115,
+ "tokens_seen": 1165492224
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005227362086258776,
+ "loss": 1.3692,
+ "theoretical_loss": 3.6255773096323254,
+ "tokens_seen": 1165557760
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005227201604814443,
+ "loss": 1.2346,
+ "theoretical_loss": 3.625558603509881,
+ "tokens_seen": 1165623296
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522704112337011,
+ "loss": 1.2338,
+ "theoretical_loss": 3.6255398987336065,
+ "tokens_seen": 1165688832
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226880641925777,
+ "loss": 1.3505,
+ "theoretical_loss": 3.6255211953033295,
+ "tokens_seen": 1165754368
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226720160481444,
+ "loss": 1.3322,
+ "theoretical_loss": 3.6255024932188773,
+ "tokens_seen": 1165819904
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226559679037111,
+ "loss": 1.397,
+ "theoretical_loss": 3.625483792480077,
+ "tokens_seen": 1165885440
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226399197592778,
+ "loss": 1.3544,
+ "theoretical_loss": 3.6254650930867562,
+ "tokens_seen": 1165950976
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226238716148445,
+ "loss": 1.3373,
+ "theoretical_loss": 3.625446395038743,
+ "tokens_seen": 1166016512
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005226078234704112,
+ "loss": 1.4169,
+ "theoretical_loss": 3.625427698335865,
+ "tokens_seen": 1166082048
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005225917753259779,
+ "loss": 1.4729,
+ "theoretical_loss": 3.625409002977949,
+ "tokens_seen": 1166147584
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005225757271815446,
+ "loss": 1.3368,
+ "theoretical_loss": 3.6253903089648234,
+ "tokens_seen": 1166213120
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005225596790371113,
+ "loss": 1.2234,
+ "theoretical_loss": 3.6253716162963157,
+ "tokens_seen": 1166278656
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522543630892678,
+ "loss": 1.3524,
+ "theoretical_loss": 3.625352924972254,
+ "tokens_seen": 1166344192
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005225275827482447,
+ "loss": 1.3436,
+ "theoretical_loss": 3.625334234992465,
+ "tokens_seen": 1166409728
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005225115346038114,
+ "loss": 1.3063,
+ "theoretical_loss": 3.625315546356778,
+ "tokens_seen": 1166475264
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 392693,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.403052806854248,
+ "objective/train/theoretical_loss": 3.62529685906502,
+ "objective/train/tokens_used": 1187000800,
+ "theoretical_loss": 3.62529685906502,
+ "tokens_seen": 1166540800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005224954864593782,
+ "loss": 1.382,
+ "theoretical_loss": 3.62529685906502,
+ "tokens_seen": 1166540800
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005224794383149448,
+ "loss": 1.3795,
+ "theoretical_loss": 3.625278173117019,
+ "tokens_seen": 1166606336
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005224633901705116,
+ "loss": 1.3338,
+ "theoretical_loss": 3.6252594885126026,
+ "tokens_seen": 1166671872
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005224473420260782,
+ "loss": 1.2931,
+ "theoretical_loss": 3.625240805251599,
+ "tokens_seen": 1166737408
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522431293881645,
+ "loss": 1.3088,
+ "theoretical_loss": 3.625222123333836,
+ "tokens_seen": 1166802944
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005224152457372116,
+ "loss": 1.409,
+ "theoretical_loss": 3.6252034427591417,
+ "tokens_seen": 1166868480
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223991975927784,
+ "loss": 1.4136,
+ "theoretical_loss": 3.6251847635273444,
+ "tokens_seen": 1166934016
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522383149448345,
+ "loss": 1.34,
+ "theoretical_loss": 3.6251660856382717,
+ "tokens_seen": 1166999552
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223671013039118,
+ "loss": 1.4028,
+ "theoretical_loss": 3.6251474090917517,
+ "tokens_seen": 1167065088
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223510531594784,
+ "loss": 1.2295,
+ "theoretical_loss": 3.625128733887613,
+ "tokens_seen": 1167130624
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223350050150452,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6251100600256834,
+ "tokens_seen": 1167196160
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223189568706118,
+ "loss": 1.2543,
+ "theoretical_loss": 3.6250913875057904,
+ "tokens_seen": 1167261696
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005223029087261786,
+ "loss": 1.307,
+ "theoretical_loss": 3.6250727163277636,
+ "tokens_seen": 1167327232
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005222868605817452,
+ "loss": 1.3599,
+ "theoretical_loss": 3.6250540464914307,
+ "tokens_seen": 1167392768
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522270812437312,
+ "loss": 1.3695,
+ "theoretical_loss": 3.6250353779966193,
+ "tokens_seen": 1167458304
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005222547642928787,
+ "loss": 1.239,
+ "theoretical_loss": 3.625016710843159,
+ "tokens_seen": 1167523840
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005222387161484454,
+ "loss": 1.3781,
+ "theoretical_loss": 3.6249980450308765,
+ "tokens_seen": 1167589376
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522222668004012,
+ "loss": 1.2695,
+ "theoretical_loss": 3.6249793805596013,
+ "tokens_seen": 1167654912
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005222066198595788,
+ "loss": 1.4009,
+ "theoretical_loss": 3.624960717429162,
+ "tokens_seen": 1167720448
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005221905717151455,
+ "loss": 1.3018,
+ "theoretical_loss": 3.6249420556393863,
+ "tokens_seen": 1167785984
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005221745235707122,
+ "loss": 1.3338,
+ "theoretical_loss": 3.6249233951901028,
+ "tokens_seen": 1167851520
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005221584754262789,
+ "loss": 1.3396,
+ "theoretical_loss": 3.62490473608114,
+ "tokens_seen": 1167917056
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005221424272818456,
+ "loss": 1.4346,
+ "theoretical_loss": 3.624886078312327,
+ "tokens_seen": 1167982592
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005221263791374123,
+ "loss": 1.3923,
+ "theoretical_loss": 3.6248674218834918,
+ "tokens_seen": 1168048128
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000522110330992979,
+ "loss": 1.3886,
+ "theoretical_loss": 3.6248487667944627,
+ "tokens_seen": 1168113664
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220942828485457,
+ "loss": 1.3706,
+ "theoretical_loss": 3.624830113045069,
+ "tokens_seen": 1168179200
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220782347041124,
+ "loss": 1.3453,
+ "theoretical_loss": 3.624811460635139,
+ "tokens_seen": 1168244736
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220621865596791,
+ "loss": 1.2965,
+ "theoretical_loss": 3.624792809564502,
+ "tokens_seen": 1168310272
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220461384152458,
+ "loss": 1.2545,
+ "theoretical_loss": 3.6247741598329855,
+ "tokens_seen": 1168375808
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220300902708125,
+ "loss": 1.3799,
+ "theoretical_loss": 3.6247555114404193,
+ "tokens_seen": 1168441344
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005220140421263792,
+ "loss": 1.2763,
+ "theoretical_loss": 3.624736864386632,
+ "tokens_seen": 1168506880
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219979939819459,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6247182186714517,
+ "tokens_seen": 1168572416
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219819458375126,
+ "loss": 1.4166,
+ "theoretical_loss": 3.6246995742947083,
+ "tokens_seen": 1168637952
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219658976930793,
+ "loss": 1.3735,
+ "theoretical_loss": 3.6246809312562296,
+ "tokens_seen": 1168703488
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000521949849548646,
+ "loss": 1.3781,
+ "theoretical_loss": 3.6246622895558454,
+ "tokens_seen": 1168769024
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219338014042127,
+ "loss": 1.4817,
+ "theoretical_loss": 3.624643649193384,
+ "tokens_seen": 1168834560
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219177532597794,
+ "loss": 1.321,
+ "theoretical_loss": 3.6246250101686748,
+ "tokens_seen": 1168900096
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005219017051153461,
+ "loss": 1.2949,
+ "theoretical_loss": 3.624606372481547,
+ "tokens_seen": 1168965632
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218856569709128,
+ "loss": 1.3907,
+ "theoretical_loss": 3.624587736131829,
+ "tokens_seen": 1169031168
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218696088264795,
+ "loss": 1.3235,
+ "theoretical_loss": 3.62456910111935,
+ "tokens_seen": 1169096704
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218535606820462,
+ "loss": 1.3947,
+ "theoretical_loss": 3.6245504674439393,
+ "tokens_seen": 1169162240
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218375125376129,
+ "loss": 1.2989,
+ "theoretical_loss": 3.624531835105426,
+ "tokens_seen": 1169227776
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218214643931796,
+ "loss": 1.4171,
+ "theoretical_loss": 3.624513204103639,
+ "tokens_seen": 1169293312
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005218054162487463,
+ "loss": 1.3067,
+ "theoretical_loss": 3.6244945744384083,
+ "tokens_seen": 1169358848
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000521789368104313,
+ "loss": 1.3451,
+ "theoretical_loss": 3.6244759461095617,
+ "tokens_seen": 1169424384
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005217733199598797,
+ "loss": 1.3654,
+ "theoretical_loss": 3.62445731911693,
+ "tokens_seen": 1169489920
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005217572718154464,
+ "loss": 1.3649,
+ "theoretical_loss": 3.6244386934603416,
+ "tokens_seen": 1169555456
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005217412236710131,
+ "loss": 1.3148,
+ "theoretical_loss": 3.6244200691396258,
+ "tokens_seen": 1169620992
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005217251755265798,
+ "loss": 1.2983,
+ "theoretical_loss": 3.624401446154612,
+ "tokens_seen": 1169686528
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005217091273821465,
+ "loss": 1.255,
+ "theoretical_loss": 3.6243828245051297,
+ "tokens_seen": 1169752064
+ },
+ {
+ "epoch": 0.35,
+ "objective/train/docs_used": 393563,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2607676982879639,
+ "objective/train/theoretical_loss": 3.624364204191009,
+ "objective/train/tokens_used": 1190277600,
+ "theoretical_loss": 3.624364204191009,
+ "tokens_seen": 1169817600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005216930792377132,
+ "loss": 1.2299,
+ "theoretical_loss": 3.624364204191009,
+ "tokens_seen": 1169817600
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005216770310932799,
+ "loss": 1.3175,
+ "theoretical_loss": 3.624345585212078,
+ "tokens_seen": 1169883136
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005216609829488466,
+ "loss": 1.4497,
+ "theoretical_loss": 3.624326967568167,
+ "tokens_seen": 1169948672
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005216449348044133,
+ "loss": 1.3844,
+ "theoretical_loss": 3.624308351259105,
+ "tokens_seen": 1170014208
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.00052162888665998,
+ "loss": 1.3511,
+ "theoretical_loss": 3.624289736284722,
+ "tokens_seen": 1170079744
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005216128385155467,
+ "loss": 1.2804,
+ "theoretical_loss": 3.6242711226448474,
+ "tokens_seen": 1170145280
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215967903711133,
+ "loss": 1.2419,
+ "theoretical_loss": 3.624252510339311,
+ "tokens_seen": 1170210816
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215807422266801,
+ "loss": 1.2794,
+ "theoretical_loss": 3.624233899367942,
+ "tokens_seen": 1170276352
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215646940822467,
+ "loss": 1.2997,
+ "theoretical_loss": 3.6242152897305706,
+ "tokens_seen": 1170341888
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215486459378135,
+ "loss": 1.278,
+ "theoretical_loss": 3.624196681427026,
+ "tokens_seen": 1170407424
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215325977933801,
+ "loss": 1.3937,
+ "theoretical_loss": 3.624178074457138,
+ "tokens_seen": 1170472960
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215165496489469,
+ "loss": 1.3346,
+ "theoretical_loss": 3.6241594688207366,
+ "tokens_seen": 1170538496
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005215005015045135,
+ "loss": 1.3682,
+ "theoretical_loss": 3.6241408645176514,
+ "tokens_seen": 1170604032
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005214844533600803,
+ "loss": 1.2385,
+ "theoretical_loss": 3.624122261547712,
+ "tokens_seen": 1170669568
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005214684052156469,
+ "loss": 1.3522,
+ "theoretical_loss": 3.624103659910749,
+ "tokens_seen": 1170735104
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005214523570712136,
+ "loss": 1.2782,
+ "theoretical_loss": 3.6240850596065917,
+ "tokens_seen": 1170800640
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005214363089267803,
+ "loss": 1.3125,
+ "theoretical_loss": 3.6240664606350697,
+ "tokens_seen": 1170866176
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.000521420260782347,
+ "loss": 1.3384,
+ "theoretical_loss": 3.6240478629960133,
+ "tokens_seen": 1170931712
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005214042126379137,
+ "loss": 1.3271,
+ "theoretical_loss": 3.624029266689253,
+ "tokens_seen": 1170997248
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213881644934804,
+ "loss": 1.4168,
+ "theoretical_loss": 3.6240106717146174,
+ "tokens_seen": 1171062784
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213721163490471,
+ "loss": 1.3534,
+ "theoretical_loss": 3.6239920780719386,
+ "tokens_seen": 1171128320
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213560682046138,
+ "loss": 1.4672,
+ "theoretical_loss": 3.6239734857610446,
+ "tokens_seen": 1171193856
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213400200601805,
+ "loss": 1.3727,
+ "theoretical_loss": 3.623954894781767,
+ "tokens_seen": 1171259392
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213239719157472,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6239363051339346,
+ "tokens_seen": 1171324928
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005213079237713139,
+ "loss": 1.3116,
+ "theoretical_loss": 3.623917716817379,
+ "tokens_seen": 1171390464
+ },
+ {
+ "epoch": 0.35,
+ "learning_rate": 0.0005212918756268807,
+ "loss": 1.2633,
+ "theoretical_loss": 3.623899129831929,
+ "tokens_seen": 1171456000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005212758274824473,
+ "loss": 1.3765,
+ "theoretical_loss": 3.623880544177416,
+ "tokens_seen": 1171521536
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005212597793380141,
+ "loss": 1.3197,
+ "theoretical_loss": 3.6238619598536697,
+ "tokens_seen": 1171587072
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005212437311935807,
+ "loss": 1.4117,
+ "theoretical_loss": 3.6238433768605205,
+ "tokens_seen": 1171652608
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005212276830491475,
+ "loss": 1.3921,
+ "theoretical_loss": 3.623824795197798,
+ "tokens_seen": 1171718144
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005212116349047141,
+ "loss": 1.2963,
+ "theoretical_loss": 3.6238062148653336,
+ "tokens_seen": 1171783680
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211955867602809,
+ "loss": 1.3229,
+ "theoretical_loss": 3.623787635862957,
+ "tokens_seen": 1171849216
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211795386158475,
+ "loss": 1.3587,
+ "theoretical_loss": 3.6237690581904993,
+ "tokens_seen": 1171914752
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211634904714143,
+ "loss": 1.3557,
+ "theoretical_loss": 3.6237504818477904,
+ "tokens_seen": 1171980288
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211474423269809,
+ "loss": 1.4415,
+ "theoretical_loss": 3.6237319068346605,
+ "tokens_seen": 1172045824
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211313941825477,
+ "loss": 1.31,
+ "theoretical_loss": 3.6237133331509406,
+ "tokens_seen": 1172111360
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005211153460381143,
+ "loss": 1.3444,
+ "theoretical_loss": 3.623694760796461,
+ "tokens_seen": 1172176896
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210992978936811,
+ "loss": 1.3207,
+ "theoretical_loss": 3.6236761897710528,
+ "tokens_seen": 1172242432
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210832497492477,
+ "loss": 1.2704,
+ "theoretical_loss": 3.623657620074546,
+ "tokens_seen": 1172307968
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210672016048145,
+ "loss": 1.3772,
+ "theoretical_loss": 3.623639051706771,
+ "tokens_seen": 1172373504
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210511534603811,
+ "loss": 1.3308,
+ "theoretical_loss": 3.6236204846675593,
+ "tokens_seen": 1172439040
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210351053159479,
+ "loss": 1.2544,
+ "theoretical_loss": 3.6236019189567408,
+ "tokens_seen": 1172504576
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210190571715145,
+ "loss": 1.3313,
+ "theoretical_loss": 3.623583354574147,
+ "tokens_seen": 1172570112
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005210030090270813,
+ "loss": 1.3694,
+ "theoretical_loss": 3.623564791519607,
+ "tokens_seen": 1172635648
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520986960882648,
+ "loss": 1.5035,
+ "theoretical_loss": 3.623546229792954,
+ "tokens_seen": 1172701184
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005209709127382147,
+ "loss": 1.3508,
+ "theoretical_loss": 3.6235276693940173,
+ "tokens_seen": 1172766720
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005209548645937813,
+ "loss": 1.2774,
+ "theoretical_loss": 3.623509110322628,
+ "tokens_seen": 1172832256
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005209388164493481,
+ "loss": 1.4738,
+ "theoretical_loss": 3.623490552578617,
+ "tokens_seen": 1172897792
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005209227683049147,
+ "loss": 1.2105,
+ "theoretical_loss": 3.6234719961618147,
+ "tokens_seen": 1172963328
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005209067201604815,
+ "loss": 1.2656,
+ "theoretical_loss": 3.623453441072053,
+ "tokens_seen": 1173028864
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 394748,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4302725791931152,
+ "objective/train/theoretical_loss": 3.623434887309162,
+ "objective/train/tokens_used": 1193554400,
+ "theoretical_loss": 3.623434887309162,
+ "tokens_seen": 1173094400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005208906720160482,
+ "loss": 1.3879,
+ "theoretical_loss": 3.623434887309162,
+ "tokens_seen": 1173094400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005208746238716149,
+ "loss": 1.3935,
+ "theoretical_loss": 3.6234163348729735,
+ "tokens_seen": 1173159936
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005208585757271816,
+ "loss": 1.3199,
+ "theoretical_loss": 3.623397783763318,
+ "tokens_seen": 1173225472
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005208425275827483,
+ "loss": 1.2786,
+ "theoretical_loss": 3.623379233980027,
+ "tokens_seen": 1173291008
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520826479438315,
+ "loss": 1.3943,
+ "theoretical_loss": 3.623360685522931,
+ "tokens_seen": 1173356544
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005208104312938817,
+ "loss": 1.4016,
+ "theoretical_loss": 3.6233421383918607,
+ "tokens_seen": 1173422080
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207943831494484,
+ "loss": 1.4327,
+ "theoretical_loss": 3.623323592586649,
+ "tokens_seen": 1173487616
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207783350050151,
+ "loss": 1.3676,
+ "theoretical_loss": 3.6233050481071256,
+ "tokens_seen": 1173553152
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207622868605818,
+ "loss": 1.297,
+ "theoretical_loss": 3.623286504953122,
+ "tokens_seen": 1173618688
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207462387161485,
+ "loss": 1.2294,
+ "theoretical_loss": 3.6232679631244697,
+ "tokens_seen": 1173684224
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207301905717152,
+ "loss": 1.2713,
+ "theoretical_loss": 3.623249422621,
+ "tokens_seen": 1173749760
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005207141424272819,
+ "loss": 1.3539,
+ "theoretical_loss": 3.6232308834425435,
+ "tokens_seen": 1173815296
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206980942828486,
+ "loss": 1.28,
+ "theoretical_loss": 3.623212345588933,
+ "tokens_seen": 1173880832
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206820461384153,
+ "loss": 1.4316,
+ "theoretical_loss": 3.623193809059998,
+ "tokens_seen": 1173946368
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520665997993982,
+ "loss": 1.3246,
+ "theoretical_loss": 3.623175273855572,
+ "tokens_seen": 1174011904
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206499498495487,
+ "loss": 1.3382,
+ "theoretical_loss": 3.6231567399754843,
+ "tokens_seen": 1174077440
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206339017051154,
+ "loss": 1.4022,
+ "theoretical_loss": 3.6231382074195677,
+ "tokens_seen": 1174142976
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206178535606821,
+ "loss": 1.3721,
+ "theoretical_loss": 3.6231196761876534,
+ "tokens_seen": 1174208512
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005206018054162488,
+ "loss": 1.3147,
+ "theoretical_loss": 3.6231011462795726,
+ "tokens_seen": 1174274048
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005205857572718155,
+ "loss": 1.3632,
+ "theoretical_loss": 3.6230826176951574,
+ "tokens_seen": 1174339584
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005205697091273822,
+ "loss": 1.3908,
+ "theoretical_loss": 3.623064090434239,
+ "tokens_seen": 1174405120
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005205536609829489,
+ "loss": 1.3546,
+ "theoretical_loss": 3.623045564496649,
+ "tokens_seen": 1174470656
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005205376128385156,
+ "loss": 1.4011,
+ "theoretical_loss": 3.6230270398822197,
+ "tokens_seen": 1174536192
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005205215646940823,
+ "loss": 1.3003,
+ "theoretical_loss": 3.6230085165907813,
+ "tokens_seen": 1174601728
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520505516549649,
+ "loss": 1.3056,
+ "theoretical_loss": 3.6229899946221673,
+ "tokens_seen": 1174667264
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204894684052157,
+ "loss": 1.354,
+ "theoretical_loss": 3.6229714739762082,
+ "tokens_seen": 1174732800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204734202607824,
+ "loss": 1.3905,
+ "theoretical_loss": 3.622952954652736,
+ "tokens_seen": 1174798336
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204573721163491,
+ "loss": 1.3324,
+ "theoretical_loss": 3.622934436651583,
+ "tokens_seen": 1174863872
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204413239719158,
+ "loss": 1.2093,
+ "theoretical_loss": 3.6229159199725807,
+ "tokens_seen": 1174929408
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204252758274825,
+ "loss": 1.2937,
+ "theoretical_loss": 3.6228974046155606,
+ "tokens_seen": 1174994944
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005204092276830492,
+ "loss": 1.3784,
+ "theoretical_loss": 3.6228788905803553,
+ "tokens_seen": 1175060480
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005203931795386159,
+ "loss": 1.2328,
+ "theoretical_loss": 3.6228603778667967,
+ "tokens_seen": 1175126016
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005203771313941826,
+ "loss": 1.2746,
+ "theoretical_loss": 3.6228418664747157,
+ "tokens_seen": 1175191552
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005203610832497493,
+ "loss": 1.3738,
+ "theoretical_loss": 3.622823356403945,
+ "tokens_seen": 1175257088
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520345035105316,
+ "loss": 1.3804,
+ "theoretical_loss": 3.622804847654317,
+ "tokens_seen": 1175322624
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005203289869608827,
+ "loss": 1.5001,
+ "theoretical_loss": 3.6227863402256633,
+ "tokens_seen": 1175388160
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005203129388164494,
+ "loss": 1.4309,
+ "theoretical_loss": 3.6227678341178158,
+ "tokens_seen": 1175453696
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202968906720161,
+ "loss": 1.4174,
+ "theoretical_loss": 3.622749329330607,
+ "tokens_seen": 1175519232
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202808425275828,
+ "loss": 1.3994,
+ "theoretical_loss": 3.6227308258638686,
+ "tokens_seen": 1175584768
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202647943831495,
+ "loss": 1.3172,
+ "theoretical_loss": 3.6227123237174332,
+ "tokens_seen": 1175650304
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202487462387162,
+ "loss": 1.3803,
+ "theoretical_loss": 3.622693822891133,
+ "tokens_seen": 1175715840
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202326980942829,
+ "loss": 1.2936,
+ "theoretical_loss": 3.6226753233848,
+ "tokens_seen": 1175781376
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202166499498496,
+ "loss": 1.3158,
+ "theoretical_loss": 3.6226568251982667,
+ "tokens_seen": 1175846912
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005202006018054162,
+ "loss": 1.3924,
+ "theoretical_loss": 3.622638328331365,
+ "tokens_seen": 1175912448
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520184553660983,
+ "loss": 1.3513,
+ "theoretical_loss": 3.6226198327839274,
+ "tokens_seen": 1175977984
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005201685055165496,
+ "loss": 1.3921,
+ "theoretical_loss": 3.6226013385557865,
+ "tokens_seen": 1176043520
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005201524573721164,
+ "loss": 1.2881,
+ "theoretical_loss": 3.6225828456467744,
+ "tokens_seen": 1176109056
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000520136409227683,
+ "loss": 1.2704,
+ "theoretical_loss": 3.6225643540567236,
+ "tokens_seen": 1176174592
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005201203610832498,
+ "loss": 1.3317,
+ "theoretical_loss": 3.622545863785467,
+ "tokens_seen": 1176240128
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005201043129388164,
+ "loss": 1.3436,
+ "theoretical_loss": 3.6225273748328357,
+ "tokens_seen": 1176305664
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 395873,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5824906826019287,
+ "objective/train/theoretical_loss": 3.6225088871986637,
+ "objective/train/tokens_used": 1196831200,
+ "theoretical_loss": 3.6225088871986637,
+ "tokens_seen": 1176371200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005200882647943832,
+ "loss": 1.442,
+ "theoretical_loss": 3.6225088871986637,
+ "tokens_seen": 1176371200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005200722166499498,
+ "loss": 1.3582,
+ "theoretical_loss": 3.622490400882783,
+ "tokens_seen": 1176436736
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005200561685055166,
+ "loss": 1.2754,
+ "theoretical_loss": 3.6224719158850256,
+ "tokens_seen": 1176502272
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005200401203610832,
+ "loss": 1.3573,
+ "theoretical_loss": 3.622453432205225,
+ "tokens_seen": 1176567808
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00052002407221665,
+ "loss": 1.3548,
+ "theoretical_loss": 3.622434949843213,
+ "tokens_seen": 1176633344
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005200080240722166,
+ "loss": 1.3114,
+ "theoretical_loss": 3.622416468798823,
+ "tokens_seen": 1176698880
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005199919759277834,
+ "loss": 1.3777,
+ "theoretical_loss": 3.622397989071888,
+ "tokens_seen": 1176764416
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00051997592778335,
+ "loss": 1.2718,
+ "theoretical_loss": 3.6223795106622396,
+ "tokens_seen": 1176829952
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005199598796389168,
+ "loss": 1.2786,
+ "theoretical_loss": 3.622361033569711,
+ "tokens_seen": 1176895488
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005199438314944834,
+ "loss": 1.3207,
+ "theoretical_loss": 3.622342557794135,
+ "tokens_seen": 1176961024
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005199277833500502,
+ "loss": 1.3457,
+ "theoretical_loss": 3.6223240833353447,
+ "tokens_seen": 1177026560
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005199117352056168,
+ "loss": 1.3378,
+ "theoretical_loss": 3.6223056101931728,
+ "tokens_seen": 1177092096
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005198956870611836,
+ "loss": 1.346,
+ "theoretical_loss": 3.6222871383674518,
+ "tokens_seen": 1177157632
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005198796389167502,
+ "loss": 1.3451,
+ "theoretical_loss": 3.6222686678580147,
+ "tokens_seen": 1177223168
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519863590772317,
+ "loss": 1.4074,
+ "theoretical_loss": 3.6222501986646947,
+ "tokens_seen": 1177288704
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005198475426278836,
+ "loss": 1.3488,
+ "theoretical_loss": 3.622231730787325,
+ "tokens_seen": 1177354240
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005198314944834504,
+ "loss": 1.3536,
+ "theoretical_loss": 3.622213264225738,
+ "tokens_seen": 1177419776
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519815446339017,
+ "loss": 1.3691,
+ "theoretical_loss": 3.622194798979767,
+ "tokens_seen": 1177485312
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197993981945838,
+ "loss": 1.3549,
+ "theoretical_loss": 3.6221763350492457,
+ "tokens_seen": 1177550848
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197833500501504,
+ "loss": 1.3276,
+ "theoretical_loss": 3.6221578724340056,
+ "tokens_seen": 1177616384
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197673019057172,
+ "loss": 1.2978,
+ "theoretical_loss": 3.622139411133881,
+ "tokens_seen": 1177681920
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197512537612838,
+ "loss": 1.2934,
+ "theoretical_loss": 3.622120951148705,
+ "tokens_seen": 1177747456
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197352056168506,
+ "loss": 1.3778,
+ "theoretical_loss": 3.6221024924783105,
+ "tokens_seen": 1177812992
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005197191574724172,
+ "loss": 1.3326,
+ "theoretical_loss": 3.6220840351225307,
+ "tokens_seen": 1177878528
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519703109327984,
+ "loss": 1.3671,
+ "theoretical_loss": 3.6220655790811986,
+ "tokens_seen": 1177944064
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005196870611835506,
+ "loss": 1.2988,
+ "theoretical_loss": 3.622047124354148,
+ "tokens_seen": 1178009600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005196710130391174,
+ "loss": 1.3643,
+ "theoretical_loss": 3.622028670941212,
+ "tokens_seen": 1178075136
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519654964894684,
+ "loss": 1.2444,
+ "theoretical_loss": 3.6220102188422243,
+ "tokens_seen": 1178140672
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005196389167502508,
+ "loss": 1.4353,
+ "theoretical_loss": 3.6219917680570175,
+ "tokens_seen": 1178206208
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005196228686058174,
+ "loss": 1.3703,
+ "theoretical_loss": 3.6219733185854253,
+ "tokens_seen": 1178271744
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005196068204613842,
+ "loss": 1.3585,
+ "theoretical_loss": 3.621954870427282,
+ "tokens_seen": 1178337280
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005195907723169508,
+ "loss": 1.3883,
+ "theoretical_loss": 3.6219364235824187,
+ "tokens_seen": 1178402816
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005195747241725176,
+ "loss": 1.4193,
+ "theoretical_loss": 3.6219179780506714,
+ "tokens_seen": 1178468352
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005195586760280842,
+ "loss": 1.3301,
+ "theoretical_loss": 3.621899533831873,
+ "tokens_seen": 1178533888
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519542627883651,
+ "loss": 1.3868,
+ "theoretical_loss": 3.621881090925856,
+ "tokens_seen": 1178599424
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005195265797392177,
+ "loss": 1.2952,
+ "theoretical_loss": 3.6218626493324546,
+ "tokens_seen": 1178664960
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005195105315947844,
+ "loss": 1.3821,
+ "theoretical_loss": 3.6218442090515026,
+ "tokens_seen": 1178730496
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519494483450351,
+ "loss": 1.4559,
+ "theoretical_loss": 3.6218257700828334,
+ "tokens_seen": 1178796032
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005194784353059178,
+ "loss": 1.3733,
+ "theoretical_loss": 3.6218073324262807,
+ "tokens_seen": 1178861568
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005194623871614845,
+ "loss": 1.2628,
+ "theoretical_loss": 3.6217888960816786,
+ "tokens_seen": 1178927104
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005194463390170512,
+ "loss": 1.3803,
+ "theoretical_loss": 3.62177046104886,
+ "tokens_seen": 1178992640
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005194302908726179,
+ "loss": 1.3956,
+ "theoretical_loss": 3.6217520273276596,
+ "tokens_seen": 1179058176
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005194142427281846,
+ "loss": 1.3624,
+ "theoretical_loss": 3.6217335949179104,
+ "tokens_seen": 1179123712
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193981945837513,
+ "loss": 1.4153,
+ "theoretical_loss": 3.6217151638194465,
+ "tokens_seen": 1179189248
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519382146439318,
+ "loss": 1.317,
+ "theoretical_loss": 3.621696734032102,
+ "tokens_seen": 1179254784
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193660982948847,
+ "loss": 1.3697,
+ "theoretical_loss": 3.62167830555571,
+ "tokens_seen": 1179320320
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193500501504514,
+ "loss": 1.3489,
+ "theoretical_loss": 3.6216598783901053,
+ "tokens_seen": 1179385856
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193340020060181,
+ "loss": 1.3594,
+ "theoretical_loss": 3.6216414525351217,
+ "tokens_seen": 1179451392
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193179538615848,
+ "loss": 1.3834,
+ "theoretical_loss": 3.6216230279905925,
+ "tokens_seen": 1179516928
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005193019057171515,
+ "loss": 1.3705,
+ "theoretical_loss": 3.621604604756352,
+ "tokens_seen": 1179582464
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 396821,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.240499496459961,
+ "objective/train/theoretical_loss": 3.621586182832235,
+ "objective/train/tokens_used": 1200108000,
+ "theoretical_loss": 3.621586182832235,
+ "tokens_seen": 1179648000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005192858575727182,
+ "loss": 1.2934,
+ "theoretical_loss": 3.621586182832235,
+ "tokens_seen": 1179648000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005192698094282849,
+ "loss": 1.2857,
+ "theoretical_loss": 3.621567762218074,
+ "tokens_seen": 1179713536
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005192537612838516,
+ "loss": 1.3332,
+ "theoretical_loss": 3.621549342913705,
+ "tokens_seen": 1179779072
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005192377131394183,
+ "loss": 1.2775,
+ "theoretical_loss": 3.621530924918961,
+ "tokens_seen": 1179844608
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519221664994985,
+ "loss": 1.3946,
+ "theoretical_loss": 3.6215125082336757,
+ "tokens_seen": 1179910144
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005192056168505517,
+ "loss": 1.2751,
+ "theoretical_loss": 3.621494092857684,
+ "tokens_seen": 1179975680
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191895687061184,
+ "loss": 1.3485,
+ "theoretical_loss": 3.62147567879082,
+ "tokens_seen": 1180041216
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191735205616851,
+ "loss": 1.3742,
+ "theoretical_loss": 3.621457266032918,
+ "tokens_seen": 1180106752
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191574724172518,
+ "loss": 1.281,
+ "theoretical_loss": 3.621438854583812,
+ "tokens_seen": 1180172288
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191414242728185,
+ "loss": 1.4004,
+ "theoretical_loss": 3.621420444443337,
+ "tokens_seen": 1180237824
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191253761283852,
+ "loss": 1.2787,
+ "theoretical_loss": 3.621402035611326,
+ "tokens_seen": 1180303360
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005191093279839519,
+ "loss": 1.4473,
+ "theoretical_loss": 3.6213836280876146,
+ "tokens_seen": 1180368896
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005190932798395186,
+ "loss": 1.3844,
+ "theoretical_loss": 3.6213652218720367,
+ "tokens_seen": 1180434432
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005190772316950853,
+ "loss": 1.38,
+ "theoretical_loss": 3.6213468169644267,
+ "tokens_seen": 1180499968
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000519061183550652,
+ "loss": 1.3562,
+ "theoretical_loss": 3.621328413364619,
+ "tokens_seen": 1180565504
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005190451354062187,
+ "loss": 1.3592,
+ "theoretical_loss": 3.621310011072448,
+ "tokens_seen": 1180631040
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005190290872617854,
+ "loss": 1.4273,
+ "theoretical_loss": 3.621291610087749,
+ "tokens_seen": 1180696576
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005190130391173521,
+ "loss": 1.3981,
+ "theoretical_loss": 3.6212732104103558,
+ "tokens_seen": 1180762112
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189969909729188,
+ "loss": 1.4812,
+ "theoretical_loss": 3.621254812040103,
+ "tokens_seen": 1180827648
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189809428284855,
+ "loss": 1.3422,
+ "theoretical_loss": 3.621236414976825,
+ "tokens_seen": 1180893184
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189648946840522,
+ "loss": 1.5115,
+ "theoretical_loss": 3.621218019220357,
+ "tokens_seen": 1180958720
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189488465396189,
+ "loss": 1.3807,
+ "theoretical_loss": 3.621199624770534,
+ "tokens_seen": 1181024256
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189327983951856,
+ "loss": 1.2891,
+ "theoretical_loss": 3.621181231627189,
+ "tokens_seen": 1181089792
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005189167502507523,
+ "loss": 1.3865,
+ "theoretical_loss": 3.6211628397901583,
+ "tokens_seen": 1181155328
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000518900702106319,
+ "loss": 1.363,
+ "theoretical_loss": 3.621144449259276,
+ "tokens_seen": 1181220864
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188846539618857,
+ "loss": 1.3727,
+ "theoretical_loss": 3.6211260600343773,
+ "tokens_seen": 1181286400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188686058174524,
+ "loss": 1.3512,
+ "theoretical_loss": 3.6211076721152966,
+ "tokens_seen": 1181351936
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188525576730191,
+ "loss": 1.4427,
+ "theoretical_loss": 3.621089285501869,
+ "tokens_seen": 1181417472
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188365095285858,
+ "loss": 1.3918,
+ "theoretical_loss": 3.621070900193929,
+ "tokens_seen": 1181483008
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188204613841525,
+ "loss": 1.2408,
+ "theoretical_loss": 3.621052516191312,
+ "tokens_seen": 1181548544
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005188044132397191,
+ "loss": 1.3617,
+ "theoretical_loss": 3.6210341334938527,
+ "tokens_seen": 1181614080
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187883650952859,
+ "loss": 1.3777,
+ "theoretical_loss": 3.621015752101386,
+ "tokens_seen": 1181679616
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187723169508525,
+ "loss": 1.3017,
+ "theoretical_loss": 3.620997372013747,
+ "tokens_seen": 1181745152
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187562688064193,
+ "loss": 1.4179,
+ "theoretical_loss": 3.6209789932307705,
+ "tokens_seen": 1181810688
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187402206619859,
+ "loss": 1.321,
+ "theoretical_loss": 3.620960615752292,
+ "tokens_seen": 1181876224
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187241725175527,
+ "loss": 1.2739,
+ "theoretical_loss": 3.620942239578146,
+ "tokens_seen": 1181941760
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005187081243731193,
+ "loss": 1.3463,
+ "theoretical_loss": 3.620923864708168,
+ "tokens_seen": 1182007296
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186920762286861,
+ "loss": 1.2876,
+ "theoretical_loss": 3.620905491142193,
+ "tokens_seen": 1182072832
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186760280842527,
+ "loss": 1.4414,
+ "theoretical_loss": 3.6208871188800567,
+ "tokens_seen": 1182138368
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186599799398195,
+ "loss": 1.3373,
+ "theoretical_loss": 3.620868747921593,
+ "tokens_seen": 1182203904
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186439317953861,
+ "loss": 1.4227,
+ "theoretical_loss": 3.6208503782666384,
+ "tokens_seen": 1182269440
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186278836509529,
+ "loss": 1.4225,
+ "theoretical_loss": 3.620832009915028,
+ "tokens_seen": 1182334976
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005186118355065195,
+ "loss": 1.3005,
+ "theoretical_loss": 3.6208136428665965,
+ "tokens_seen": 1182400512
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185957873620863,
+ "loss": 1.275,
+ "theoretical_loss": 3.6207952771211795,
+ "tokens_seen": 1182466048
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185797392176529,
+ "loss": 1.2888,
+ "theoretical_loss": 3.620776912678612,
+ "tokens_seen": 1182531584
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185636910732197,
+ "loss": 1.3743,
+ "theoretical_loss": 3.6207585495387304,
+ "tokens_seen": 1182597120
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185476429287863,
+ "loss": 1.4577,
+ "theoretical_loss": 3.620740187701369,
+ "tokens_seen": 1182662656
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185315947843531,
+ "loss": 1.2731,
+ "theoretical_loss": 3.6207218271663644,
+ "tokens_seen": 1182728192
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005185155466399197,
+ "loss": 1.2278,
+ "theoretical_loss": 3.6207034679335504,
+ "tokens_seen": 1182793728
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184994984954865,
+ "loss": 1.3427,
+ "theoretical_loss": 3.620685110002764,
+ "tokens_seen": 1182859264
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 398018,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4799141883850098,
+ "objective/train/theoretical_loss": 3.6206667533738406,
+ "objective/train/tokens_used": 1203384800,
+ "theoretical_loss": 3.6206667533738406,
+ "tokens_seen": 1182924800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184834503510531,
+ "loss": 1.3542,
+ "theoretical_loss": 3.6206667533738406,
+ "tokens_seen": 1182924800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184674022066199,
+ "loss": 1.2616,
+ "theoretical_loss": 3.6206483980466144,
+ "tokens_seen": 1182990336
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184513540621865,
+ "loss": 1.4289,
+ "theoretical_loss": 3.620630044020923,
+ "tokens_seen": 1183055872
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184353059177533,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6206116912966007,
+ "tokens_seen": 1183121408
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184192577733199,
+ "loss": 1.3258,
+ "theoretical_loss": 3.620593339873483,
+ "tokens_seen": 1183186944
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005184032096288867,
+ "loss": 1.3038,
+ "theoretical_loss": 3.620574989751406,
+ "tokens_seen": 1183252480
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183871614844533,
+ "loss": 1.4068,
+ "theoretical_loss": 3.620556640930206,
+ "tokens_seen": 1183318016
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183711133400201,
+ "loss": 1.4106,
+ "theoretical_loss": 3.6205382934097177,
+ "tokens_seen": 1183383552
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183550651955867,
+ "loss": 1.1708,
+ "theoretical_loss": 3.6205199471897775,
+ "tokens_seen": 1183449088
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183390170511535,
+ "loss": 1.3734,
+ "theoretical_loss": 3.6205016022702208,
+ "tokens_seen": 1183514624
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183229689067201,
+ "loss": 1.2589,
+ "theoretical_loss": 3.620483258650884,
+ "tokens_seen": 1183580160
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005183069207622869,
+ "loss": 1.3248,
+ "theoretical_loss": 3.6204649163316023,
+ "tokens_seen": 1183645696
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005182908726178535,
+ "loss": 1.2841,
+ "theoretical_loss": 3.6204465753122124,
+ "tokens_seen": 1183711232
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005182748244734203,
+ "loss": 1.449,
+ "theoretical_loss": 3.620428235592549,
+ "tokens_seen": 1183776768
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000518258776328987,
+ "loss": 1.3521,
+ "theoretical_loss": 3.6204098971724488,
+ "tokens_seen": 1183842304
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005182427281845537,
+ "loss": 1.4387,
+ "theoretical_loss": 3.6203915600517482,
+ "tokens_seen": 1183907840
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005182266800401203,
+ "loss": 1.3705,
+ "theoretical_loss": 3.6203732242302826,
+ "tokens_seen": 1183973376
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005182106318956871,
+ "loss": 1.3759,
+ "theoretical_loss": 3.6203548897078885,
+ "tokens_seen": 1184038912
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181945837512537,
+ "loss": 1.3856,
+ "theoretical_loss": 3.6203365564844012,
+ "tokens_seen": 1184104448
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181785356068205,
+ "loss": 1.3476,
+ "theoretical_loss": 3.6203182245596572,
+ "tokens_seen": 1184169984
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181624874623871,
+ "loss": 1.2597,
+ "theoretical_loss": 3.6202998939334927,
+ "tokens_seen": 1184235520
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181464393179539,
+ "loss": 1.2945,
+ "theoretical_loss": 3.6202815646057447,
+ "tokens_seen": 1184301056
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181303911735206,
+ "loss": 1.2322,
+ "theoretical_loss": 3.6202632365762475,
+ "tokens_seen": 1184366592
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005181143430290873,
+ "loss": 1.3941,
+ "theoretical_loss": 3.6202449098448386,
+ "tokens_seen": 1184432128
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000518098294884654,
+ "loss": 1.3156,
+ "theoretical_loss": 3.6202265844113537,
+ "tokens_seen": 1184497664
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180822467402207,
+ "loss": 1.2034,
+ "theoretical_loss": 3.6202082602756303,
+ "tokens_seen": 1184563200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180661985957874,
+ "loss": 1.3516,
+ "theoretical_loss": 3.6201899374375026,
+ "tokens_seen": 1184628736
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180501504513541,
+ "loss": 1.3206,
+ "theoretical_loss": 3.620171615896809,
+ "tokens_seen": 1184694272
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180341023069208,
+ "loss": 1.2562,
+ "theoretical_loss": 3.6201532956533846,
+ "tokens_seen": 1184759808
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180180541624875,
+ "loss": 1.3374,
+ "theoretical_loss": 3.6201349767070665,
+ "tokens_seen": 1184825344
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005180020060180542,
+ "loss": 1.4135,
+ "theoretical_loss": 3.62011665905769,
+ "tokens_seen": 1184890880
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005179859578736209,
+ "loss": 1.2787,
+ "theoretical_loss": 3.6200983427050932,
+ "tokens_seen": 1184956416
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005179699097291876,
+ "loss": 1.267,
+ "theoretical_loss": 3.620080027649111,
+ "tokens_seen": 1185021952
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005179538615847543,
+ "loss": 1.3703,
+ "theoretical_loss": 3.620061713889581,
+ "tokens_seen": 1185087488
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517937813440321,
+ "loss": 1.2718,
+ "theoretical_loss": 3.620043401426339,
+ "tokens_seen": 1185153024
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005179217652958877,
+ "loss": 1.1488,
+ "theoretical_loss": 3.6200250902592224,
+ "tokens_seen": 1185218560
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005179057171514544,
+ "loss": 1.3381,
+ "theoretical_loss": 3.620006780388067,
+ "tokens_seen": 1185284096
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178896690070211,
+ "loss": 1.3544,
+ "theoretical_loss": 3.61998847181271,
+ "tokens_seen": 1185349632
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178736208625878,
+ "loss": 1.3417,
+ "theoretical_loss": 3.619970164532988,
+ "tokens_seen": 1185415168
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178575727181545,
+ "loss": 1.2851,
+ "theoretical_loss": 3.6199518585487374,
+ "tokens_seen": 1185480704
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178415245737212,
+ "loss": 1.3352,
+ "theoretical_loss": 3.619933553859795,
+ "tokens_seen": 1185546240
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178254764292879,
+ "loss": 1.3918,
+ "theoretical_loss": 3.6199152504659975,
+ "tokens_seen": 1185611776
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005178094282848546,
+ "loss": 1.4179,
+ "theoretical_loss": 3.619896948367182,
+ "tokens_seen": 1185677312
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005177933801404213,
+ "loss": 1.3772,
+ "theoretical_loss": 3.619878647563185,
+ "tokens_seen": 1185742848
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517777331995988,
+ "loss": 1.2345,
+ "theoretical_loss": 3.6198603480538427,
+ "tokens_seen": 1185808384
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005177612838515547,
+ "loss": 1.2903,
+ "theoretical_loss": 3.6198420498389936,
+ "tokens_seen": 1185873920
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005177452357071214,
+ "loss": 1.2857,
+ "theoretical_loss": 3.6198237529184736,
+ "tokens_seen": 1185939456
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005177291875626881,
+ "loss": 1.3838,
+ "theoretical_loss": 3.619805457292119,
+ "tokens_seen": 1186004992
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005177131394182549,
+ "loss": 1.2333,
+ "theoretical_loss": 3.6197871629597684,
+ "tokens_seen": 1186070528
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176970912738215,
+ "loss": 1.3619,
+ "theoretical_loss": 3.6197688699212573,
+ "tokens_seen": 1186136064
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 398884,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.390918493270874,
+ "objective/train/theoretical_loss": 3.619750578176424,
+ "objective/train/tokens_used": 1206661600,
+ "theoretical_loss": 3.619750578176424,
+ "tokens_seen": 1186201600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176810431293883,
+ "loss": 1.4301,
+ "theoretical_loss": 3.619750578176424,
+ "tokens_seen": 1186201600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176649949849549,
+ "loss": 1.5078,
+ "theoretical_loss": 3.619732287725104,
+ "tokens_seen": 1186267136
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176489468405217,
+ "loss": 1.3039,
+ "theoretical_loss": 3.619713998567135,
+ "tokens_seen": 1186332672
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176328986960883,
+ "loss": 1.3442,
+ "theoretical_loss": 3.619695710702355,
+ "tokens_seen": 1186398208
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176168505516551,
+ "loss": 1.2442,
+ "theoretical_loss": 3.6196774241306002,
+ "tokens_seen": 1186463744
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005176008024072217,
+ "loss": 1.3391,
+ "theoretical_loss": 3.619659138851708,
+ "tokens_seen": 1186529280
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175847542627885,
+ "loss": 1.4667,
+ "theoretical_loss": 3.6196408548655157,
+ "tokens_seen": 1186594816
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175687061183551,
+ "loss": 1.3806,
+ "theoretical_loss": 3.61962257217186,
+ "tokens_seen": 1186660352
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175526579739219,
+ "loss": 1.3643,
+ "theoretical_loss": 3.619604290770579,
+ "tokens_seen": 1186725888
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175366098294885,
+ "loss": 1.3412,
+ "theoretical_loss": 3.619586010661509,
+ "tokens_seen": 1186791424
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175205616850552,
+ "loss": 1.3849,
+ "theoretical_loss": 3.6195677318444885,
+ "tokens_seen": 1186856960
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005175045135406219,
+ "loss": 1.456,
+ "theoretical_loss": 3.619549454319354,
+ "tokens_seen": 1186922496
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005174884653961886,
+ "loss": 1.2811,
+ "theoretical_loss": 3.6195311780859427,
+ "tokens_seen": 1186988032
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005174724172517553,
+ "loss": 1.258,
+ "theoretical_loss": 3.619512903144093,
+ "tokens_seen": 1187053568
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517456369107322,
+ "loss": 1.4333,
+ "theoretical_loss": 3.619494629493641,
+ "tokens_seen": 1187119104
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005174403209628887,
+ "loss": 1.3053,
+ "theoretical_loss": 3.6194763571344257,
+ "tokens_seen": 1187184640
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005174242728184554,
+ "loss": 1.2799,
+ "theoretical_loss": 3.619458086066283,
+ "tokens_seen": 1187250176
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517408224674022,
+ "loss": 1.4062,
+ "theoretical_loss": 3.6194398162890513,
+ "tokens_seen": 1187315712
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173921765295888,
+ "loss": 1.4374,
+ "theoretical_loss": 3.6194215478025678,
+ "tokens_seen": 1187381248
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173761283851554,
+ "loss": 1.3834,
+ "theoretical_loss": 3.6194032806066705,
+ "tokens_seen": 1187446784
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173600802407222,
+ "loss": 1.3529,
+ "theoretical_loss": 3.619385014701197,
+ "tokens_seen": 1187512320
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173440320962888,
+ "loss": 1.3175,
+ "theoretical_loss": 3.6193667500859847,
+ "tokens_seen": 1187577856
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173279839518556,
+ "loss": 1.428,
+ "theoretical_loss": 3.619348486760871,
+ "tokens_seen": 1187643392
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005173119358074222,
+ "loss": 1.279,
+ "theoretical_loss": 3.619330224725694,
+ "tokens_seen": 1187708928
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517295887662989,
+ "loss": 1.3363,
+ "theoretical_loss": 3.6193119639802918,
+ "tokens_seen": 1187774464
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005172798395185556,
+ "loss": 1.3446,
+ "theoretical_loss": 3.619293704524501,
+ "tokens_seen": 1187840000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005172637913741224,
+ "loss": 1.3324,
+ "theoretical_loss": 3.6192754463581602,
+ "tokens_seen": 1187905536
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517247743229689,
+ "loss": 1.4595,
+ "theoretical_loss": 3.619257189481107,
+ "tokens_seen": 1187971072
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005172316950852558,
+ "loss": 1.3313,
+ "theoretical_loss": 3.619238933893179,
+ "tokens_seen": 1188036608
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005172156469408224,
+ "loss": 1.3008,
+ "theoretical_loss": 3.619220679594215,
+ "tokens_seen": 1188102144
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171995987963892,
+ "loss": 1.3326,
+ "theoretical_loss": 3.619202426584052,
+ "tokens_seen": 1188167680
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171835506519558,
+ "loss": 1.4644,
+ "theoretical_loss": 3.6191841748625277,
+ "tokens_seen": 1188233216
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171675025075226,
+ "loss": 1.2918,
+ "theoretical_loss": 3.619165924429481,
+ "tokens_seen": 1188298752
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171514543630892,
+ "loss": 1.3477,
+ "theoretical_loss": 3.61914767528475,
+ "tokens_seen": 1188364288
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517135406218656,
+ "loss": 1.4141,
+ "theoretical_loss": 3.619129427428171,
+ "tokens_seen": 1188429824
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171193580742226,
+ "loss": 1.4162,
+ "theoretical_loss": 3.6191111808595835,
+ "tokens_seen": 1188495360
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005171033099297894,
+ "loss": 1.2769,
+ "theoretical_loss": 3.6190929355788253,
+ "tokens_seen": 1188560896
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000517087261785356,
+ "loss": 1.2547,
+ "theoretical_loss": 3.619074691585735,
+ "tokens_seen": 1188626432
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005170712136409228,
+ "loss": 1.3061,
+ "theoretical_loss": 3.6190564488801495,
+ "tokens_seen": 1188691968
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005170551654964894,
+ "loss": 1.3649,
+ "theoretical_loss": 3.619038207461908,
+ "tokens_seen": 1188757504
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005170391173520562,
+ "loss": 1.4416,
+ "theoretical_loss": 3.6190199673308485,
+ "tokens_seen": 1188823040
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005170230692076228,
+ "loss": 1.4086,
+ "theoretical_loss": 3.6190017284868086,
+ "tokens_seen": 1188888576
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005170070210631896,
+ "loss": 1.5011,
+ "theoretical_loss": 3.6189834909296277,
+ "tokens_seen": 1188954112
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005169909729187562,
+ "loss": 1.3175,
+ "theoretical_loss": 3.618965254659143,
+ "tokens_seen": 1189019648
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516974924774323,
+ "loss": 1.3763,
+ "theoretical_loss": 3.618947019675193,
+ "tokens_seen": 1189085184
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005169588766298896,
+ "loss": 1.3976,
+ "theoretical_loss": 3.6189287859776167,
+ "tokens_seen": 1189150720
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005169428284854564,
+ "loss": 1.3302,
+ "theoretical_loss": 3.6189105535662516,
+ "tokens_seen": 1189216256
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516926780341023,
+ "loss": 1.377,
+ "theoretical_loss": 3.618892322440937,
+ "tokens_seen": 1189281792
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005169107321965898,
+ "loss": 1.3334,
+ "theoretical_loss": 3.61887409260151,
+ "tokens_seen": 1189347328
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005168946840521564,
+ "loss": 1.2878,
+ "theoretical_loss": 3.6188558640478106,
+ "tokens_seen": 1189412864
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 400204,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2898616790771484,
+ "objective/train/theoretical_loss": 3.618837636779676,
+ "objective/train/tokens_used": 1209938400,
+ "theoretical_loss": 3.618837636779676,
+ "tokens_seen": 1189478400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005168786359077232,
+ "loss": 1.3022,
+ "theoretical_loss": 3.618837636779676,
+ "tokens_seen": 1189478400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005168625877632898,
+ "loss": 1.3268,
+ "theoretical_loss": 3.618819410796946,
+ "tokens_seen": 1189543936
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005168465396188566,
+ "loss": 1.4245,
+ "theoretical_loss": 3.618801186099458,
+ "tokens_seen": 1189609472
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005168304914744232,
+ "loss": 1.2795,
+ "theoretical_loss": 3.618782962687051,
+ "tokens_seen": 1189675008
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00051681444332999,
+ "loss": 1.2759,
+ "theoretical_loss": 3.6187647405595635,
+ "tokens_seen": 1189740544
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167983951855566,
+ "loss": 1.3017,
+ "theoretical_loss": 3.6187465197168347,
+ "tokens_seen": 1189806080
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167823470411234,
+ "loss": 1.2985,
+ "theoretical_loss": 3.6187283001587023,
+ "tokens_seen": 1189871616
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00051676629889669,
+ "loss": 1.264,
+ "theoretical_loss": 3.6187100818850055,
+ "tokens_seen": 1189937152
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167502507522568,
+ "loss": 1.327,
+ "theoretical_loss": 3.6186918648955837,
+ "tokens_seen": 1190002688
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167342026078235,
+ "loss": 1.392,
+ "theoretical_loss": 3.6186736491902742,
+ "tokens_seen": 1190068224
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167181544633902,
+ "loss": 1.267,
+ "theoretical_loss": 3.6186554347689164,
+ "tokens_seen": 1190133760
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005167021063189569,
+ "loss": 1.3462,
+ "theoretical_loss": 3.6186372216313503,
+ "tokens_seen": 1190199296
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005166860581745236,
+ "loss": 1.2363,
+ "theoretical_loss": 3.6186190097774125,
+ "tokens_seen": 1190264832
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005166700100300903,
+ "loss": 1.3146,
+ "theoretical_loss": 3.618600799206943,
+ "tokens_seen": 1190330368
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516653961885657,
+ "loss": 1.4205,
+ "theoretical_loss": 3.6185825899197814,
+ "tokens_seen": 1190395904
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005166379137412237,
+ "loss": 1.3104,
+ "theoretical_loss": 3.618564381915766,
+ "tokens_seen": 1190461440
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005166218655967904,
+ "loss": 1.3198,
+ "theoretical_loss": 3.6185461751947354,
+ "tokens_seen": 1190526976
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005166058174523571,
+ "loss": 1.3626,
+ "theoretical_loss": 3.6185279697565287,
+ "tokens_seen": 1190592512
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165897693079238,
+ "loss": 1.2887,
+ "theoretical_loss": 3.6185097656009853,
+ "tokens_seen": 1190658048
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165737211634905,
+ "loss": 1.3333,
+ "theoretical_loss": 3.618491562727944,
+ "tokens_seen": 1190723584
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165576730190572,
+ "loss": 1.2951,
+ "theoretical_loss": 3.6184733611372435,
+ "tokens_seen": 1190789120
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165416248746239,
+ "loss": 1.3484,
+ "theoretical_loss": 3.6184551608287236,
+ "tokens_seen": 1190854656
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165255767301906,
+ "loss": 1.3487,
+ "theoretical_loss": 3.6184369618022236,
+ "tokens_seen": 1190920192
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005165095285857574,
+ "loss": 1.3191,
+ "theoretical_loss": 3.618418764057581,
+ "tokens_seen": 1190985728
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516493480441324,
+ "loss": 1.2842,
+ "theoretical_loss": 3.618400567594637,
+ "tokens_seen": 1191051264
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005164774322968908,
+ "loss": 1.3415,
+ "theoretical_loss": 3.6183823724132296,
+ "tokens_seen": 1191116800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005164613841524574,
+ "loss": 1.4422,
+ "theoretical_loss": 3.6183641785131986,
+ "tokens_seen": 1191182336
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005164453360080242,
+ "loss": 1.2918,
+ "theoretical_loss": 3.6183459858943827,
+ "tokens_seen": 1191247872
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005164292878635908,
+ "loss": 1.483,
+ "theoretical_loss": 3.618327794556622,
+ "tokens_seen": 1191313408
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005164132397191576,
+ "loss": 1.2428,
+ "theoretical_loss": 3.618309604499755,
+ "tokens_seen": 1191378944
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005163971915747242,
+ "loss": 1.3704,
+ "theoretical_loss": 3.618291415723622,
+ "tokens_seen": 1191444480
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516381143430291,
+ "loss": 1.4868,
+ "theoretical_loss": 3.6182732282280607,
+ "tokens_seen": 1191510016
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005163650952858576,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6182550420129127,
+ "tokens_seen": 1191575552
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005163490471414244,
+ "loss": 1.4112,
+ "theoretical_loss": 3.618236857078016,
+ "tokens_seen": 1191641088
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516332998996991,
+ "loss": 1.344,
+ "theoretical_loss": 3.6182186734232102,
+ "tokens_seen": 1191706624
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005163169508525578,
+ "loss": 1.4586,
+ "theoretical_loss": 3.6182004910483356,
+ "tokens_seen": 1191772160
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005163009027081244,
+ "loss": 1.3765,
+ "theoretical_loss": 3.618182309953231,
+ "tokens_seen": 1191837696
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005162848545636912,
+ "loss": 1.2811,
+ "theoretical_loss": 3.618164130137736,
+ "tokens_seen": 1191903232
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005162688064192578,
+ "loss": 1.3366,
+ "theoretical_loss": 3.61814595160169,
+ "tokens_seen": 1191968768
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005162527582748246,
+ "loss": 1.2032,
+ "theoretical_loss": 3.618127774344934,
+ "tokens_seen": 1192034304
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005162367101303912,
+ "loss": 1.3796,
+ "theoretical_loss": 3.6181095983673055,
+ "tokens_seen": 1192099840
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516220661985958,
+ "loss": 1.2893,
+ "theoretical_loss": 3.6180914236686457,
+ "tokens_seen": 1192165376
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005162046138415246,
+ "loss": 1.2538,
+ "theoretical_loss": 3.618073250248794,
+ "tokens_seen": 1192230912
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005161885656970914,
+ "loss": 1.4381,
+ "theoretical_loss": 3.6180550781075898,
+ "tokens_seen": 1192296448
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516172517552658,
+ "loss": 1.1963,
+ "theoretical_loss": 3.618036907244873,
+ "tokens_seen": 1192361984
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005161564694082248,
+ "loss": 1.4107,
+ "theoretical_loss": 3.618018737660484,
+ "tokens_seen": 1192427520
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005161404212637914,
+ "loss": 1.294,
+ "theoretical_loss": 3.6180005693542614,
+ "tokens_seen": 1192493056
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005161243731193581,
+ "loss": 1.3024,
+ "theoretical_loss": 3.617982402326046,
+ "tokens_seen": 1192558592
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005161083249749248,
+ "loss": 1.341,
+ "theoretical_loss": 3.617964236575678,
+ "tokens_seen": 1192624128
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005160922768304915,
+ "loss": 1.4014,
+ "theoretical_loss": 3.6179460721029955,
+ "tokens_seen": 1192689664
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 401134,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1924456357955933,
+ "objective/train/theoretical_loss": 3.6179279089078404,
+ "objective/train/tokens_used": 1213215200,
+ "theoretical_loss": 3.6179279089078404,
+ "tokens_seen": 1192755200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005160762286860582,
+ "loss": 1.2172,
+ "theoretical_loss": 3.6179279089078404,
+ "tokens_seen": 1192755200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005160601805416249,
+ "loss": 1.2603,
+ "theoretical_loss": 3.617909746990052,
+ "tokens_seen": 1192820736
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005160441323971916,
+ "loss": 1.3865,
+ "theoretical_loss": 3.6178915863494696,
+ "tokens_seen": 1192886272
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005160280842527583,
+ "loss": 1.2988,
+ "theoretical_loss": 3.6178734269859345,
+ "tokens_seen": 1192951808
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000516012036108325,
+ "loss": 1.4073,
+ "theoretical_loss": 3.6178552688992855,
+ "tokens_seen": 1193017344
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159959879638917,
+ "loss": 1.3006,
+ "theoretical_loss": 3.6178371120893638,
+ "tokens_seen": 1193082880
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159799398194583,
+ "loss": 1.4048,
+ "theoretical_loss": 3.6178189565560084,
+ "tokens_seen": 1193148416
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159638916750251,
+ "loss": 1.2509,
+ "theoretical_loss": 3.617800802299061,
+ "tokens_seen": 1193213952
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159478435305917,
+ "loss": 1.3963,
+ "theoretical_loss": 3.61778264931836,
+ "tokens_seen": 1193279488
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159317953861585,
+ "loss": 1.3635,
+ "theoretical_loss": 3.617764497613746,
+ "tokens_seen": 1193345024
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005159157472417251,
+ "loss": 1.4002,
+ "theoretical_loss": 3.6177463471850597,
+ "tokens_seen": 1193410560
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158996990972919,
+ "loss": 1.2974,
+ "theoretical_loss": 3.617728198032142,
+ "tokens_seen": 1193476096
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158836509528585,
+ "loss": 1.3488,
+ "theoretical_loss": 3.617710050154832,
+ "tokens_seen": 1193541632
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158676028084253,
+ "loss": 1.3583,
+ "theoretical_loss": 3.61769190355297,
+ "tokens_seen": 1193607168
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158515546639919,
+ "loss": 1.2704,
+ "theoretical_loss": 3.617673758226397,
+ "tokens_seen": 1193672704
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158355065195587,
+ "loss": 1.3382,
+ "theoretical_loss": 3.6176556141749536,
+ "tokens_seen": 1193738240
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158194583751253,
+ "loss": 1.332,
+ "theoretical_loss": 3.6176374713984796,
+ "tokens_seen": 1193803776
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005158034102306921,
+ "loss": 1.3715,
+ "theoretical_loss": 3.617619329896815,
+ "tokens_seen": 1193869312
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157873620862587,
+ "loss": 1.3082,
+ "theoretical_loss": 3.6176011896698013,
+ "tokens_seen": 1193934848
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157713139418255,
+ "loss": 1.3234,
+ "theoretical_loss": 3.617583050717278,
+ "tokens_seen": 1194000384
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157552657973921,
+ "loss": 1.415,
+ "theoretical_loss": 3.617564913039087,
+ "tokens_seen": 1194065920
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157392176529589,
+ "loss": 1.372,
+ "theoretical_loss": 3.617546776635067,
+ "tokens_seen": 1194131456
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157231695085255,
+ "loss": 1.2965,
+ "theoretical_loss": 3.61752864150506,
+ "tokens_seen": 1194196992
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005157071213640923,
+ "loss": 1.278,
+ "theoretical_loss": 3.617510507648906,
+ "tokens_seen": 1194262528
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156910732196589,
+ "loss": 1.3113,
+ "theoretical_loss": 3.617492375066446,
+ "tokens_seen": 1194328064
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156750250752257,
+ "loss": 1.2497,
+ "theoretical_loss": 3.6174742437575196,
+ "tokens_seen": 1194393600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156589769307923,
+ "loss": 1.3184,
+ "theoretical_loss": 3.6174561137219685,
+ "tokens_seen": 1194459136
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156429287863591,
+ "loss": 1.3982,
+ "theoretical_loss": 3.6174379849596336,
+ "tokens_seen": 1194524672
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156268806419257,
+ "loss": 1.2823,
+ "theoretical_loss": 3.617419857470355,
+ "tokens_seen": 1194590208
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005156108324974925,
+ "loss": 1.2944,
+ "theoretical_loss": 3.6174017312539735,
+ "tokens_seen": 1194655744
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155947843530591,
+ "loss": 1.4242,
+ "theoretical_loss": 3.61738360631033,
+ "tokens_seen": 1194721280
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155787362086259,
+ "loss": 1.3206,
+ "theoretical_loss": 3.617365482639266,
+ "tokens_seen": 1194786816
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155626880641925,
+ "loss": 1.346,
+ "theoretical_loss": 3.617347360240621,
+ "tokens_seen": 1194852352
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155466399197593,
+ "loss": 1.3717,
+ "theoretical_loss": 3.6173292391142375,
+ "tokens_seen": 1194917888
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155305917753259,
+ "loss": 1.3413,
+ "theoretical_loss": 3.6173111192599547,
+ "tokens_seen": 1194983424
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005155145436308927,
+ "loss": 1.3822,
+ "theoretical_loss": 3.6172930006776154,
+ "tokens_seen": 1195048960
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154984954864593,
+ "loss": 1.3607,
+ "theoretical_loss": 3.6172748833670587,
+ "tokens_seen": 1195114496
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154824473420261,
+ "loss": 1.3311,
+ "theoretical_loss": 3.6172567673281266,
+ "tokens_seen": 1195180032
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154663991975927,
+ "loss": 1.3801,
+ "theoretical_loss": 3.61723865256066,
+ "tokens_seen": 1195245568
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154503510531595,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6172205390645003,
+ "tokens_seen": 1195311104
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154343029087261,
+ "loss": 1.3877,
+ "theoretical_loss": 3.617202426839488,
+ "tokens_seen": 1195376640
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154182547642929,
+ "loss": 1.3405,
+ "theoretical_loss": 3.6171843158854644,
+ "tokens_seen": 1195442176
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005154022066198596,
+ "loss": 1.2665,
+ "theoretical_loss": 3.6171662062022705,
+ "tokens_seen": 1195507712
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005153861584754263,
+ "loss": 1.3175,
+ "theoretical_loss": 3.617148097789748,
+ "tokens_seen": 1195573248
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000515370110330993,
+ "loss": 1.2281,
+ "theoretical_loss": 3.6171299906477374,
+ "tokens_seen": 1195638784
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005153540621865597,
+ "loss": 1.341,
+ "theoretical_loss": 3.617111884776081,
+ "tokens_seen": 1195704320
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005153380140421264,
+ "loss": 1.4207,
+ "theoretical_loss": 3.6170937801746184,
+ "tokens_seen": 1195769856
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005153219658976931,
+ "loss": 1.3215,
+ "theoretical_loss": 3.617075676843192,
+ "tokens_seen": 1195835392
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005153059177532599,
+ "loss": 1.3226,
+ "theoretical_loss": 3.6170575747816436,
+ "tokens_seen": 1195900928
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152898696088265,
+ "loss": 1.4244,
+ "theoretical_loss": 3.6170394739898133,
+ "tokens_seen": 1195966464
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 402272,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3172639608383179,
+ "objective/train/theoretical_loss": 3.617021374467543,
+ "objective/train/tokens_used": 1216492000,
+ "theoretical_loss": 3.617021374467543,
+ "tokens_seen": 1196032000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152738214643933,
+ "loss": 1.3791,
+ "theoretical_loss": 3.617021374467543,
+ "tokens_seen": 1196032000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152577733199599,
+ "loss": 1.3855,
+ "theoretical_loss": 3.617003276214674,
+ "tokens_seen": 1196097536
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152417251755267,
+ "loss": 1.4151,
+ "theoretical_loss": 3.6169851792310483,
+ "tokens_seen": 1196163072
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152256770310933,
+ "loss": 1.3781,
+ "theoretical_loss": 3.616967083516507,
+ "tokens_seen": 1196228608
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005152096288866601,
+ "loss": 1.2994,
+ "theoretical_loss": 3.616948989070891,
+ "tokens_seen": 1196294144
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151935807422267,
+ "loss": 1.3409,
+ "theoretical_loss": 3.6169308958940425,
+ "tokens_seen": 1196359680
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151775325977935,
+ "loss": 1.363,
+ "theoretical_loss": 3.616912803985803,
+ "tokens_seen": 1196425216
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151614844533601,
+ "loss": 1.3576,
+ "theoretical_loss": 3.6168947133460136,
+ "tokens_seen": 1196490752
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151454363089269,
+ "loss": 1.3667,
+ "theoretical_loss": 3.6168766239745165,
+ "tokens_seen": 1196556288
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151293881644935,
+ "loss": 1.3921,
+ "theoretical_loss": 3.616858535871153,
+ "tokens_seen": 1196621824
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005151133400200603,
+ "loss": 1.4119,
+ "theoretical_loss": 3.616840449035765,
+ "tokens_seen": 1196687360
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150972918756269,
+ "loss": 1.3082,
+ "theoretical_loss": 3.6168223634681933,
+ "tokens_seen": 1196752896
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150812437311937,
+ "loss": 1.238,
+ "theoretical_loss": 3.6168042791682806,
+ "tokens_seen": 1196818432
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150651955867603,
+ "loss": 1.4655,
+ "theoretical_loss": 3.6167861961358687,
+ "tokens_seen": 1196883968
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150491474423271,
+ "loss": 1.2184,
+ "theoretical_loss": 3.6167681143707986,
+ "tokens_seen": 1196949504
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150330992978937,
+ "loss": 1.3114,
+ "theoretical_loss": 3.616750033872912,
+ "tokens_seen": 1197015040
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150170511534605,
+ "loss": 1.3726,
+ "theoretical_loss": 3.616731954642052,
+ "tokens_seen": 1197080576
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005150010030090271,
+ "loss": 1.2718,
+ "theoretical_loss": 3.6167138766780598,
+ "tokens_seen": 1197146112
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149849548645939,
+ "loss": 1.4189,
+ "theoretical_loss": 3.6166957999807767,
+ "tokens_seen": 1197211648
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149689067201605,
+ "loss": 1.3034,
+ "theoretical_loss": 3.616677724550045,
+ "tokens_seen": 1197277184
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149528585757273,
+ "loss": 1.3077,
+ "theoretical_loss": 3.616659650385707,
+ "tokens_seen": 1197342720
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149368104312939,
+ "loss": 1.4082,
+ "theoretical_loss": 3.616641577487604,
+ "tokens_seen": 1197408256
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149207622868607,
+ "loss": 1.3105,
+ "theoretical_loss": 3.6166235058555785,
+ "tokens_seen": 1197473792
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005149047141424273,
+ "loss": 1.2711,
+ "theoretical_loss": 3.6166054354894728,
+ "tokens_seen": 1197539328
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148886659979941,
+ "loss": 1.3306,
+ "theoretical_loss": 3.616587366389128,
+ "tokens_seen": 1197604864
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148726178535607,
+ "loss": 1.3604,
+ "theoretical_loss": 3.6165692985543867,
+ "tokens_seen": 1197670400
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148565697091275,
+ "loss": 1.4035,
+ "theoretical_loss": 3.616551231985091,
+ "tokens_seen": 1197735936
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148405215646941,
+ "loss": 1.3933,
+ "theoretical_loss": 3.616533166681083,
+ "tokens_seen": 1197801472
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148244734202609,
+ "loss": 1.4038,
+ "theoretical_loss": 3.6165151026422055,
+ "tokens_seen": 1197867008
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005148084252758275,
+ "loss": 1.4169,
+ "theoretical_loss": 3.6164970398682996,
+ "tokens_seen": 1197932544
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005147923771313943,
+ "loss": 1.225,
+ "theoretical_loss": 3.616478978359208,
+ "tokens_seen": 1197998080
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005147763289869609,
+ "loss": 1.2246,
+ "theoretical_loss": 3.6164609181147735,
+ "tokens_seen": 1198063616
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005147602808425277,
+ "loss": 1.3435,
+ "theoretical_loss": 3.616442859134837,
+ "tokens_seen": 1198129152
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005147442326980943,
+ "loss": 1.3922,
+ "theoretical_loss": 3.6164248014192424,
+ "tokens_seen": 1198194688
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514728184553661,
+ "loss": 1.3292,
+ "theoretical_loss": 3.616406744967831,
+ "tokens_seen": 1198260224
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005147121364092277,
+ "loss": 1.3444,
+ "theoretical_loss": 3.616388689780446,
+ "tokens_seen": 1198325760
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146960882647944,
+ "loss": 1.3101,
+ "theoretical_loss": 3.6163706358569288,
+ "tokens_seen": 1198391296
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146800401203611,
+ "loss": 1.3164,
+ "theoretical_loss": 3.6163525831971217,
+ "tokens_seen": 1198456832
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146639919759278,
+ "loss": 1.3864,
+ "theoretical_loss": 3.6163345318008684,
+ "tokens_seen": 1198522368
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146479438314945,
+ "loss": 1.3088,
+ "theoretical_loss": 3.6163164816680107,
+ "tokens_seen": 1198587904
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146318956870612,
+ "loss": 1.4454,
+ "theoretical_loss": 3.616298432798391,
+ "tokens_seen": 1198653440
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005146158475426279,
+ "loss": 1.3278,
+ "theoretical_loss": 3.6162803851918515,
+ "tokens_seen": 1198718976
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005145997993981946,
+ "loss": 1.311,
+ "theoretical_loss": 3.6162623388482356,
+ "tokens_seen": 1198784512
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005145837512537612,
+ "loss": 1.3142,
+ "theoretical_loss": 3.6162442937673855,
+ "tokens_seen": 1198850048
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514567703109328,
+ "loss": 1.3755,
+ "theoretical_loss": 3.6162262499491438,
+ "tokens_seen": 1198915584
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005145516549648946,
+ "loss": 1.3546,
+ "theoretical_loss": 3.616208207393353,
+ "tokens_seen": 1198981120
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005145356068204614,
+ "loss": 1.2408,
+ "theoretical_loss": 3.616190166099856,
+ "tokens_seen": 1199046656
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514519558676028,
+ "loss": 1.3636,
+ "theoretical_loss": 3.616172126068496,
+ "tokens_seen": 1199112192
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005145035105315948,
+ "loss": 1.325,
+ "theoretical_loss": 3.6161540872991145,
+ "tokens_seen": 1199177728
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005144874623871614,
+ "loss": 1.3188,
+ "theoretical_loss": 3.616136049791555,
+ "tokens_seen": 1199243264
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 403082,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2459710836410522,
+ "objective/train/theoretical_loss": 3.61611801354566,
+ "objective/train/tokens_used": 1219768800,
+ "theoretical_loss": 3.61611801354566,
+ "tokens_seen": 1199308800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005144714142427282,
+ "loss": 1.3009,
+ "theoretical_loss": 3.61611801354566,
+ "tokens_seen": 1199308800
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005144553660982948,
+ "loss": 1.3199,
+ "theoretical_loss": 3.6160999785612726,
+ "tokens_seen": 1199374336
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005144393179538616,
+ "loss": 1.2832,
+ "theoretical_loss": 3.6160819448382355,
+ "tokens_seen": 1199439872
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005144232698094282,
+ "loss": 1.2378,
+ "theoretical_loss": 3.616063912376392,
+ "tokens_seen": 1199505408
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514407221664995,
+ "loss": 1.3436,
+ "theoretical_loss": 3.6160458811755842,
+ "tokens_seen": 1199570944
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005143911735205616,
+ "loss": 1.2775,
+ "theoretical_loss": 3.6160278512356556,
+ "tokens_seen": 1199636480
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005143751253761284,
+ "loss": 1.3245,
+ "theoretical_loss": 3.6160098225564488,
+ "tokens_seen": 1199702016
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514359077231695,
+ "loss": 1.3329,
+ "theoretical_loss": 3.6159917951378073,
+ "tokens_seen": 1199767552
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005143430290872618,
+ "loss": 1.3655,
+ "theoretical_loss": 3.6159737689795737,
+ "tokens_seen": 1199833088
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005143269809428284,
+ "loss": 1.3092,
+ "theoretical_loss": 3.6159557440815915,
+ "tokens_seen": 1199898624
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005143109327983952,
+ "loss": 1.4172,
+ "theoretical_loss": 3.6159377204437027,
+ "tokens_seen": 1199964160
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005142948846539618,
+ "loss": 1.1557,
+ "theoretical_loss": 3.6159196980657518,
+ "tokens_seen": 1200029696
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005142788365095286,
+ "loss": 1.3506,
+ "theoretical_loss": 3.6159016769475807,
+ "tokens_seen": 1200095232
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005142627883650952,
+ "loss": 1.3797,
+ "theoretical_loss": 3.615883657089033,
+ "tokens_seen": 1200160768
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514246740220662,
+ "loss": 1.4379,
+ "theoretical_loss": 3.615865638489953,
+ "tokens_seen": 1200226304
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005142306920762286,
+ "loss": 1.41,
+ "theoretical_loss": 3.6158476211501815,
+ "tokens_seen": 1200291840
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005142146439317954,
+ "loss": 1.4144,
+ "theoretical_loss": 3.615829605069564,
+ "tokens_seen": 1200357376
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514198595787362,
+ "loss": 1.4482,
+ "theoretical_loss": 3.6158115902479424,
+ "tokens_seen": 1200422912
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141825476429288,
+ "loss": 1.3713,
+ "theoretical_loss": 3.6157935766851605,
+ "tokens_seen": 1200488448
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141664994984954,
+ "loss": 1.3692,
+ "theoretical_loss": 3.615775564381062,
+ "tokens_seen": 1200553984
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141504513540622,
+ "loss": 1.2828,
+ "theoretical_loss": 3.6157575533354893,
+ "tokens_seen": 1200619520
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141344032096288,
+ "loss": 1.2701,
+ "theoretical_loss": 3.6157395435482864,
+ "tokens_seen": 1200685056
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141183550651956,
+ "loss": 1.4091,
+ "theoretical_loss": 3.615721535019297,
+ "tokens_seen": 1200750592
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005141023069207624,
+ "loss": 1.3496,
+ "theoretical_loss": 3.6157035277483636,
+ "tokens_seen": 1200816128
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000514086258776329,
+ "loss": 1.3302,
+ "theoretical_loss": 3.6156855217353305,
+ "tokens_seen": 1200881664
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005140702106318958,
+ "loss": 1.414,
+ "theoretical_loss": 3.615667516980041,
+ "tokens_seen": 1200947200
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005140541624874624,
+ "loss": 1.4534,
+ "theoretical_loss": 3.6156495134823383,
+ "tokens_seen": 1201012736
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005140381143430292,
+ "loss": 1.3739,
+ "theoretical_loss": 3.615631511242066,
+ "tokens_seen": 1201078272
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005140220661985958,
+ "loss": 1.3974,
+ "theoretical_loss": 3.6156135102590685,
+ "tokens_seen": 1201143808
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005140060180541626,
+ "loss": 1.3475,
+ "theoretical_loss": 3.615595510533188,
+ "tokens_seen": 1201209344
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005139899699097292,
+ "loss": 1.3214,
+ "theoretical_loss": 3.6155775120642692,
+ "tokens_seen": 1201274880
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513973921765296,
+ "loss": 1.2765,
+ "theoretical_loss": 3.6155595148521553,
+ "tokens_seen": 1201340416
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005139578736208626,
+ "loss": 1.249,
+ "theoretical_loss": 3.6155415188966904,
+ "tokens_seen": 1201405952
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005139418254764294,
+ "loss": 1.3862,
+ "theoretical_loss": 3.6155235241977177,
+ "tokens_seen": 1201471488
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513925777331996,
+ "loss": 1.4308,
+ "theoretical_loss": 3.615505530755081,
+ "tokens_seen": 1201537024
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005139097291875628,
+ "loss": 1.3439,
+ "theoretical_loss": 3.6154875385686243,
+ "tokens_seen": 1201602560
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005138936810431294,
+ "loss": 1.3738,
+ "theoretical_loss": 3.6154695476381913,
+ "tokens_seen": 1201668096
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005138776328986962,
+ "loss": 1.3634,
+ "theoretical_loss": 3.6154515579636257,
+ "tokens_seen": 1201733632
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005138615847542628,
+ "loss": 1.2396,
+ "theoretical_loss": 3.6154335695447717,
+ "tokens_seen": 1201799168
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005138455366098296,
+ "loss": 1.3243,
+ "theoretical_loss": 3.615415582381473,
+ "tokens_seen": 1201864704
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005138294884653962,
+ "loss": 1.3334,
+ "theoretical_loss": 3.6153975964735734,
+ "tokens_seen": 1201930240
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513813440320963,
+ "loss": 1.3817,
+ "theoretical_loss": 3.615379611820917,
+ "tokens_seen": 1201995776
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137973921765296,
+ "loss": 1.4029,
+ "theoretical_loss": 3.6153616284233476,
+ "tokens_seen": 1202061312
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137813440320964,
+ "loss": 1.2957,
+ "theoretical_loss": 3.615343646280709,
+ "tokens_seen": 1202126848
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513765295887663,
+ "loss": 1.3365,
+ "theoretical_loss": 3.615325665392846,
+ "tokens_seen": 1202192384
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137492477432298,
+ "loss": 1.3333,
+ "theoretical_loss": 3.6153076857596016,
+ "tokens_seen": 1202257920
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137331995987964,
+ "loss": 1.3629,
+ "theoretical_loss": 3.6152897073808212,
+ "tokens_seen": 1202323456
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137171514543632,
+ "loss": 1.2974,
+ "theoretical_loss": 3.6152717302563477,
+ "tokens_seen": 1202388992
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005137011033099298,
+ "loss": 1.3039,
+ "theoretical_loss": 3.6152537543860257,
+ "tokens_seen": 1202454528
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005136850551654966,
+ "loss": 1.4214,
+ "theoretical_loss": 3.615235779769699,
+ "tokens_seen": 1202520064
+ },
+ {
+ "epoch": 0.36,
+ "objective/train/docs_used": 404343,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.367863416671753,
+ "objective/train/theoretical_loss": 3.6152178064072125,
+ "objective/train/tokens_used": 1223045600,
+ "theoretical_loss": 3.6152178064072125,
+ "tokens_seen": 1202585600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005136690070210632,
+ "loss": 1.3764,
+ "theoretical_loss": 3.6152178064072125,
+ "tokens_seen": 1202585600
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00051365295887663,
+ "loss": 1.3665,
+ "theoretical_loss": 3.6151998342984095,
+ "tokens_seen": 1202651136
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005136369107321966,
+ "loss": 1.3252,
+ "theoretical_loss": 3.6151818634431354,
+ "tokens_seen": 1202716672
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005136208625877634,
+ "loss": 1.356,
+ "theoretical_loss": 3.615163893841233,
+ "tokens_seen": 1202782208
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.00051360481444333,
+ "loss": 1.5441,
+ "theoretical_loss": 3.6151459254925484,
+ "tokens_seen": 1202847744
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135887662988968,
+ "loss": 1.2836,
+ "theoretical_loss": 3.6151279583969247,
+ "tokens_seen": 1202913280
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135727181544634,
+ "loss": 1.1651,
+ "theoretical_loss": 3.6151099925542063,
+ "tokens_seen": 1202978816
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135566700100302,
+ "loss": 1.4052,
+ "theoretical_loss": 3.6150920279642373,
+ "tokens_seen": 1203044352
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135406218655968,
+ "loss": 1.3768,
+ "theoretical_loss": 3.615074064626863,
+ "tokens_seen": 1203109888
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135245737211636,
+ "loss": 1.3209,
+ "theoretical_loss": 3.615056102541928,
+ "tokens_seen": 1203175424
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005135085255767302,
+ "loss": 1.3026,
+ "theoretical_loss": 3.6150381417092756,
+ "tokens_seen": 1203240960
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513492477432297,
+ "loss": 1.3266,
+ "theoretical_loss": 3.6150201821287506,
+ "tokens_seen": 1203306496
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005134764292878636,
+ "loss": 1.3884,
+ "theoretical_loss": 3.6150022238001984,
+ "tokens_seen": 1203372032
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005134603811434304,
+ "loss": 1.2748,
+ "theoretical_loss": 3.6149842667234626,
+ "tokens_seen": 1203437568
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513444332998997,
+ "loss": 1.2208,
+ "theoretical_loss": 3.6149663108983887,
+ "tokens_seen": 1203503104
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005134282848545638,
+ "loss": 1.4069,
+ "theoretical_loss": 3.6149483563248204,
+ "tokens_seen": 1203568640
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005134122367101304,
+ "loss": 1.2371,
+ "theoretical_loss": 3.614930403002602,
+ "tokens_seen": 1203634176
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133961885656972,
+ "loss": 1.2641,
+ "theoretical_loss": 3.61491245093158,
+ "tokens_seen": 1203699712
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133801404212638,
+ "loss": 1.2983,
+ "theoretical_loss": 3.614894500111597,
+ "tokens_seen": 1203765248
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133640922768306,
+ "loss": 1.4068,
+ "theoretical_loss": 3.6148765505424993,
+ "tokens_seen": 1203830784
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133480441323972,
+ "loss": 1.2553,
+ "theoretical_loss": 3.614858602224131,
+ "tokens_seen": 1203896320
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133319959879639,
+ "loss": 1.3769,
+ "theoretical_loss": 3.6148406551563363,
+ "tokens_seen": 1203961856
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005133159478435306,
+ "loss": 1.3133,
+ "theoretical_loss": 3.614822709338961,
+ "tokens_seen": 1204027392
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132998996990973,
+ "loss": 1.4305,
+ "theoretical_loss": 3.6148047647718493,
+ "tokens_seen": 1204092928
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.000513283851554664,
+ "loss": 1.3142,
+ "theoretical_loss": 3.614786821454846,
+ "tokens_seen": 1204158464
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132678034102307,
+ "loss": 1.2143,
+ "theoretical_loss": 3.6147688793877966,
+ "tokens_seen": 1204224000
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132517552657974,
+ "loss": 1.2778,
+ "theoretical_loss": 3.614750938570545,
+ "tokens_seen": 1204289536
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132357071213641,
+ "loss": 1.286,
+ "theoretical_loss": 3.6147329990029373,
+ "tokens_seen": 1204355072
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132196589769308,
+ "loss": 1.3337,
+ "theoretical_loss": 3.6147150606848184,
+ "tokens_seen": 1204420608
+ },
+ {
+ "epoch": 0.36,
+ "learning_rate": 0.0005132036108324975,
+ "loss": 1.3435,
+ "theoretical_loss": 3.6146971236160317,
+ "tokens_seen": 1204486144
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131875626880641,
+ "loss": 1.4116,
+ "theoretical_loss": 3.6146791877964244,
+ "tokens_seen": 1204551680
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131715145436309,
+ "loss": 1.2683,
+ "theoretical_loss": 3.6146612532258398,
+ "tokens_seen": 1204617216
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131554663991975,
+ "loss": 1.3302,
+ "theoretical_loss": 3.6146433199041237,
+ "tokens_seen": 1204682752
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131394182547643,
+ "loss": 1.3733,
+ "theoretical_loss": 3.614625387831121,
+ "tokens_seen": 1204748288
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131233701103309,
+ "loss": 1.4139,
+ "theoretical_loss": 3.6146074570066773,
+ "tokens_seen": 1204813824
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005131073219658977,
+ "loss": 1.3556,
+ "theoretical_loss": 3.6145895274306374,
+ "tokens_seen": 1204879360
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130912738214643,
+ "loss": 1.3724,
+ "theoretical_loss": 3.6145715991028466,
+ "tokens_seen": 1204944896
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130752256770311,
+ "loss": 1.2736,
+ "theoretical_loss": 3.61455367202315,
+ "tokens_seen": 1205010432
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130591775325977,
+ "loss": 1.4024,
+ "theoretical_loss": 3.614535746191393,
+ "tokens_seen": 1205075968
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130431293881645,
+ "loss": 1.3528,
+ "theoretical_loss": 3.6145178216074205,
+ "tokens_seen": 1205141504
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130270812437311,
+ "loss": 1.3643,
+ "theoretical_loss": 3.6144998982710783,
+ "tokens_seen": 1205207040
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005130110330992979,
+ "loss": 1.3988,
+ "theoretical_loss": 3.614481976182211,
+ "tokens_seen": 1205272576
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129949849548645,
+ "loss": 1.3279,
+ "theoretical_loss": 3.6144640553406653,
+ "tokens_seen": 1205338112
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129789368104313,
+ "loss": 1.3316,
+ "theoretical_loss": 3.614446135746285,
+ "tokens_seen": 1205403648
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129628886659979,
+ "loss": 1.4332,
+ "theoretical_loss": 3.6144282173989164,
+ "tokens_seen": 1205469184
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129468405215647,
+ "loss": 1.3009,
+ "theoretical_loss": 3.6144103002984047,
+ "tokens_seen": 1205534720
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129307923771313,
+ "loss": 1.4368,
+ "theoretical_loss": 3.6143923844445958,
+ "tokens_seen": 1205600256
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005129147442326981,
+ "loss": 1.3376,
+ "theoretical_loss": 3.614374469837334,
+ "tokens_seen": 1205665792
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128986960882648,
+ "loss": 1.4212,
+ "theoretical_loss": 3.6143565564764666,
+ "tokens_seen": 1205731328
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128826479438315,
+ "loss": 1.3553,
+ "theoretical_loss": 3.6143386443618373,
+ "tokens_seen": 1205796864
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 405583,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.383839726448059,
+ "objective/train/theoretical_loss": 3.614320733493293,
+ "objective/train/tokens_used": 1226322400,
+ "theoretical_loss": 3.614320733493293,
+ "tokens_seen": 1205862400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128665997993983,
+ "loss": 1.3701,
+ "theoretical_loss": 3.614320733493293,
+ "tokens_seen": 1205862400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128505516549649,
+ "loss": 1.3188,
+ "theoretical_loss": 3.6143028238706787,
+ "tokens_seen": 1205927936
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128345035105317,
+ "loss": 1.329,
+ "theoretical_loss": 3.61428491549384,
+ "tokens_seen": 1205993472
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005128184553660983,
+ "loss": 1.3674,
+ "theoretical_loss": 3.6142670083626225,
+ "tokens_seen": 1206059008
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000512802407221665,
+ "loss": 1.3181,
+ "theoretical_loss": 3.6142491024768724,
+ "tokens_seen": 1206124544
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127863590772317,
+ "loss": 1.3222,
+ "theoretical_loss": 3.6142311978364354,
+ "tokens_seen": 1206190080
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127703109327985,
+ "loss": 1.3119,
+ "theoretical_loss": 3.6142132944411562,
+ "tokens_seen": 1206255616
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127542627883651,
+ "loss": 1.3041,
+ "theoretical_loss": 3.614195392290882,
+ "tokens_seen": 1206321152
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127382146439319,
+ "loss": 1.3166,
+ "theoretical_loss": 3.6141774913854574,
+ "tokens_seen": 1206386688
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127221664994985,
+ "loss": 1.434,
+ "theoretical_loss": 3.6141595917247296,
+ "tokens_seen": 1206452224
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005127061183550653,
+ "loss": 1.3401,
+ "theoretical_loss": 3.6141416933085426,
+ "tokens_seen": 1206517760
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126900702106319,
+ "loss": 1.2208,
+ "theoretical_loss": 3.6141237961367434,
+ "tokens_seen": 1206583296
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126740220661987,
+ "loss": 1.4068,
+ "theoretical_loss": 3.6141059002091778,
+ "tokens_seen": 1206648832
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126579739217653,
+ "loss": 1.2782,
+ "theoretical_loss": 3.614088005525692,
+ "tokens_seen": 1206714368
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126419257773321,
+ "loss": 1.3517,
+ "theoretical_loss": 3.6140701120861314,
+ "tokens_seen": 1206779904
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126258776328987,
+ "loss": 1.3423,
+ "theoretical_loss": 3.6140522198903424,
+ "tokens_seen": 1206845440
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005126098294884655,
+ "loss": 1.3735,
+ "theoretical_loss": 3.6140343289381702,
+ "tokens_seen": 1206910976
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125937813440321,
+ "loss": 1.4779,
+ "theoretical_loss": 3.614016439229462,
+ "tokens_seen": 1206976512
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125777331995989,
+ "loss": 1.4023,
+ "theoretical_loss": 3.6139985507640633,
+ "tokens_seen": 1207042048
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125616850551655,
+ "loss": 1.3906,
+ "theoretical_loss": 3.61398066354182,
+ "tokens_seen": 1207107584
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125456369107323,
+ "loss": 1.4067,
+ "theoretical_loss": 3.6139627775625787,
+ "tokens_seen": 1207173120
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125295887662989,
+ "loss": 1.4119,
+ "theoretical_loss": 3.6139448928261846,
+ "tokens_seen": 1207238656
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005125135406218657,
+ "loss": 1.4725,
+ "theoretical_loss": 3.6139270093324853,
+ "tokens_seen": 1207304192
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124974924774323,
+ "loss": 1.3313,
+ "theoretical_loss": 3.6139091270813264,
+ "tokens_seen": 1207369728
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124814443329991,
+ "loss": 1.3469,
+ "theoretical_loss": 3.613891246072553,
+ "tokens_seen": 1207435264
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124653961885657,
+ "loss": 1.3837,
+ "theoretical_loss": 3.613873366306013,
+ "tokens_seen": 1207500800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124493480441325,
+ "loss": 1.4254,
+ "theoretical_loss": 3.613855487781552,
+ "tokens_seen": 1207566336
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124332998996991,
+ "loss": 1.3958,
+ "theoretical_loss": 3.6138376104990164,
+ "tokens_seen": 1207631872
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124172517552659,
+ "loss": 1.4596,
+ "theoretical_loss": 3.6138197344582528,
+ "tokens_seen": 1207697408
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005124012036108325,
+ "loss": 1.4737,
+ "theoretical_loss": 3.613801859659107,
+ "tokens_seen": 1207762944
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123851554663993,
+ "loss": 1.3003,
+ "theoretical_loss": 3.613783986101425,
+ "tokens_seen": 1207828480
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123691073219659,
+ "loss": 1.2779,
+ "theoretical_loss": 3.6137661137850543,
+ "tokens_seen": 1207894016
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123530591775327,
+ "loss": 1.3878,
+ "theoretical_loss": 3.6137482427098417,
+ "tokens_seen": 1207959552
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123370110330993,
+ "loss": 1.392,
+ "theoretical_loss": 3.613730372875632,
+ "tokens_seen": 1208025088
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123209628886661,
+ "loss": 1.3946,
+ "theoretical_loss": 3.613712504282273,
+ "tokens_seen": 1208090624
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005123049147442327,
+ "loss": 1.3279,
+ "theoretical_loss": 3.6136946369296106,
+ "tokens_seen": 1208156160
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122888665997995,
+ "loss": 1.2981,
+ "theoretical_loss": 3.6136767708174915,
+ "tokens_seen": 1208221696
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122728184553661,
+ "loss": 1.3533,
+ "theoretical_loss": 3.613658905945763,
+ "tokens_seen": 1208287232
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122567703109329,
+ "loss": 1.3687,
+ "theoretical_loss": 3.61364104231427,
+ "tokens_seen": 1208352768
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122407221664995,
+ "loss": 1.3565,
+ "theoretical_loss": 3.613623179922861,
+ "tokens_seen": 1208418304
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122246740220663,
+ "loss": 1.3479,
+ "theoretical_loss": 3.613605318771382,
+ "tokens_seen": 1208483840
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005122086258776329,
+ "loss": 1.3981,
+ "theoretical_loss": 3.613587458859679,
+ "tokens_seen": 1208549376
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121925777331997,
+ "loss": 1.4024,
+ "theoretical_loss": 3.6135696001876,
+ "tokens_seen": 1208614912
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121765295887663,
+ "loss": 1.2919,
+ "theoretical_loss": 3.613551742754991,
+ "tokens_seen": 1208680448
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121604814443331,
+ "loss": 1.2458,
+ "theoretical_loss": 3.6135338865616986,
+ "tokens_seen": 1208745984
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121444332998997,
+ "loss": 1.3452,
+ "theoretical_loss": 3.6135160316075696,
+ "tokens_seen": 1208811520
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121283851554665,
+ "loss": 1.3305,
+ "theoretical_loss": 3.6134981778924518,
+ "tokens_seen": 1208877056
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005121123370110331,
+ "loss": 1.3028,
+ "theoretical_loss": 3.613480325416191,
+ "tokens_seen": 1208942592
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120962888665999,
+ "loss": 1.421,
+ "theoretical_loss": 3.613462474178634,
+ "tokens_seen": 1209008128
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120802407221665,
+ "loss": 1.3711,
+ "theoretical_loss": 3.6134446241796283,
+ "tokens_seen": 1209073664
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 406414,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3190600872039795,
+ "objective/train/theoretical_loss": 3.613426775419021,
+ "objective/train/tokens_used": 1229599200,
+ "theoretical_loss": 3.613426775419021,
+ "tokens_seen": 1209139200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120641925777333,
+ "loss": 1.3376,
+ "theoretical_loss": 3.613426775419021,
+ "tokens_seen": 1209139200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120481444332999,
+ "loss": 1.4591,
+ "theoretical_loss": 3.6134089278966584,
+ "tokens_seen": 1209204736
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120320962888667,
+ "loss": 1.4646,
+ "theoretical_loss": 3.613391081612388,
+ "tokens_seen": 1209270272
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120160481444333,
+ "loss": 1.3262,
+ "theoretical_loss": 3.6133732365660567,
+ "tokens_seen": 1209335808
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005120000000000001,
+ "loss": 1.4101,
+ "theoretical_loss": 3.6133553927575113,
+ "tokens_seen": 1209401344
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119839518555667,
+ "loss": 1.3722,
+ "theoretical_loss": 3.6133375501865994,
+ "tokens_seen": 1209466880
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119679037111335,
+ "loss": 1.4411,
+ "theoretical_loss": 3.613319708853168,
+ "tokens_seen": 1209532416
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119518555667001,
+ "loss": 1.4105,
+ "theoretical_loss": 3.6133018687570635,
+ "tokens_seen": 1209597952
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119358074222668,
+ "loss": 1.3581,
+ "theoretical_loss": 3.613284029898134,
+ "tokens_seen": 1209663488
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119197592778335,
+ "loss": 1.2052,
+ "theoretical_loss": 3.613266192276226,
+ "tokens_seen": 1209729024
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005119037111334002,
+ "loss": 1.3657,
+ "theoretical_loss": 3.613248355891187,
+ "tokens_seen": 1209794560
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005118876629889669,
+ "loss": 1.4047,
+ "theoretical_loss": 3.613230520742864,
+ "tokens_seen": 1209860096
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005118716148445336,
+ "loss": 1.3488,
+ "theoretical_loss": 3.6132126868311047,
+ "tokens_seen": 1209925632
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005118555667001003,
+ "loss": 1.3686,
+ "theoretical_loss": 3.6131948541557564,
+ "tokens_seen": 1209991168
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511839518555667,
+ "loss": 1.325,
+ "theoretical_loss": 3.613177022716666,
+ "tokens_seen": 1210056704
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005118234704112337,
+ "loss": 1.4054,
+ "theoretical_loss": 3.613159192513681,
+ "tokens_seen": 1210122240
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005118074222668004,
+ "loss": 1.369,
+ "theoretical_loss": 3.6131413635466485,
+ "tokens_seen": 1210187776
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511791374122367,
+ "loss": 1.3287,
+ "theoretical_loss": 3.613123535815417,
+ "tokens_seen": 1210253312
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005117753259779338,
+ "loss": 1.3342,
+ "theoretical_loss": 3.6131057093198327,
+ "tokens_seen": 1210318848
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005117592778335004,
+ "loss": 1.4553,
+ "theoretical_loss": 3.613087884059744,
+ "tokens_seen": 1210384384
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005117432296890672,
+ "loss": 1.3799,
+ "theoretical_loss": 3.613070060034997,
+ "tokens_seen": 1210449920
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005117271815446338,
+ "loss": 1.4055,
+ "theoretical_loss": 3.613052237245441,
+ "tokens_seen": 1210515456
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005117111334002006,
+ "loss": 1.4076,
+ "theoretical_loss": 3.6130344156909215,
+ "tokens_seen": 1210580992
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005116950852557673,
+ "loss": 1.301,
+ "theoretical_loss": 3.6130165953712883,
+ "tokens_seen": 1210646528
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511679037111334,
+ "loss": 1.4515,
+ "theoretical_loss": 3.6129987762863873,
+ "tokens_seen": 1210712064
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005116629889669007,
+ "loss": 1.4408,
+ "theoretical_loss": 3.612980958436067,
+ "tokens_seen": 1210777600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005116469408224674,
+ "loss": 1.4515,
+ "theoretical_loss": 3.6129631418201744,
+ "tokens_seen": 1210843136
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005116308926780341,
+ "loss": 1.3465,
+ "theoretical_loss": 3.612945326438558,
+ "tokens_seen": 1210908672
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005116148445336008,
+ "loss": 1.2689,
+ "theoretical_loss": 3.6129275122910647,
+ "tokens_seen": 1210974208
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005115987963891675,
+ "loss": 1.4271,
+ "theoretical_loss": 3.6129096993775427,
+ "tokens_seen": 1211039744
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005115827482447342,
+ "loss": 1.3598,
+ "theoretical_loss": 3.6128918876978395,
+ "tokens_seen": 1211105280
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511566700100301,
+ "loss": 1.2845,
+ "theoretical_loss": 3.612874077251803,
+ "tokens_seen": 1211170816
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005115506519558676,
+ "loss": 1.4193,
+ "theoretical_loss": 3.6128562680392813,
+ "tokens_seen": 1211236352
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005115346038114343,
+ "loss": 1.31,
+ "theoretical_loss": 3.6128384600601215,
+ "tokens_seen": 1211301888
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511518555667001,
+ "loss": 1.3781,
+ "theoretical_loss": 3.6128206533141722,
+ "tokens_seen": 1211367424
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005115025075225678,
+ "loss": 1.3714,
+ "theoretical_loss": 3.612802847801281,
+ "tokens_seen": 1211432960
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005114864593781344,
+ "loss": 1.3739,
+ "theoretical_loss": 3.6127850435212956,
+ "tokens_seen": 1211498496
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005114704112337012,
+ "loss": 1.4279,
+ "theoretical_loss": 3.612767240474064,
+ "tokens_seen": 1211564032
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005114543630892678,
+ "loss": 1.3829,
+ "theoretical_loss": 3.612749438659435,
+ "tokens_seen": 1211629568
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005114383149448346,
+ "loss": 1.45,
+ "theoretical_loss": 3.612731638077255,
+ "tokens_seen": 1211695104
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005114222668004012,
+ "loss": 1.3736,
+ "theoretical_loss": 3.612713838727373,
+ "tokens_seen": 1211760640
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511406218655968,
+ "loss": 1.3915,
+ "theoretical_loss": 3.612696040609638,
+ "tokens_seen": 1211826176
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005113901705115346,
+ "loss": 1.2323,
+ "theoretical_loss": 3.612678243723896,
+ "tokens_seen": 1211891712
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005113741223671014,
+ "loss": 1.2744,
+ "theoretical_loss": 3.612660448069997,
+ "tokens_seen": 1211957248
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511358074222668,
+ "loss": 1.3926,
+ "theoretical_loss": 3.6126426536477876,
+ "tokens_seen": 1212022784
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005113420260782348,
+ "loss": 1.4082,
+ "theoretical_loss": 3.6126248604571174,
+ "tokens_seen": 1212088320
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005113259779338014,
+ "loss": 1.3457,
+ "theoretical_loss": 3.6126070684978333,
+ "tokens_seen": 1212153856
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005113099297893682,
+ "loss": 1.2181,
+ "theoretical_loss": 3.612589277769784,
+ "tokens_seen": 1212219392
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005112938816449348,
+ "loss": 1.3968,
+ "theoretical_loss": 3.612571488272818,
+ "tokens_seen": 1212284928
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005112778335005016,
+ "loss": 1.4058,
+ "theoretical_loss": 3.6125537000067833,
+ "tokens_seen": 1212350464
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 407314,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.369948148727417,
+ "objective/train/theoretical_loss": 3.6125359129715284,
+ "objective/train/tokens_used": 1232876000,
+ "theoretical_loss": 3.6125359129715284,
+ "tokens_seen": 1212416000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005112617853560682,
+ "loss": 1.3099,
+ "theoretical_loss": 3.6125359129715284,
+ "tokens_seen": 1212416000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511245737211635,
+ "loss": 1.3631,
+ "theoretical_loss": 3.6125181271669016,
+ "tokens_seen": 1212481536
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005112296890672016,
+ "loss": 1.3632,
+ "theoretical_loss": 3.6125003425927504,
+ "tokens_seen": 1212547072
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005112136409227684,
+ "loss": 1.2103,
+ "theoretical_loss": 3.6124825592489245,
+ "tokens_seen": 1212612608
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511197592778335,
+ "loss": 1.3042,
+ "theoretical_loss": 3.612464777135272,
+ "tokens_seen": 1212678144
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111815446339018,
+ "loss": 1.3234,
+ "theoretical_loss": 3.6124469962516406,
+ "tokens_seen": 1212743680
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111654964894684,
+ "loss": 1.4118,
+ "theoretical_loss": 3.612429216597879,
+ "tokens_seen": 1212809216
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111494483450352,
+ "loss": 1.3437,
+ "theoretical_loss": 3.6124114381738357,
+ "tokens_seen": 1212874752
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111334002006018,
+ "loss": 1.3639,
+ "theoretical_loss": 3.61239366097936,
+ "tokens_seen": 1212940288
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111173520561686,
+ "loss": 1.3357,
+ "theoretical_loss": 3.6123758850142993,
+ "tokens_seen": 1213005824
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005111013039117352,
+ "loss": 1.3217,
+ "theoretical_loss": 3.612358110278503,
+ "tokens_seen": 1213071360
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511085255767302,
+ "loss": 1.3236,
+ "theoretical_loss": 3.612340336771819,
+ "tokens_seen": 1213136896
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005110692076228686,
+ "loss": 1.2687,
+ "theoretical_loss": 3.6123225644940966,
+ "tokens_seen": 1213202432
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005110531594784354,
+ "loss": 1.3901,
+ "theoretical_loss": 3.612304793445184,
+ "tokens_seen": 1213267968
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000511037111334002,
+ "loss": 1.4303,
+ "theoretical_loss": 3.61228702362493,
+ "tokens_seen": 1213333504
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005110210631895688,
+ "loss": 1.2671,
+ "theoretical_loss": 3.6122692550331834,
+ "tokens_seen": 1213399040
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005110050150451354,
+ "loss": 1.337,
+ "theoretical_loss": 3.612251487669792,
+ "tokens_seen": 1213464576
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005109889669007022,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6122337215346065,
+ "tokens_seen": 1213530112
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005109729187562688,
+ "loss": 1.39,
+ "theoretical_loss": 3.612215956627474,
+ "tokens_seen": 1213595648
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005109568706118356,
+ "loss": 1.3999,
+ "theoretical_loss": 3.612198192948244,
+ "tokens_seen": 1213661184
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005109408224674022,
+ "loss": 1.3007,
+ "theoretical_loss": 3.612180430496765,
+ "tokens_seen": 1213726720
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510924774322969,
+ "loss": 1.3578,
+ "theoretical_loss": 3.6121626692728857,
+ "tokens_seen": 1213792256
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005109087261785356,
+ "loss": 1.2689,
+ "theoretical_loss": 3.6121449092764557,
+ "tokens_seen": 1213857792
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005108926780341024,
+ "loss": 1.2337,
+ "theoretical_loss": 3.6121271505073236,
+ "tokens_seen": 1213923328
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510876629889669,
+ "loss": 1.384,
+ "theoretical_loss": 3.612109392965338,
+ "tokens_seen": 1213988864
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005108605817452358,
+ "loss": 1.3511,
+ "theoretical_loss": 3.612091636650348,
+ "tokens_seen": 1214054400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005108445336008024,
+ "loss": 1.3522,
+ "theoretical_loss": 3.6120738815622033,
+ "tokens_seen": 1214119936
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005108284854563692,
+ "loss": 1.2782,
+ "theoretical_loss": 3.6120561277007517,
+ "tokens_seen": 1214185472
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005108124373119358,
+ "loss": 1.347,
+ "theoretical_loss": 3.612038375065843,
+ "tokens_seen": 1214251008
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005107963891675026,
+ "loss": 1.322,
+ "theoretical_loss": 3.612020623657326,
+ "tokens_seen": 1214316544
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005107803410230692,
+ "loss": 1.3969,
+ "theoretical_loss": 3.61200287347505,
+ "tokens_seen": 1214382080
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510764292878636,
+ "loss": 1.4987,
+ "theoretical_loss": 3.6119851245188643,
+ "tokens_seen": 1214447616
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005107482447342026,
+ "loss": 1.2335,
+ "theoretical_loss": 3.6119673767886176,
+ "tokens_seen": 1214513152
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005107321965897694,
+ "loss": 1.374,
+ "theoretical_loss": 3.611949630284159,
+ "tokens_seen": 1214578688
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510716148445336,
+ "loss": 1.4743,
+ "theoretical_loss": 3.6119318850053386,
+ "tokens_seen": 1214644224
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005107001003009028,
+ "loss": 1.3383,
+ "theoretical_loss": 3.611914140952005,
+ "tokens_seen": 1214709760
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005106840521564694,
+ "loss": 1.3486,
+ "theoretical_loss": 3.6118963981240073,
+ "tokens_seen": 1214775296
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005106680040120362,
+ "loss": 1.4368,
+ "theoretical_loss": 3.6118786565211947,
+ "tokens_seen": 1214840832
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005106519558676028,
+ "loss": 1.3773,
+ "theoretical_loss": 3.6118609161434168,
+ "tokens_seen": 1214906368
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005106359077231696,
+ "loss": 1.4838,
+ "theoretical_loss": 3.611843176990523,
+ "tokens_seen": 1214971904
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005106198595787362,
+ "loss": 1.4517,
+ "theoretical_loss": 3.6118254390623625,
+ "tokens_seen": 1215037440
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510603811434303,
+ "loss": 1.278,
+ "theoretical_loss": 3.611807702358785,
+ "tokens_seen": 1215102976
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005105877632898696,
+ "loss": 1.3935,
+ "theoretical_loss": 3.6117899668796394,
+ "tokens_seen": 1215168512
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005105717151454364,
+ "loss": 1.2404,
+ "theoretical_loss": 3.6117722326247756,
+ "tokens_seen": 1215234048
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510555667001003,
+ "loss": 1.4078,
+ "theoretical_loss": 3.611754499594043,
+ "tokens_seen": 1215299584
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005105396188565698,
+ "loss": 1.3754,
+ "theoretical_loss": 3.6117367677872907,
+ "tokens_seen": 1215365120
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005105235707121364,
+ "loss": 1.334,
+ "theoretical_loss": 3.6117190372043684,
+ "tokens_seen": 1215430656
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005105075225677031,
+ "loss": 1.2575,
+ "theoretical_loss": 3.611701307845126,
+ "tokens_seen": 1215496192
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104914744232698,
+ "loss": 1.3585,
+ "theoretical_loss": 3.611683579709413,
+ "tokens_seen": 1215561728
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104754262788365,
+ "loss": 1.2716,
+ "theoretical_loss": 3.6116658527970786,
+ "tokens_seen": 1215627264
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 408591,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5982543230056763,
+ "objective/train/theoretical_loss": 3.611648127107973,
+ "objective/train/tokens_used": 1236152800,
+ "theoretical_loss": 3.611648127107973,
+ "tokens_seen": 1215692800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104593781344032,
+ "loss": 1.5032,
+ "theoretical_loss": 3.611648127107973,
+ "tokens_seen": 1215692800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104433299899699,
+ "loss": 1.345,
+ "theoretical_loss": 3.6116304026419446,
+ "tokens_seen": 1215758336
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104272818455366,
+ "loss": 1.3119,
+ "theoretical_loss": 3.611612679398845,
+ "tokens_seen": 1215823872
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005104112337011033,
+ "loss": 1.3545,
+ "theoretical_loss": 3.6115949573785224,
+ "tokens_seen": 1215889408
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.00051039518555667,
+ "loss": 1.4228,
+ "theoretical_loss": 3.6115772365808274,
+ "tokens_seen": 1215954944
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005103791374122367,
+ "loss": 1.3251,
+ "theoretical_loss": 3.611559517005609,
+ "tokens_seen": 1216020480
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005103630892678034,
+ "loss": 1.4169,
+ "theoretical_loss": 3.611541798652718,
+ "tokens_seen": 1216086016
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005103470411233701,
+ "loss": 1.4134,
+ "theoretical_loss": 3.6115240815220035,
+ "tokens_seen": 1216151552
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005103309929789368,
+ "loss": 1.3452,
+ "theoretical_loss": 3.611506365613315,
+ "tokens_seen": 1216217088
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005103149448345035,
+ "loss": 1.5155,
+ "theoretical_loss": 3.6114886509265034,
+ "tokens_seen": 1216282624
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102988966900702,
+ "loss": 1.4263,
+ "theoretical_loss": 3.6114709374614176,
+ "tokens_seen": 1216348160
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102828485456369,
+ "loss": 1.4095,
+ "theoretical_loss": 3.611453225217908,
+ "tokens_seen": 1216413696
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102668004012036,
+ "loss": 1.3857,
+ "theoretical_loss": 3.6114355141958248,
+ "tokens_seen": 1216479232
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102507522567703,
+ "loss": 1.495,
+ "theoretical_loss": 3.6114178043950176,
+ "tokens_seen": 1216544768
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510234704112337,
+ "loss": 1.2786,
+ "theoretical_loss": 3.6114000958153363,
+ "tokens_seen": 1216610304
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102186559679037,
+ "loss": 1.253,
+ "theoretical_loss": 3.611382388456631,
+ "tokens_seen": 1216675840
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005102026078234704,
+ "loss": 1.4706,
+ "theoretical_loss": 3.611364682318752,
+ "tokens_seen": 1216741376
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101865596790371,
+ "loss": 1.4155,
+ "theoretical_loss": 3.611346977401549,
+ "tokens_seen": 1216806912
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101705115346038,
+ "loss": 1.4166,
+ "theoretical_loss": 3.6113292737048734,
+ "tokens_seen": 1216872448
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101544633901705,
+ "loss": 1.3025,
+ "theoretical_loss": 3.6113115712285735,
+ "tokens_seen": 1216937984
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101384152457373,
+ "loss": 1.3737,
+ "theoretical_loss": 3.6112938699725,
+ "tokens_seen": 1217003520
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101223671013039,
+ "loss": 1.4429,
+ "theoretical_loss": 3.611276169936504,
+ "tokens_seen": 1217069056
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005101063189568707,
+ "loss": 1.4673,
+ "theoretical_loss": 3.6112584711204345,
+ "tokens_seen": 1217134592
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005100902708124373,
+ "loss": 1.3303,
+ "theoretical_loss": 3.611240773524143,
+ "tokens_seen": 1217200128
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000510074222668004,
+ "loss": 1.3982,
+ "theoretical_loss": 3.6112230771474785,
+ "tokens_seen": 1217265664
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005100581745235707,
+ "loss": 1.3113,
+ "theoretical_loss": 3.611205381990292,
+ "tokens_seen": 1217331200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005100421263791375,
+ "loss": 1.3865,
+ "theoretical_loss": 3.611187688052433,
+ "tokens_seen": 1217396736
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005100260782347041,
+ "loss": 1.3629,
+ "theoretical_loss": 3.6111699953337535,
+ "tokens_seen": 1217462272
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005100100300902709,
+ "loss": 1.3549,
+ "theoretical_loss": 3.6111523038341025,
+ "tokens_seen": 1217527808
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099939819458375,
+ "loss": 1.2067,
+ "theoretical_loss": 3.6111346135533307,
+ "tokens_seen": 1217593344
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099779338014043,
+ "loss": 1.2571,
+ "theoretical_loss": 3.611116924491289,
+ "tokens_seen": 1217658880
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099618856569709,
+ "loss": 1.2841,
+ "theoretical_loss": 3.611099236647827,
+ "tokens_seen": 1217724416
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099458375125377,
+ "loss": 1.321,
+ "theoretical_loss": 3.6110815500227957,
+ "tokens_seen": 1217789952
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099297893681043,
+ "loss": 1.5426,
+ "theoretical_loss": 3.611063864616046,
+ "tokens_seen": 1217855488
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005099137412236711,
+ "loss": 1.2651,
+ "theoretical_loss": 3.6110461804274276,
+ "tokens_seen": 1217921024
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098976930792377,
+ "loss": 1.4044,
+ "theoretical_loss": 3.6110284974567914,
+ "tokens_seen": 1217986560
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098816449348045,
+ "loss": 1.2246,
+ "theoretical_loss": 3.611010815703988,
+ "tokens_seen": 1218052096
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098655967903711,
+ "loss": 1.2565,
+ "theoretical_loss": 3.610993135168868,
+ "tokens_seen": 1218117632
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098495486459379,
+ "loss": 1.4739,
+ "theoretical_loss": 3.6109754558512828,
+ "tokens_seen": 1218183168
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098335005015045,
+ "loss": 1.4314,
+ "theoretical_loss": 3.6109577777510813,
+ "tokens_seen": 1218248704
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098174523570713,
+ "loss": 1.4238,
+ "theoretical_loss": 3.610940100868116,
+ "tokens_seen": 1218314240
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005098014042126379,
+ "loss": 1.4001,
+ "theoretical_loss": 3.6109224252022365,
+ "tokens_seen": 1218379776
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097853560682047,
+ "loss": 1.2788,
+ "theoretical_loss": 3.610904750753294,
+ "tokens_seen": 1218445312
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097693079237713,
+ "loss": 1.3308,
+ "theoretical_loss": 3.610887077521139,
+ "tokens_seen": 1218510848
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097532597793381,
+ "loss": 1.2774,
+ "theoretical_loss": 3.610869405505622,
+ "tokens_seen": 1218576384
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097372116349047,
+ "loss": 1.3115,
+ "theoretical_loss": 3.6108517347065954,
+ "tokens_seen": 1218641920
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097211634904715,
+ "loss": 1.392,
+ "theoretical_loss": 3.610834065123908,
+ "tokens_seen": 1218707456
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005097051153460381,
+ "loss": 1.3531,
+ "theoretical_loss": 3.610816396757412,
+ "tokens_seen": 1218772992
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096890672016049,
+ "loss": 1.3569,
+ "theoretical_loss": 3.610798729606958,
+ "tokens_seen": 1218838528
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096730190571715,
+ "loss": 1.3418,
+ "theoretical_loss": 3.6107810636723965,
+ "tokens_seen": 1218904064
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 409566,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3452516794204712,
+ "objective/train/theoretical_loss": 3.6107633989535786,
+ "objective/train/tokens_used": 1239429600,
+ "theoretical_loss": 3.6107633989535786,
+ "tokens_seen": 1218969600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096569709127383,
+ "loss": 1.4007,
+ "theoretical_loss": 3.6107633989535786,
+ "tokens_seen": 1218969600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096409227683049,
+ "loss": 1.3499,
+ "theoretical_loss": 3.6107457354503554,
+ "tokens_seen": 1219035136
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096248746238717,
+ "loss": 1.4341,
+ "theoretical_loss": 3.610728073162578,
+ "tokens_seen": 1219100672
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005096088264794383,
+ "loss": 1.3204,
+ "theoretical_loss": 3.610710412090098,
+ "tokens_seen": 1219166208
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095927783350051,
+ "loss": 1.3421,
+ "theoretical_loss": 3.610692752232765,
+ "tokens_seen": 1219231744
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095767301905717,
+ "loss": 1.2549,
+ "theoretical_loss": 3.6106750935904315,
+ "tokens_seen": 1219297280
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095606820461385,
+ "loss": 1.3546,
+ "theoretical_loss": 3.610657436162948,
+ "tokens_seen": 1219362816
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095446339017051,
+ "loss": 1.2459,
+ "theoretical_loss": 3.6106397799501653,
+ "tokens_seen": 1219428352
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095285857572719,
+ "loss": 1.3483,
+ "theoretical_loss": 3.6106221249519352,
+ "tokens_seen": 1219493888
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005095125376128385,
+ "loss": 1.5332,
+ "theoretical_loss": 3.6106044711681085,
+ "tokens_seen": 1219559424
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094964894684053,
+ "loss": 1.4287,
+ "theoretical_loss": 3.6105868185985366,
+ "tokens_seen": 1219624960
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094804413239719,
+ "loss": 1.3836,
+ "theoretical_loss": 3.6105691672430704,
+ "tokens_seen": 1219690496
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094643931795387,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6105515171015616,
+ "tokens_seen": 1219756032
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094483450351053,
+ "loss": 1.2934,
+ "theoretical_loss": 3.6105338681738615,
+ "tokens_seen": 1219821568
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094322968906721,
+ "loss": 1.3215,
+ "theoretical_loss": 3.610516220459821,
+ "tokens_seen": 1219887104
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094162487462387,
+ "loss": 1.3114,
+ "theoretical_loss": 3.610498573959292,
+ "tokens_seen": 1219952640
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005094002006018055,
+ "loss": 1.3689,
+ "theoretical_loss": 3.610480928672125,
+ "tokens_seen": 1220018176
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093841524573721,
+ "loss": 1.396,
+ "theoretical_loss": 3.610463284598173,
+ "tokens_seen": 1220083712
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093681043129389,
+ "loss": 1.3632,
+ "theoretical_loss": 3.610445641737285,
+ "tokens_seen": 1220149248
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093520561685055,
+ "loss": 1.3965,
+ "theoretical_loss": 3.610428000089315,
+ "tokens_seen": 1220214784
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093360080240723,
+ "loss": 1.3503,
+ "theoretical_loss": 3.6104103596541126,
+ "tokens_seen": 1220280320
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093199598796389,
+ "loss": 1.3583,
+ "theoretical_loss": 3.6103927204315305,
+ "tokens_seen": 1220345856
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005093039117352057,
+ "loss": 1.3688,
+ "theoretical_loss": 3.6103750824214194,
+ "tokens_seen": 1220411392
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092878635907724,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6103574456236314,
+ "tokens_seen": 1220476928
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092718154463391,
+ "loss": 1.3285,
+ "theoretical_loss": 3.6103398100380177,
+ "tokens_seen": 1220542464
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092557673019057,
+ "loss": 1.36,
+ "theoretical_loss": 3.61032217566443,
+ "tokens_seen": 1220608000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092397191574725,
+ "loss": 1.3331,
+ "theoretical_loss": 3.6103045425027203,
+ "tokens_seen": 1220673536
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092236710130391,
+ "loss": 1.4157,
+ "theoretical_loss": 3.61028691055274,
+ "tokens_seen": 1220739072
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005092076228686059,
+ "loss": 1.3788,
+ "theoretical_loss": 3.6102692798143408,
+ "tokens_seen": 1220804608
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005091915747241725,
+ "loss": 1.3119,
+ "theoretical_loss": 3.610251650287374,
+ "tokens_seen": 1220870144
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005091755265797393,
+ "loss": 1.3138,
+ "theoretical_loss": 3.610234021971692,
+ "tokens_seen": 1220935680
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005091594784353059,
+ "loss": 1.386,
+ "theoretical_loss": 3.6102163948671464,
+ "tokens_seen": 1221001216
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005091434302908727,
+ "loss": 1.4066,
+ "theoretical_loss": 3.6101987689735884,
+ "tokens_seen": 1221066752
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005091273821464393,
+ "loss": 1.3648,
+ "theoretical_loss": 3.6101811442908707,
+ "tokens_seen": 1221132288
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000509111334002006,
+ "loss": 1.4272,
+ "theoretical_loss": 3.610163520818845,
+ "tokens_seen": 1221197824
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090952858575727,
+ "loss": 1.3886,
+ "theoretical_loss": 3.6101458985573625,
+ "tokens_seen": 1221263360
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090792377131394,
+ "loss": 1.4172,
+ "theoretical_loss": 3.6101282775062753,
+ "tokens_seen": 1221328896
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090631895687061,
+ "loss": 1.2674,
+ "theoretical_loss": 3.610110657665436,
+ "tokens_seen": 1221394432
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090471414242728,
+ "loss": 1.4391,
+ "theoretical_loss": 3.6100930390346955,
+ "tokens_seen": 1221459968
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090310932798395,
+ "loss": 1.2714,
+ "theoretical_loss": 3.610075421613907,
+ "tokens_seen": 1221525504
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005090150451354062,
+ "loss": 1.3585,
+ "theoretical_loss": 3.6100578054029215,
+ "tokens_seen": 1221591040
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089989969909729,
+ "loss": 1.3147,
+ "theoretical_loss": 3.610040190401591,
+ "tokens_seen": 1221656576
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089829488465396,
+ "loss": 1.3047,
+ "theoretical_loss": 3.6100225766097687,
+ "tokens_seen": 1221722112
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089669007021063,
+ "loss": 1.3707,
+ "theoretical_loss": 3.610004964027305,
+ "tokens_seen": 1221787648
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508950852557673,
+ "loss": 1.3458,
+ "theoretical_loss": 3.609987352654054,
+ "tokens_seen": 1221853184
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089348044132397,
+ "loss": 1.252,
+ "theoretical_loss": 3.6099697424898656,
+ "tokens_seen": 1221918720
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089187562688064,
+ "loss": 1.3129,
+ "theoretical_loss": 3.609952133534594,
+ "tokens_seen": 1221984256
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005089027081243731,
+ "loss": 1.3465,
+ "theoretical_loss": 3.6099345257880895,
+ "tokens_seen": 1222049792
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005088866599799398,
+ "loss": 1.4346,
+ "theoretical_loss": 3.609916919250206,
+ "tokens_seen": 1222115328
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005088706118355065,
+ "loss": 1.271,
+ "theoretical_loss": 3.6098993139207947,
+ "tokens_seen": 1222180864
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 410881,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3531014919281006,
+ "objective/train/theoretical_loss": 3.6098817097997085,
+ "objective/train/tokens_used": 1242706400,
+ "theoretical_loss": 3.6098817097997085,
+ "tokens_seen": 1222246400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005088545636910732,
+ "loss": 1.4237,
+ "theoretical_loss": 3.6098817097997085,
+ "tokens_seen": 1222246400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.00050883851554664,
+ "loss": 1.3529,
+ "theoretical_loss": 3.609864106886799,
+ "tokens_seen": 1222311936
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005088224674022066,
+ "loss": 1.2548,
+ "theoretical_loss": 3.6098465051819186,
+ "tokens_seen": 1222377472
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005088064192577733,
+ "loss": 1.3921,
+ "theoretical_loss": 3.60982890468492,
+ "tokens_seen": 1222443008
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.00050879037111334,
+ "loss": 1.4104,
+ "theoretical_loss": 3.609811305395656,
+ "tokens_seen": 1222508544
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005087743229689068,
+ "loss": 1.3592,
+ "theoretical_loss": 3.609793707313978,
+ "tokens_seen": 1222574080
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005087582748244734,
+ "loss": 1.326,
+ "theoretical_loss": 3.609776110439739,
+ "tokens_seen": 1222639616
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005087422266800402,
+ "loss": 1.3587,
+ "theoretical_loss": 3.609758514772791,
+ "tokens_seen": 1222705152
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005087261785356068,
+ "loss": 1.3131,
+ "theoretical_loss": 3.609740920312987,
+ "tokens_seen": 1222770688
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005087101303911736,
+ "loss": 1.4157,
+ "theoretical_loss": 3.609723327060179,
+ "tokens_seen": 1222836224
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005086940822467402,
+ "loss": 1.2677,
+ "theoretical_loss": 3.60970573501422,
+ "tokens_seen": 1222901760
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508678034102307,
+ "loss": 1.3885,
+ "theoretical_loss": 3.6096881441749624,
+ "tokens_seen": 1222967296
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005086619859578736,
+ "loss": 1.3491,
+ "theoretical_loss": 3.6096705545422587,
+ "tokens_seen": 1223032832
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005086459378134404,
+ "loss": 1.254,
+ "theoretical_loss": 3.609652966115961,
+ "tokens_seen": 1223098368
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508629889669007,
+ "loss": 1.3122,
+ "theoretical_loss": 3.609635378895923,
+ "tokens_seen": 1223163904
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005086138415245738,
+ "loss": 1.2439,
+ "theoretical_loss": 3.6096177928819966,
+ "tokens_seen": 1223229440
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085977933801404,
+ "loss": 1.4453,
+ "theoretical_loss": 3.609600208074035,
+ "tokens_seen": 1223294976
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085817452357072,
+ "loss": 1.3788,
+ "theoretical_loss": 3.60958262447189,
+ "tokens_seen": 1223360512
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085656970912738,
+ "loss": 1.3732,
+ "theoretical_loss": 3.609565042075414,
+ "tokens_seen": 1223426048
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085496489468406,
+ "loss": 1.3837,
+ "theoretical_loss": 3.609547460884462,
+ "tokens_seen": 1223491584
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085336008024072,
+ "loss": 1.2681,
+ "theoretical_loss": 3.6095298808988847,
+ "tokens_seen": 1223557120
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508517552657974,
+ "loss": 1.4613,
+ "theoretical_loss": 3.609512302118536,
+ "tokens_seen": 1223622656
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005085015045135406,
+ "loss": 1.3357,
+ "theoretical_loss": 3.6094947245432683,
+ "tokens_seen": 1223688192
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005084854563691074,
+ "loss": 1.3543,
+ "theoretical_loss": 3.6094771481729344,
+ "tokens_seen": 1223753728
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508469408224674,
+ "loss": 1.3181,
+ "theoretical_loss": 3.609459573007387,
+ "tokens_seen": 1223819264
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005084533600802408,
+ "loss": 1.3672,
+ "theoretical_loss": 3.609441999046479,
+ "tokens_seen": 1223884800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005084373119358074,
+ "loss": 1.3234,
+ "theoretical_loss": 3.6094244262900643,
+ "tokens_seen": 1223950336
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005084212637913742,
+ "loss": 1.2581,
+ "theoretical_loss": 3.609406854737995,
+ "tokens_seen": 1224015872
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005084052156469408,
+ "loss": 1.3042,
+ "theoretical_loss": 3.609389284390124,
+ "tokens_seen": 1224081408
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005083891675025076,
+ "loss": 1.3076,
+ "theoretical_loss": 3.6093717152463043,
+ "tokens_seen": 1224146944
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005083731193580742,
+ "loss": 1.3272,
+ "theoretical_loss": 3.60935414730639,
+ "tokens_seen": 1224212480
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508357071213641,
+ "loss": 1.3615,
+ "theoretical_loss": 3.6093365805702327,
+ "tokens_seen": 1224278016
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005083410230692076,
+ "loss": 1.4112,
+ "theoretical_loss": 3.609319015037686,
+ "tokens_seen": 1224343552
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005083249749247744,
+ "loss": 1.2945,
+ "theoretical_loss": 3.609301450708603,
+ "tokens_seen": 1224409088
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508308926780341,
+ "loss": 1.3688,
+ "theoretical_loss": 3.609283887582838,
+ "tokens_seen": 1224474624
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082928786359078,
+ "loss": 1.4659,
+ "theoretical_loss": 3.6092663256602426,
+ "tokens_seen": 1224540160
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082768304914744,
+ "loss": 1.371,
+ "theoretical_loss": 3.60924876494067,
+ "tokens_seen": 1224605696
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082607823470412,
+ "loss": 1.4184,
+ "theoretical_loss": 3.6092312054239746,
+ "tokens_seen": 1224671232
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082447342026078,
+ "loss": 1.344,
+ "theoretical_loss": 3.6092136471100087,
+ "tokens_seen": 1224736768
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082286860581746,
+ "loss": 1.2974,
+ "theoretical_loss": 3.609196089998626,
+ "tokens_seen": 1224802304
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005082126379137412,
+ "loss": 1.3757,
+ "theoretical_loss": 3.6091785340896796,
+ "tokens_seen": 1224867840
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508196589769308,
+ "loss": 1.4648,
+ "theoretical_loss": 3.609160979383023,
+ "tokens_seen": 1224933376
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005081805416248746,
+ "loss": 1.286,
+ "theoretical_loss": 3.609143425878509,
+ "tokens_seen": 1224998912
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005081644934804414,
+ "loss": 1.3262,
+ "theoretical_loss": 3.6091258735759917,
+ "tokens_seen": 1225064448
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508148445336008,
+ "loss": 1.3932,
+ "theoretical_loss": 3.609108322475324,
+ "tokens_seen": 1225129984
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005081323971915748,
+ "loss": 1.3762,
+ "theoretical_loss": 3.60909077257636,
+ "tokens_seen": 1225195520
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005081163490471414,
+ "loss": 1.3016,
+ "theoretical_loss": 3.6090732238789522,
+ "tokens_seen": 1225261056
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005081003009027082,
+ "loss": 1.4088,
+ "theoretical_loss": 3.6090556763829547,
+ "tokens_seen": 1225326592
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005080842527582749,
+ "loss": 1.3233,
+ "theoretical_loss": 3.609038130088221,
+ "tokens_seen": 1225392128
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005080682046138416,
+ "loss": 1.3382,
+ "theoretical_loss": 3.609020584994604,
+ "tokens_seen": 1225457664
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 411823,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3464809656143188,
+ "objective/train/theoretical_loss": 3.609003041101958,
+ "objective/train/tokens_used": 1245983200,
+ "theoretical_loss": 3.609003041101958,
+ "tokens_seen": 1225523200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005080521564694083,
+ "loss": 1.4249,
+ "theoretical_loss": 3.609003041101958,
+ "tokens_seen": 1225523200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000508036108324975,
+ "loss": 1.3424,
+ "theoretical_loss": 3.608985498410136,
+ "tokens_seen": 1225588736
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005080200601805417,
+ "loss": 1.3674,
+ "theoretical_loss": 3.6089679569189923,
+ "tokens_seen": 1225654272
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005080040120361084,
+ "loss": 1.291,
+ "theoretical_loss": 3.60895041662838,
+ "tokens_seen": 1225719808
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079879638916751,
+ "loss": 1.3059,
+ "theoretical_loss": 3.6089328775381526,
+ "tokens_seen": 1225785344
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079719157472418,
+ "loss": 1.4017,
+ "theoretical_loss": 3.6089153396481644,
+ "tokens_seen": 1225850880
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079558676028085,
+ "loss": 1.3051,
+ "theoretical_loss": 3.6088978029582686,
+ "tokens_seen": 1225916416
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079398194583752,
+ "loss": 1.3591,
+ "theoretical_loss": 3.608880267468319,
+ "tokens_seen": 1225981952
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079237713139419,
+ "loss": 1.3276,
+ "theoretical_loss": 3.6088627331781695,
+ "tokens_seen": 1226047488
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005079077231695086,
+ "loss": 1.3382,
+ "theoretical_loss": 3.608845200087674,
+ "tokens_seen": 1226113024
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005078916750250753,
+ "loss": 1.3726,
+ "theoretical_loss": 3.6088276681966858,
+ "tokens_seen": 1226178560
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000507875626880642,
+ "loss": 1.3957,
+ "theoretical_loss": 3.608810137505059,
+ "tokens_seen": 1226244096
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005078595787362086,
+ "loss": 1.382,
+ "theoretical_loss": 3.6087926080126476,
+ "tokens_seen": 1226309632
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005078435305917754,
+ "loss": 1.2793,
+ "theoretical_loss": 3.6087750797193054,
+ "tokens_seen": 1226375168
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000507827482447342,
+ "loss": 1.376,
+ "theoretical_loss": 3.608757552624887,
+ "tokens_seen": 1226440704
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005078114343029088,
+ "loss": 1.3469,
+ "theoretical_loss": 3.6087400267292447,
+ "tokens_seen": 1226506240
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077953861584754,
+ "loss": 1.4904,
+ "theoretical_loss": 3.6087225020322338,
+ "tokens_seen": 1226571776
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077793380140422,
+ "loss": 1.3039,
+ "theoretical_loss": 3.6087049785337078,
+ "tokens_seen": 1226637312
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077632898696088,
+ "loss": 1.3077,
+ "theoretical_loss": 3.608687456233521,
+ "tokens_seen": 1226702848
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077472417251756,
+ "loss": 1.3648,
+ "theoretical_loss": 3.6086699351315272,
+ "tokens_seen": 1226768384
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077311935807422,
+ "loss": 1.3395,
+ "theoretical_loss": 3.6086524152275805,
+ "tokens_seen": 1226833920
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005077151454363089,
+ "loss": 1.3836,
+ "theoretical_loss": 3.6086348965215347,
+ "tokens_seen": 1226899456
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076990972918756,
+ "loss": 1.2947,
+ "theoretical_loss": 3.6086173790132445,
+ "tokens_seen": 1226964992
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076830491474423,
+ "loss": 1.4456,
+ "theoretical_loss": 3.608599862702564,
+ "tokens_seen": 1227030528
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000507667001003009,
+ "loss": 1.3566,
+ "theoretical_loss": 3.608582347589347,
+ "tokens_seen": 1227096064
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076509528585757,
+ "loss": 1.3627,
+ "theoretical_loss": 3.608564833673447,
+ "tokens_seen": 1227161600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076349047141424,
+ "loss": 1.3629,
+ "theoretical_loss": 3.60854732095472,
+ "tokens_seen": 1227227136
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076188565697091,
+ "loss": 1.2041,
+ "theoretical_loss": 3.608529809433019,
+ "tokens_seen": 1227292672
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005076028084252758,
+ "loss": 1.4043,
+ "theoretical_loss": 3.6085122991081984,
+ "tokens_seen": 1227358208
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005075867602808425,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6084947899801127,
+ "tokens_seen": 1227423744
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005075707121364092,
+ "loss": 1.3638,
+ "theoretical_loss": 3.608477282048616,
+ "tokens_seen": 1227489280
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005075546639919759,
+ "loss": 1.3437,
+ "theoretical_loss": 3.6084597753135625,
+ "tokens_seen": 1227554816
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005075386158475426,
+ "loss": 1.2933,
+ "theoretical_loss": 3.6084422697748075,
+ "tokens_seen": 1227620352
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005075225677031093,
+ "loss": 1.3695,
+ "theoretical_loss": 3.608424765432205,
+ "tokens_seen": 1227685888
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000507506519558676,
+ "loss": 1.3435,
+ "theoretical_loss": 3.608407262285608,
+ "tokens_seen": 1227751424
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074904714142427,
+ "loss": 1.3852,
+ "theoretical_loss": 3.608389760334873,
+ "tokens_seen": 1227816960
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074744232698094,
+ "loss": 1.3372,
+ "theoretical_loss": 3.6083722595798537,
+ "tokens_seen": 1227882496
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074583751253761,
+ "loss": 1.2699,
+ "theoretical_loss": 3.6083547600204042,
+ "tokens_seen": 1227948032
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074423269809428,
+ "loss": 1.4889,
+ "theoretical_loss": 3.6083372616563794,
+ "tokens_seen": 1228013568
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074262788365095,
+ "loss": 1.3554,
+ "theoretical_loss": 3.6083197644876335,
+ "tokens_seen": 1228079104
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005074102306920762,
+ "loss": 1.3262,
+ "theoretical_loss": 3.608302268514022,
+ "tokens_seen": 1228144640
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005073941825476429,
+ "loss": 1.3699,
+ "theoretical_loss": 3.608284773735398,
+ "tokens_seen": 1228210176
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005073781344032097,
+ "loss": 1.2882,
+ "theoretical_loss": 3.6082672801516176,
+ "tokens_seen": 1228275712
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005073620862587763,
+ "loss": 1.2018,
+ "theoretical_loss": 3.608249787762534,
+ "tokens_seen": 1228341248
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000507346038114343,
+ "loss": 1.2808,
+ "theoretical_loss": 3.6082322965680036,
+ "tokens_seen": 1228406784
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005073299899699097,
+ "loss": 1.4004,
+ "theoretical_loss": 3.6082148065678794,
+ "tokens_seen": 1228472320
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005073139418254765,
+ "loss": 1.3758,
+ "theoretical_loss": 3.608197317762018,
+ "tokens_seen": 1228537856
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072978936810431,
+ "loss": 1.3479,
+ "theoretical_loss": 3.6081798301502723,
+ "tokens_seen": 1228603392
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072818455366099,
+ "loss": 1.3519,
+ "theoretical_loss": 3.6081623437324977,
+ "tokens_seen": 1228668928
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072657973921765,
+ "loss": 1.4185,
+ "theoretical_loss": 3.60814485850855,
+ "tokens_seen": 1228734464
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 413048,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3460556268692017,
+ "objective/train/theoretical_loss": 3.608127374478282,
+ "objective/train/tokens_used": 1249260000,
+ "theoretical_loss": 3.608127374478282,
+ "tokens_seen": 1228800000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072497492477433,
+ "loss": 1.3908,
+ "theoretical_loss": 3.608127374478282,
+ "tokens_seen": 1228800000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072337011033099,
+ "loss": 1.286,
+ "theoretical_loss": 3.6081098916415506,
+ "tokens_seen": 1228865536
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072176529588767,
+ "loss": 1.3386,
+ "theoretical_loss": 3.6080924099982092,
+ "tokens_seen": 1228931072
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005072016048144433,
+ "loss": 1.3347,
+ "theoretical_loss": 3.608074929548114,
+ "tokens_seen": 1228996608
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071855566700101,
+ "loss": 1.4125,
+ "theoretical_loss": 3.6080574502911187,
+ "tokens_seen": 1229062144
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071695085255767,
+ "loss": 1.3884,
+ "theoretical_loss": 3.608039972227079,
+ "tokens_seen": 1229127680
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071534603811435,
+ "loss": 1.2577,
+ "theoretical_loss": 3.6080224953558497,
+ "tokens_seen": 1229193216
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071374122367101,
+ "loss": 1.3599,
+ "theoretical_loss": 3.6080050196772855,
+ "tokens_seen": 1229258752
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071213640922769,
+ "loss": 1.4663,
+ "theoretical_loss": 3.607987545191242,
+ "tokens_seen": 1229324288
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005071053159478435,
+ "loss": 1.2423,
+ "theoretical_loss": 3.607970071897574,
+ "tokens_seen": 1229389824
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070892678034103,
+ "loss": 1.3145,
+ "theoretical_loss": 3.6079525997961364,
+ "tokens_seen": 1229455360
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070732196589769,
+ "loss": 1.3346,
+ "theoretical_loss": 3.6079351288867842,
+ "tokens_seen": 1229520896
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070571715145437,
+ "loss": 1.3632,
+ "theoretical_loss": 3.607917659169373,
+ "tokens_seen": 1229586432
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070411233701103,
+ "loss": 1.3559,
+ "theoretical_loss": 3.607900190643758,
+ "tokens_seen": 1229651968
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070250752256771,
+ "loss": 1.3823,
+ "theoretical_loss": 3.607882723309794,
+ "tokens_seen": 1229717504
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005070090270812437,
+ "loss": 1.3266,
+ "theoretical_loss": 3.607865257167336,
+ "tokens_seen": 1229783040
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069929789368105,
+ "loss": 1.3763,
+ "theoretical_loss": 3.60784779221624,
+ "tokens_seen": 1229848576
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069769307923771,
+ "loss": 1.4658,
+ "theoretical_loss": 3.6078303284563606,
+ "tokens_seen": 1229914112
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069608826479439,
+ "loss": 1.273,
+ "theoretical_loss": 3.6078128658875537,
+ "tokens_seen": 1229979648
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069448345035105,
+ "loss": 1.3706,
+ "theoretical_loss": 3.607795404509674,
+ "tokens_seen": 1230045184
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069287863590773,
+ "loss": 1.3904,
+ "theoretical_loss": 3.607777944322577,
+ "tokens_seen": 1230110720
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005069127382146439,
+ "loss": 1.3611,
+ "theoretical_loss": 3.607760485326118,
+ "tokens_seen": 1230176256
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068966900702107,
+ "loss": 1.5583,
+ "theoretical_loss": 3.6077430275201525,
+ "tokens_seen": 1230241792
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068806419257774,
+ "loss": 1.3757,
+ "theoretical_loss": 3.607725570904536,
+ "tokens_seen": 1230307328
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068645937813441,
+ "loss": 1.4449,
+ "theoretical_loss": 3.6077081154791237,
+ "tokens_seen": 1230372864
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068485456369108,
+ "loss": 1.3612,
+ "theoretical_loss": 3.6076906612437716,
+ "tokens_seen": 1230438400
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068324974924775,
+ "loss": 1.2978,
+ "theoretical_loss": 3.6076732081983347,
+ "tokens_seen": 1230503936
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068164493480442,
+ "loss": 1.3877,
+ "theoretical_loss": 3.6076557563426688,
+ "tokens_seen": 1230569472
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005068004012036109,
+ "loss": 1.3809,
+ "theoretical_loss": 3.6076383056766286,
+ "tokens_seen": 1230635008
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005067843530591776,
+ "loss": 1.3526,
+ "theoretical_loss": 3.6076208562000707,
+ "tokens_seen": 1230700544
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005067683049147443,
+ "loss": 1.3833,
+ "theoretical_loss": 3.6076034079128503,
+ "tokens_seen": 1230766080
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000506752256770311,
+ "loss": 1.3462,
+ "theoretical_loss": 3.6075859608148226,
+ "tokens_seen": 1230831616
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005067362086258777,
+ "loss": 1.3983,
+ "theoretical_loss": 3.6075685149058443,
+ "tokens_seen": 1230897152
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005067201604814444,
+ "loss": 1.3677,
+ "theoretical_loss": 3.60755107018577,
+ "tokens_seen": 1230962688
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005067041123370111,
+ "loss": 1.3699,
+ "theoretical_loss": 3.607533626654456,
+ "tokens_seen": 1231028224
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066880641925778,
+ "loss": 1.3833,
+ "theoretical_loss": 3.607516184311758,
+ "tokens_seen": 1231093760
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066720160481445,
+ "loss": 1.3335,
+ "theoretical_loss": 3.6074987431575307,
+ "tokens_seen": 1231159296
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066559679037112,
+ "loss": 1.397,
+ "theoretical_loss": 3.6074813031916317,
+ "tokens_seen": 1231224832
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066399197592779,
+ "loss": 1.3204,
+ "theoretical_loss": 3.607463864413915,
+ "tokens_seen": 1231290368
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066238716148446,
+ "loss": 1.329,
+ "theoretical_loss": 3.6074464268242377,
+ "tokens_seen": 1231355904
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005066078234704113,
+ "loss": 1.4027,
+ "theoretical_loss": 3.607428990422455,
+ "tokens_seen": 1231421440
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000506591775325978,
+ "loss": 1.328,
+ "theoretical_loss": 3.607411555208423,
+ "tokens_seen": 1231486976
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005065757271815447,
+ "loss": 1.3285,
+ "theoretical_loss": 3.6073941211819975,
+ "tokens_seen": 1231552512
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005065596790371114,
+ "loss": 1.2685,
+ "theoretical_loss": 3.6073766883430345,
+ "tokens_seen": 1231618048
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005065436308926781,
+ "loss": 1.3626,
+ "theoretical_loss": 3.6073592566913897,
+ "tokens_seen": 1231683584
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005065275827482448,
+ "loss": 1.3369,
+ "theoretical_loss": 3.6073418262269197,
+ "tokens_seen": 1231749120
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005065115346038115,
+ "loss": 1.3733,
+ "theoretical_loss": 3.6073243969494793,
+ "tokens_seen": 1231814656
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064954864593782,
+ "loss": 1.3092,
+ "theoretical_loss": 3.6073069688589254,
+ "tokens_seen": 1231880192
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064794383149449,
+ "loss": 1.2845,
+ "theoretical_loss": 3.6072895419551143,
+ "tokens_seen": 1231945728
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064633901705115,
+ "loss": 1.3873,
+ "theoretical_loss": 3.607272116237901,
+ "tokens_seen": 1232011264
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 414319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.253023624420166,
+ "objective/train/theoretical_loss": 3.6072546917071424,
+ "objective/train/tokens_used": 1252536800,
+ "theoretical_loss": 3.6072546917071424,
+ "tokens_seen": 1232076800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064473420260783,
+ "loss": 1.2735,
+ "theoretical_loss": 3.6072546917071424,
+ "tokens_seen": 1232076800
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064312938816449,
+ "loss": 1.3149,
+ "theoretical_loss": 3.607237268362695,
+ "tokens_seen": 1232142336
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005064152457372117,
+ "loss": 1.4091,
+ "theoretical_loss": 3.607219846204414,
+ "tokens_seen": 1232207872
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063991975927783,
+ "loss": 1.4218,
+ "theoretical_loss": 3.607202425232156,
+ "tokens_seen": 1232273408
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063831494483451,
+ "loss": 1.41,
+ "theoretical_loss": 3.607185005445777,
+ "tokens_seen": 1232338944
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063671013039117,
+ "loss": 1.3715,
+ "theoretical_loss": 3.607167586845134,
+ "tokens_seen": 1232404480
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063510531594785,
+ "loss": 1.3617,
+ "theoretical_loss": 3.6071501694300814,
+ "tokens_seen": 1232470016
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063350050150451,
+ "loss": 1.3739,
+ "theoretical_loss": 3.6071327532004775,
+ "tokens_seen": 1232535552
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063189568706118,
+ "loss": 1.2954,
+ "theoretical_loss": 3.607115338156178,
+ "tokens_seen": 1232601088
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005063029087261785,
+ "loss": 1.3607,
+ "theoretical_loss": 3.6070979242970385,
+ "tokens_seen": 1232666624
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005062868605817452,
+ "loss": 1.3799,
+ "theoretical_loss": 3.6070805116229154,
+ "tokens_seen": 1232732160
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005062708124373119,
+ "loss": 1.4167,
+ "theoretical_loss": 3.607063100133666,
+ "tokens_seen": 1232797696
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005062547642928786,
+ "loss": 1.3713,
+ "theoretical_loss": 3.607045689829146,
+ "tokens_seen": 1232863232
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005062387161484453,
+ "loss": 1.3901,
+ "theoretical_loss": 3.607028280709213,
+ "tokens_seen": 1232928768
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000506222668004012,
+ "loss": 1.2927,
+ "theoretical_loss": 3.6070108727737207,
+ "tokens_seen": 1232994304
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005062066198595787,
+ "loss": 1.285,
+ "theoretical_loss": 3.6069934660225287,
+ "tokens_seen": 1233059840
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005061905717151454,
+ "loss": 1.2497,
+ "theoretical_loss": 3.6069760604554912,
+ "tokens_seen": 1233125376
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005061745235707121,
+ "loss": 1.368,
+ "theoretical_loss": 3.6069586560724662,
+ "tokens_seen": 1233190912
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005061584754262788,
+ "loss": 1.3899,
+ "theoretical_loss": 3.6069412528733094,
+ "tokens_seen": 1233256448
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005061424272818455,
+ "loss": 1.5183,
+ "theoretical_loss": 3.6069238508578776,
+ "tokens_seen": 1233321984
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005061263791374122,
+ "loss": 1.3898,
+ "theoretical_loss": 3.6069064500260275,
+ "tokens_seen": 1233387520
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000506110330992979,
+ "loss": 1.4818,
+ "theoretical_loss": 3.6068890503776156,
+ "tokens_seen": 1233453056
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005060942828485456,
+ "loss": 1.3643,
+ "theoretical_loss": 3.606871651912498,
+ "tokens_seen": 1233518592
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005060782347041123,
+ "loss": 1.3459,
+ "theoretical_loss": 3.606854254630533,
+ "tokens_seen": 1233584128
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000506062186559679,
+ "loss": 1.3058,
+ "theoretical_loss": 3.606836858531576,
+ "tokens_seen": 1233649664
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005060461384152457,
+ "loss": 1.4604,
+ "theoretical_loss": 3.606819463615483,
+ "tokens_seen": 1233715200
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005060300902708124,
+ "loss": 1.365,
+ "theoretical_loss": 3.6068020698821126,
+ "tokens_seen": 1233780736
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005060140421263792,
+ "loss": 1.316,
+ "theoretical_loss": 3.60678467733132,
+ "tokens_seen": 1233846272
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005059979939819458,
+ "loss": 1.3043,
+ "theoretical_loss": 3.606767285962963,
+ "tokens_seen": 1233911808
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005059819458375126,
+ "loss": 1.3951,
+ "theoretical_loss": 3.6067498957768986,
+ "tokens_seen": 1233977344
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005059658976930792,
+ "loss": 1.2649,
+ "theoretical_loss": 3.606732506772982,
+ "tokens_seen": 1234042880
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505949849548646,
+ "loss": 1.4024,
+ "theoretical_loss": 3.606715118951072,
+ "tokens_seen": 1234108416
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005059338014042126,
+ "loss": 1.2778,
+ "theoretical_loss": 3.6066977323110243,
+ "tokens_seen": 1234173952
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005059177532597794,
+ "loss": 1.3418,
+ "theoretical_loss": 3.6066803468526967,
+ "tokens_seen": 1234239488
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505901705115346,
+ "loss": 1.3942,
+ "theoretical_loss": 3.6066629625759448,
+ "tokens_seen": 1234305024
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058856569709128,
+ "loss": 1.2371,
+ "theoretical_loss": 3.6066455794806265,
+ "tokens_seen": 1234370560
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058696088264794,
+ "loss": 1.3195,
+ "theoretical_loss": 3.6066281975665992,
+ "tokens_seen": 1234436096
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058535606820462,
+ "loss": 1.4218,
+ "theoretical_loss": 3.6066108168337188,
+ "tokens_seen": 1234501632
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058375125376128,
+ "loss": 1.4048,
+ "theoretical_loss": 3.6065934372818433,
+ "tokens_seen": 1234567168
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058214643931796,
+ "loss": 1.3959,
+ "theoretical_loss": 3.6065760589108296,
+ "tokens_seen": 1234632704
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005058054162487462,
+ "loss": 1.4088,
+ "theoretical_loss": 3.606558681720534,
+ "tokens_seen": 1234698240
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505789368104313,
+ "loss": 1.3888,
+ "theoretical_loss": 3.6065413057108144,
+ "tokens_seen": 1234763776
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005057733199598796,
+ "loss": 1.3972,
+ "theoretical_loss": 3.606523930881528,
+ "tokens_seen": 1234829312
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005057572718154464,
+ "loss": 1.4086,
+ "theoretical_loss": 3.6065065572325317,
+ "tokens_seen": 1234894848
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505741223671013,
+ "loss": 1.2821,
+ "theoretical_loss": 3.6064891847636824,
+ "tokens_seen": 1234960384
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005057251755265798,
+ "loss": 1.4076,
+ "theoretical_loss": 3.6064718134748377,
+ "tokens_seen": 1235025920
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005057091273821464,
+ "loss": 1.4631,
+ "theoretical_loss": 3.6064544433658545,
+ "tokens_seen": 1235091456
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005056930792377132,
+ "loss": 1.3723,
+ "theoretical_loss": 3.606437074436591,
+ "tokens_seen": 1235156992
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005056770310932799,
+ "loss": 1.2252,
+ "theoretical_loss": 3.6064197066869035,
+ "tokens_seen": 1235222528
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005056609829488466,
+ "loss": 1.3153,
+ "theoretical_loss": 3.606402340116649,
+ "tokens_seen": 1235288064
+ },
+ {
+ "epoch": 0.37,
+ "objective/train/docs_used": 415196,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2908198833465576,
+ "objective/train/theoretical_loss": 3.606384974725686,
+ "objective/train/tokens_used": 1255813600,
+ "theoretical_loss": 3.606384974725686,
+ "tokens_seen": 1235353600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005056449348044133,
+ "loss": 1.3514,
+ "theoretical_loss": 3.606384974725686,
+ "tokens_seen": 1235353600
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.00050562888665998,
+ "loss": 1.3917,
+ "theoretical_loss": 3.606367610513871,
+ "tokens_seen": 1235419136
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005056128385155467,
+ "loss": 1.3514,
+ "theoretical_loss": 3.6063502474810623,
+ "tokens_seen": 1235484672
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055967903711134,
+ "loss": 1.3734,
+ "theoretical_loss": 3.6063328856271166,
+ "tokens_seen": 1235550208
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055807422266801,
+ "loss": 1.2518,
+ "theoretical_loss": 3.6063155249518912,
+ "tokens_seen": 1235615744
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055646940822468,
+ "loss": 1.3106,
+ "theoretical_loss": 3.606298165455244,
+ "tokens_seen": 1235681280
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055486459378135,
+ "loss": 1.4104,
+ "theoretical_loss": 3.606280807137032,
+ "tokens_seen": 1235746816
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055325977933802,
+ "loss": 1.284,
+ "theoretical_loss": 3.606263449997113,
+ "tokens_seen": 1235812352
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055165496489469,
+ "loss": 1.3137,
+ "theoretical_loss": 3.606246094035345,
+ "tokens_seen": 1235877888
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005055005015045136,
+ "loss": 1.364,
+ "theoretical_loss": 3.606228739251585,
+ "tokens_seen": 1235943424
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005054844533600803,
+ "loss": 1.4349,
+ "theoretical_loss": 3.6062113856456905,
+ "tokens_seen": 1236008960
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505468405215647,
+ "loss": 1.4239,
+ "theoretical_loss": 3.6061940332175193,
+ "tokens_seen": 1236074496
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005054523570712137,
+ "loss": 1.393,
+ "theoretical_loss": 3.606176681966929,
+ "tokens_seen": 1236140032
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005054363089267804,
+ "loss": 1.4988,
+ "theoretical_loss": 3.6061593318937777,
+ "tokens_seen": 1236205568
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005054202607823471,
+ "loss": 1.4439,
+ "theoretical_loss": 3.606141982997922,
+ "tokens_seen": 1236271104
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005054042126379138,
+ "loss": 1.4327,
+ "theoretical_loss": 3.6061246352792216,
+ "tokens_seen": 1236336640
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005053881644934805,
+ "loss": 1.3594,
+ "theoretical_loss": 3.606107288737532,
+ "tokens_seen": 1236402176
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005053721163490472,
+ "loss": 1.357,
+ "theoretical_loss": 3.606089943372712,
+ "tokens_seen": 1236467712
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005053560682046139,
+ "loss": 1.2525,
+ "theoretical_loss": 3.6060725991846194,
+ "tokens_seen": 1236533248
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005053400200601806,
+ "loss": 1.3922,
+ "theoretical_loss": 3.6060552561731116,
+ "tokens_seen": 1236598784
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005053239719157473,
+ "loss": 1.4451,
+ "theoretical_loss": 3.6060379143380468,
+ "tokens_seen": 1236664320
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505307923771314,
+ "loss": 1.4254,
+ "theoretical_loss": 3.6060205736792836,
+ "tokens_seen": 1236729856
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052918756268807,
+ "loss": 1.3562,
+ "theoretical_loss": 3.6060032341966783,
+ "tokens_seen": 1236795392
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052758274824474,
+ "loss": 1.4196,
+ "theoretical_loss": 3.6059858958900897,
+ "tokens_seen": 1236860928
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052597793380141,
+ "loss": 1.2609,
+ "theoretical_loss": 3.6059685587593755,
+ "tokens_seen": 1236926464
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052437311935808,
+ "loss": 1.4501,
+ "theoretical_loss": 3.605951222804394,
+ "tokens_seen": 1236992000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052276830491475,
+ "loss": 1.2625,
+ "theoretical_loss": 3.605933888025003,
+ "tokens_seen": 1237057536
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005052116349047142,
+ "loss": 1.4949,
+ "theoretical_loss": 3.6059165544210607,
+ "tokens_seen": 1237123072
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005051955867602809,
+ "loss": 1.2802,
+ "theoretical_loss": 3.6058992219924244,
+ "tokens_seen": 1237188608
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005051795386158476,
+ "loss": 1.3527,
+ "theoretical_loss": 3.605881890738953,
+ "tokens_seen": 1237254144
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005051634904714143,
+ "loss": 1.3396,
+ "theoretical_loss": 3.605864560660504,
+ "tokens_seen": 1237319680
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.000505147442326981,
+ "loss": 1.3599,
+ "theoretical_loss": 3.605847231756936,
+ "tokens_seen": 1237385216
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 0.0005051313941825477,
+ "loss": 1.4324,
+ "theoretical_loss": 3.6058299040281065,
+ "tokens_seen": 1237450752
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005051153460381144,
+ "loss": 1.3222,
+ "theoretical_loss": 3.6058125774738743,
+ "tokens_seen": 1237516288
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050992978936811,
+ "loss": 1.3546,
+ "theoretical_loss": 3.6057952520940972,
+ "tokens_seen": 1237581824
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050832497492478,
+ "loss": 1.3409,
+ "theoretical_loss": 3.6057779278886337,
+ "tokens_seen": 1237647360
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050672016048145,
+ "loss": 1.3855,
+ "theoretical_loss": 3.6057606048573416,
+ "tokens_seen": 1237712896
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050511534603812,
+ "loss": 1.2924,
+ "theoretical_loss": 3.6057432830000797,
+ "tokens_seen": 1237778432
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050351053159478,
+ "loss": 1.2803,
+ "theoretical_loss": 3.605725962316706,
+ "tokens_seen": 1237843968
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050190571715146,
+ "loss": 1.3581,
+ "theoretical_loss": 3.6057086428070786,
+ "tokens_seen": 1237909504
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005050030090270812,
+ "loss": 1.3183,
+ "theoretical_loss": 3.6056913244710556,
+ "tokens_seen": 1237975040
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504986960882648,
+ "loss": 1.3255,
+ "theoretical_loss": 3.6056740073084965,
+ "tokens_seen": 1238040576
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005049709127382146,
+ "loss": 1.512,
+ "theoretical_loss": 3.6056566913192585,
+ "tokens_seen": 1238106112
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005049548645937814,
+ "loss": 1.3427,
+ "theoretical_loss": 3.6056393765032007,
+ "tokens_seen": 1238171648
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504938816449348,
+ "loss": 1.3479,
+ "theoretical_loss": 3.605622062860181,
+ "tokens_seen": 1238237184
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005049227683049147,
+ "loss": 1.3233,
+ "theoretical_loss": 3.605604750390058,
+ "tokens_seen": 1238302720
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005049067201604814,
+ "loss": 1.3147,
+ "theoretical_loss": 3.6055874390926905,
+ "tokens_seen": 1238368256
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048906720160481,
+ "loss": 1.406,
+ "theoretical_loss": 3.605570128967937,
+ "tokens_seen": 1238433792
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048746238716148,
+ "loss": 1.4015,
+ "theoretical_loss": 3.6055528200156557,
+ "tokens_seen": 1238499328
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048585757271815,
+ "loss": 1.3123,
+ "theoretical_loss": 3.605535512235705,
+ "tokens_seen": 1238564864
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 416369,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4090187549591064,
+ "objective/train/theoretical_loss": 3.605518205627944,
+ "objective/train/tokens_used": 1259090400,
+ "theoretical_loss": 3.605518205627944,
+ "tokens_seen": 1238630400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048425275827482,
+ "loss": 1.4503,
+ "theoretical_loss": 3.605518205627944,
+ "tokens_seen": 1238630400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048264794383149,
+ "loss": 1.4952,
+ "theoretical_loss": 3.605500900192231,
+ "tokens_seen": 1238695936
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005048104312938816,
+ "loss": 1.3989,
+ "theoretical_loss": 3.6054835959284244,
+ "tokens_seen": 1238761472
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005047943831494483,
+ "loss": 1.289,
+ "theoretical_loss": 3.6054662928363834,
+ "tokens_seen": 1238827008
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504778335005015,
+ "loss": 1.2864,
+ "theoretical_loss": 3.605448990915966,
+ "tokens_seen": 1238892544
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005047622868605817,
+ "loss": 1.4136,
+ "theoretical_loss": 3.605431690167032,
+ "tokens_seen": 1238958080
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005047462387161484,
+ "loss": 1.3431,
+ "theoretical_loss": 3.6054143905894387,
+ "tokens_seen": 1239023616
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005047301905717151,
+ "loss": 1.4101,
+ "theoretical_loss": 3.605397092183046,
+ "tokens_seen": 1239089152
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005047141424272818,
+ "loss": 1.3925,
+ "theoretical_loss": 3.6053797949477118,
+ "tokens_seen": 1239154688
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046980942828485,
+ "loss": 1.2969,
+ "theoretical_loss": 3.605362498883296,
+ "tokens_seen": 1239220224
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046820461384152,
+ "loss": 1.3725,
+ "theoretical_loss": 3.605345203989656,
+ "tokens_seen": 1239285760
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046659979939819,
+ "loss": 1.2839,
+ "theoretical_loss": 3.6053279102666513,
+ "tokens_seen": 1239351296
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046499498495487,
+ "loss": 1.3753,
+ "theoretical_loss": 3.6053106177141414,
+ "tokens_seen": 1239416832
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046339017051153,
+ "loss": 1.3842,
+ "theoretical_loss": 3.605293326331984,
+ "tokens_seen": 1239482368
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504617853560682,
+ "loss": 1.3695,
+ "theoretical_loss": 3.6052760361200393,
+ "tokens_seen": 1239547904
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005046018054162487,
+ "loss": 1.2988,
+ "theoretical_loss": 3.6052587470781647,
+ "tokens_seen": 1239613440
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045857572718155,
+ "loss": 1.369,
+ "theoretical_loss": 3.605241459206221,
+ "tokens_seen": 1239678976
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045697091273821,
+ "loss": 1.4293,
+ "theoretical_loss": 3.605224172504066,
+ "tokens_seen": 1239744512
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045536609829489,
+ "loss": 1.3443,
+ "theoretical_loss": 3.6052068869715583,
+ "tokens_seen": 1239810048
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045376128385155,
+ "loss": 1.42,
+ "theoretical_loss": 3.605189602608558,
+ "tokens_seen": 1239875584
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045215646940823,
+ "loss": 1.3387,
+ "theoretical_loss": 3.605172319414924,
+ "tokens_seen": 1239941120
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005045055165496489,
+ "loss": 1.3442,
+ "theoretical_loss": 3.6051550373905146,
+ "tokens_seen": 1240006656
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044894684052157,
+ "loss": 1.2619,
+ "theoretical_loss": 3.60513775653519,
+ "tokens_seen": 1240072192
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044734202607824,
+ "loss": 1.3328,
+ "theoretical_loss": 3.6051204768488088,
+ "tokens_seen": 1240137728
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044573721163491,
+ "loss": 1.3232,
+ "theoretical_loss": 3.6051031983312294,
+ "tokens_seen": 1240203264
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044413239719158,
+ "loss": 1.4697,
+ "theoretical_loss": 3.6050859209823125,
+ "tokens_seen": 1240268800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044252758274825,
+ "loss": 1.3618,
+ "theoretical_loss": 3.605068644801916,
+ "tokens_seen": 1240334336
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005044092276830492,
+ "loss": 1.3062,
+ "theoretical_loss": 3.6050513697899,
+ "tokens_seen": 1240399872
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005043931795386159,
+ "loss": 1.3823,
+ "theoretical_loss": 3.6050340959461233,
+ "tokens_seen": 1240465408
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005043771313941826,
+ "loss": 1.3717,
+ "theoretical_loss": 3.6050168232704456,
+ "tokens_seen": 1240530944
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005043610832497493,
+ "loss": 1.4089,
+ "theoretical_loss": 3.6049995517627256,
+ "tokens_seen": 1240596480
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504345035105316,
+ "loss": 1.3627,
+ "theoretical_loss": 3.604982281422823,
+ "tokens_seen": 1240662016
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005043289869608827,
+ "loss": 1.2779,
+ "theoretical_loss": 3.604965012250597,
+ "tokens_seen": 1240727552
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005043129388164494,
+ "loss": 1.483,
+ "theoretical_loss": 3.6049477442459077,
+ "tokens_seen": 1240793088
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042968906720161,
+ "loss": 1.3465,
+ "theoretical_loss": 3.604930477408613,
+ "tokens_seen": 1240858624
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042808425275828,
+ "loss": 1.3855,
+ "theoretical_loss": 3.6049132117385736,
+ "tokens_seen": 1240924160
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042647943831495,
+ "loss": 1.2742,
+ "theoretical_loss": 3.604895947235649,
+ "tokens_seen": 1240989696
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042487462387162,
+ "loss": 1.3288,
+ "theoretical_loss": 3.604878683899698,
+ "tokens_seen": 1241055232
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042326980942829,
+ "loss": 1.3308,
+ "theoretical_loss": 3.60486142173058,
+ "tokens_seen": 1241120768
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042166499498496,
+ "loss": 1.4005,
+ "theoretical_loss": 3.604844160728155,
+ "tokens_seen": 1241186304
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005042006018054163,
+ "loss": 1.3795,
+ "theoretical_loss": 3.6048269008922826,
+ "tokens_seen": 1241251840
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000504184553660983,
+ "loss": 1.3427,
+ "theoretical_loss": 3.604809642222822,
+ "tokens_seen": 1241317376
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005041685055165497,
+ "loss": 1.2901,
+ "theoretical_loss": 3.6047923847196337,
+ "tokens_seen": 1241382912
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005041524573721164,
+ "loss": 1.3897,
+ "theoretical_loss": 3.6047751283825757,
+ "tokens_seen": 1241448448
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005041364092276831,
+ "loss": 1.3342,
+ "theoretical_loss": 3.6047578732115086,
+ "tokens_seen": 1241513984
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005041203610832498,
+ "loss": 1.3681,
+ "theoretical_loss": 3.604740619206293,
+ "tokens_seen": 1241579520
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005041043129388165,
+ "loss": 1.2255,
+ "theoretical_loss": 3.6047233663667866,
+ "tokens_seen": 1241645056
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005040882647943832,
+ "loss": 1.4839,
+ "theoretical_loss": 3.604706114692851,
+ "tokens_seen": 1241710592
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005040722166499499,
+ "loss": 1.2679,
+ "theoretical_loss": 3.604688864184345,
+ "tokens_seen": 1241776128
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005040561685055166,
+ "loss": 1.3852,
+ "theoretical_loss": 3.604671614841128,
+ "tokens_seen": 1241841664
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 417232,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1997463703155518,
+ "objective/train/theoretical_loss": 3.6046543666630604,
+ "objective/train/tokens_used": 1262367200,
+ "theoretical_loss": 3.6046543666630604,
+ "tokens_seen": 1241907200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005040401203610833,
+ "loss": 1.2561,
+ "theoretical_loss": 3.6046543666630604,
+ "tokens_seen": 1241907200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.00050402407221665,
+ "loss": 1.2476,
+ "theoretical_loss": 3.604637119650002,
+ "tokens_seen": 1241972736
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005040080240722167,
+ "loss": 1.3679,
+ "theoretical_loss": 3.6046198738018127,
+ "tokens_seen": 1242038272
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039919759277834,
+ "loss": 1.3728,
+ "theoretical_loss": 3.604602629118352,
+ "tokens_seen": 1242103808
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039759277833501,
+ "loss": 1.3712,
+ "theoretical_loss": 3.6045853855994796,
+ "tokens_seen": 1242169344
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039598796389168,
+ "loss": 1.4702,
+ "theoretical_loss": 3.6045681432450567,
+ "tokens_seen": 1242234880
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039438314944835,
+ "loss": 1.3163,
+ "theoretical_loss": 3.6045509020549416,
+ "tokens_seen": 1242300416
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039277833500502,
+ "loss": 1.3762,
+ "theoretical_loss": 3.6045336620289956,
+ "tokens_seen": 1242365952
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005039117352056169,
+ "loss": 1.3988,
+ "theoretical_loss": 3.6045164231670777,
+ "tokens_seen": 1242431488
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005038956870611836,
+ "loss": 1.3746,
+ "theoretical_loss": 3.6044991854690487,
+ "tokens_seen": 1242497024
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005038796389167503,
+ "loss": 1.3213,
+ "theoretical_loss": 3.604481948934768,
+ "tokens_seen": 1242562560
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503863590772317,
+ "loss": 1.3872,
+ "theoretical_loss": 3.6044647135640955,
+ "tokens_seen": 1242628096
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005038475426278837,
+ "loss": 1.1483,
+ "theoretical_loss": 3.6044474793568924,
+ "tokens_seen": 1242693632
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005038314944834504,
+ "loss": 1.2868,
+ "theoretical_loss": 3.6044302463130173,
+ "tokens_seen": 1242759168
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005038154463390171,
+ "loss": 1.3432,
+ "theoretical_loss": 3.604413014432332,
+ "tokens_seen": 1242824704
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037993981945838,
+ "loss": 1.3241,
+ "theoretical_loss": 3.604395783714695,
+ "tokens_seen": 1242890240
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037833500501505,
+ "loss": 1.3367,
+ "theoretical_loss": 3.6043785541599678,
+ "tokens_seen": 1242955776
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037673019057172,
+ "loss": 1.4916,
+ "theoretical_loss": 3.60436132576801,
+ "tokens_seen": 1243021312
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037512537612839,
+ "loss": 1.2502,
+ "theoretical_loss": 3.6043440985386814,
+ "tokens_seen": 1243086848
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037352056168506,
+ "loss": 1.338,
+ "theoretical_loss": 3.6043268724718436,
+ "tokens_seen": 1243152384
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005037191574724173,
+ "loss": 1.3496,
+ "theoretical_loss": 3.604309647567356,
+ "tokens_seen": 1243217920
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503703109327984,
+ "loss": 1.2946,
+ "theoretical_loss": 3.604292423825078,
+ "tokens_seen": 1243283456
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036870611835507,
+ "loss": 1.3432,
+ "theoretical_loss": 3.6042752012448718,
+ "tokens_seen": 1243348992
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036710130391174,
+ "loss": 1.1825,
+ "theoretical_loss": 3.6042579798265963,
+ "tokens_seen": 1243414528
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036549648946841,
+ "loss": 1.4187,
+ "theoretical_loss": 3.6042407595701125,
+ "tokens_seen": 1243480064
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036389167502507,
+ "loss": 1.3748,
+ "theoretical_loss": 3.6042235404752807,
+ "tokens_seen": 1243545600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036228686058175,
+ "loss": 1.3473,
+ "theoretical_loss": 3.6042063225419616,
+ "tokens_seen": 1243611136
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005036068204613841,
+ "loss": 1.3553,
+ "theoretical_loss": 3.604189105770015,
+ "tokens_seen": 1243676672
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035907723169509,
+ "loss": 1.3541,
+ "theoretical_loss": 3.604171890159302,
+ "tokens_seen": 1243742208
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035747241725175,
+ "loss": 1.4371,
+ "theoretical_loss": 3.6041546757096823,
+ "tokens_seen": 1243807744
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035586760280843,
+ "loss": 1.4177,
+ "theoretical_loss": 3.6041374624210176,
+ "tokens_seen": 1243873280
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035426278836509,
+ "loss": 1.4302,
+ "theoretical_loss": 3.6041202502931675,
+ "tokens_seen": 1243938816
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035265797392177,
+ "loss": 1.4668,
+ "theoretical_loss": 3.6041030393259925,
+ "tokens_seen": 1244004352
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005035105315947843,
+ "loss": 1.3746,
+ "theoretical_loss": 3.604085829519354,
+ "tokens_seen": 1244069888
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503494483450351,
+ "loss": 1.3797,
+ "theoretical_loss": 3.604068620873112,
+ "tokens_seen": 1244135424
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005034784353059177,
+ "loss": 1.3403,
+ "theoretical_loss": 3.6040514133871273,
+ "tokens_seen": 1244200960
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005034623871614844,
+ "loss": 1.3468,
+ "theoretical_loss": 3.604034207061261,
+ "tokens_seen": 1244266496
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005034463390170511,
+ "loss": 1.354,
+ "theoretical_loss": 3.6040170018953726,
+ "tokens_seen": 1244332032
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005034302908726178,
+ "loss": 1.3894,
+ "theoretical_loss": 3.603999797889324,
+ "tokens_seen": 1244397568
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005034142427281845,
+ "loss": 1.4053,
+ "theoretical_loss": 3.6039825950429756,
+ "tokens_seen": 1244463104
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005033981945837512,
+ "loss": 1.3635,
+ "theoretical_loss": 3.6039653933561877,
+ "tokens_seen": 1244528640
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503382146439318,
+ "loss": 1.3767,
+ "theoretical_loss": 3.6039481928288217,
+ "tokens_seen": 1244594176
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005033660982948846,
+ "loss": 1.3488,
+ "theoretical_loss": 3.603930993460738,
+ "tokens_seen": 1244659712
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005033500501504513,
+ "loss": 1.2356,
+ "theoretical_loss": 3.603913795251798,
+ "tokens_seen": 1244725248
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503334002006018,
+ "loss": 1.407,
+ "theoretical_loss": 3.603896598201862,
+ "tokens_seen": 1244790784
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005033179538615847,
+ "loss": 1.3197,
+ "theoretical_loss": 3.6038794023107905,
+ "tokens_seen": 1244856320
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005033019057171515,
+ "loss": 1.3136,
+ "theoretical_loss": 3.6038622075784454,
+ "tokens_seen": 1244921856
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005032858575727182,
+ "loss": 1.3627,
+ "theoretical_loss": 3.603845014004687,
+ "tokens_seen": 1244987392
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005032698094282849,
+ "loss": 1.3836,
+ "theoretical_loss": 3.6038278215893764,
+ "tokens_seen": 1245052928
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005032537612838516,
+ "loss": 1.3111,
+ "theoretical_loss": 3.603810630332375,
+ "tokens_seen": 1245118464
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 418442,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4537241458892822,
+ "objective/train/theoretical_loss": 3.6037934402335425,
+ "objective/train/tokens_used": 1265644000,
+ "theoretical_loss": 3.6037934402335425,
+ "tokens_seen": 1245184000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005032377131394183,
+ "loss": 1.3637,
+ "theoretical_loss": 3.6037934402335425,
+ "tokens_seen": 1245184000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503221664994985,
+ "loss": 1.4316,
+ "theoretical_loss": 3.6037762512927416,
+ "tokens_seen": 1245249536
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005032056168505517,
+ "loss": 1.2916,
+ "theoretical_loss": 3.603759063509832,
+ "tokens_seen": 1245315072
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031895687061184,
+ "loss": 1.3194,
+ "theoretical_loss": 3.603741876884676,
+ "tokens_seen": 1245380608
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031735205616851,
+ "loss": 1.3862,
+ "theoretical_loss": 3.6037246914171335,
+ "tokens_seen": 1245446144
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031574724172518,
+ "loss": 1.2677,
+ "theoretical_loss": 3.6037075071070666,
+ "tokens_seen": 1245511680
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031414242728185,
+ "loss": 1.333,
+ "theoretical_loss": 3.6036903239543356,
+ "tokens_seen": 1245577216
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031253761283852,
+ "loss": 1.2588,
+ "theoretical_loss": 3.6036731419588017,
+ "tokens_seen": 1245642752
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005031093279839519,
+ "loss": 1.308,
+ "theoretical_loss": 3.603655961120327,
+ "tokens_seen": 1245708288
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005030932798395186,
+ "loss": 1.3684,
+ "theoretical_loss": 3.6036387814387725,
+ "tokens_seen": 1245773824
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005030772316950853,
+ "loss": 1.362,
+ "theoretical_loss": 3.6036216029139982,
+ "tokens_seen": 1245839360
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000503061183550652,
+ "loss": 1.2078,
+ "theoretical_loss": 3.603604425545867,
+ "tokens_seen": 1245904896
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005030451354062187,
+ "loss": 1.3836,
+ "theoretical_loss": 3.603587249334239,
+ "tokens_seen": 1245970432
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005030290872617854,
+ "loss": 1.3458,
+ "theoretical_loss": 3.6035700742789762,
+ "tokens_seen": 1246035968
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005030130391173521,
+ "loss": 1.3773,
+ "theoretical_loss": 3.60355290037994,
+ "tokens_seen": 1246101504
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029969909729188,
+ "loss": 1.4511,
+ "theoretical_loss": 3.6035357276369906,
+ "tokens_seen": 1246167040
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029809428284855,
+ "loss": 1.3902,
+ "theoretical_loss": 3.603518556049991,
+ "tokens_seen": 1246232576
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029648946840522,
+ "loss": 1.3707,
+ "theoretical_loss": 3.603501385618802,
+ "tokens_seen": 1246298112
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029488465396189,
+ "loss": 1.3224,
+ "theoretical_loss": 3.6034842163432845,
+ "tokens_seen": 1246363648
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029327983951856,
+ "loss": 1.3062,
+ "theoretical_loss": 3.6034670482233,
+ "tokens_seen": 1246429184
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005029167502507523,
+ "loss": 1.3808,
+ "theoretical_loss": 3.603449881258711,
+ "tokens_seen": 1246494720
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502900702106319,
+ "loss": 1.4469,
+ "theoretical_loss": 3.6034327154493777,
+ "tokens_seen": 1246560256
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028846539618857,
+ "loss": 1.3295,
+ "theoretical_loss": 3.603415550795163,
+ "tokens_seen": 1246625792
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028686058174524,
+ "loss": 1.3878,
+ "theoretical_loss": 3.603398387295927,
+ "tokens_seen": 1246691328
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028525576730191,
+ "loss": 1.3918,
+ "theoretical_loss": 3.6033812249515327,
+ "tokens_seen": 1246756864
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028365095285858,
+ "loss": 1.4321,
+ "theoretical_loss": 3.6033640637618403,
+ "tokens_seen": 1246822400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028204613841525,
+ "loss": 1.3718,
+ "theoretical_loss": 3.6033469037267123,
+ "tokens_seen": 1246887936
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005028044132397192,
+ "loss": 1.3938,
+ "theoretical_loss": 3.60332974484601,
+ "tokens_seen": 1246953472
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005027883650952859,
+ "loss": 1.3004,
+ "theoretical_loss": 3.6033125871195955,
+ "tokens_seen": 1247019008
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005027723169508526,
+ "loss": 1.2841,
+ "theoretical_loss": 3.6032954305473304,
+ "tokens_seen": 1247084544
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005027562688064193,
+ "loss": 1.376,
+ "theoretical_loss": 3.6032782751290755,
+ "tokens_seen": 1247150080
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502740220661986,
+ "loss": 1.3305,
+ "theoretical_loss": 3.603261120864694,
+ "tokens_seen": 1247215616
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005027241725175527,
+ "loss": 1.2501,
+ "theoretical_loss": 3.6032439677540467,
+ "tokens_seen": 1247281152
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005027081243731194,
+ "loss": 1.348,
+ "theoretical_loss": 3.6032268157969956,
+ "tokens_seen": 1247346688
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026920762286861,
+ "loss": 1.2983,
+ "theoretical_loss": 3.6032096649934022,
+ "tokens_seen": 1247412224
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026760280842528,
+ "loss": 1.2707,
+ "theoretical_loss": 3.603192515343129,
+ "tokens_seen": 1247477760
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026599799398195,
+ "loss": 1.2921,
+ "theoretical_loss": 3.6031753668460373,
+ "tokens_seen": 1247543296
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026439317953862,
+ "loss": 1.2688,
+ "theoretical_loss": 3.6031582195019896,
+ "tokens_seen": 1247608832
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026278836509529,
+ "loss": 1.3406,
+ "theoretical_loss": 3.6031410733108475,
+ "tokens_seen": 1247674368
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005026118355065196,
+ "loss": 1.2678,
+ "theoretical_loss": 3.6031239282724723,
+ "tokens_seen": 1247739904
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005025957873620863,
+ "loss": 1.4064,
+ "theoretical_loss": 3.603106784386727,
+ "tokens_seen": 1247805440
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502579739217653,
+ "loss": 1.2971,
+ "theoretical_loss": 3.6030896416534732,
+ "tokens_seen": 1247870976
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005025636910732197,
+ "loss": 1.3555,
+ "theoretical_loss": 3.6030725000725723,
+ "tokens_seen": 1247936512
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005025476429287864,
+ "loss": 1.3105,
+ "theoretical_loss": 3.6030553596438875,
+ "tokens_seen": 1248002048
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005025315947843531,
+ "loss": 1.4003,
+ "theoretical_loss": 3.6030382203672793,
+ "tokens_seen": 1248067584
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005025155466399198,
+ "loss": 1.2548,
+ "theoretical_loss": 3.603021082242611,
+ "tokens_seen": 1248133120
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024994984954865,
+ "loss": 1.3411,
+ "theoretical_loss": 3.6030039452697444,
+ "tokens_seen": 1248198656
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024834503510532,
+ "loss": 1.3666,
+ "theoretical_loss": 3.6029868094485415,
+ "tokens_seen": 1248264192
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024674022066199,
+ "loss": 1.338,
+ "theoretical_loss": 3.6029696747788647,
+ "tokens_seen": 1248329728
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024513540621866,
+ "loss": 1.2971,
+ "theoretical_loss": 3.602952541260576,
+ "tokens_seen": 1248395264
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 419258,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.282519817352295,
+ "objective/train/theoretical_loss": 3.6029354088935373,
+ "objective/train/tokens_used": 1268920800,
+ "theoretical_loss": 3.6029354088935373,
+ "tokens_seen": 1248460800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024353059177533,
+ "loss": 1.4272,
+ "theoretical_loss": 3.6029354088935373,
+ "tokens_seen": 1248460800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.00050241925777332,
+ "loss": 1.212,
+ "theoretical_loss": 3.6029182776776114,
+ "tokens_seen": 1248526336
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005024032096288867,
+ "loss": 1.3939,
+ "theoretical_loss": 3.60290114761266,
+ "tokens_seen": 1248591872
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023871614844534,
+ "loss": 1.3397,
+ "theoretical_loss": 3.6028840186985462,
+ "tokens_seen": 1248657408
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023711133400201,
+ "loss": 1.4012,
+ "theoretical_loss": 3.6028668909351307,
+ "tokens_seen": 1248722944
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023550651955868,
+ "loss": 1.3113,
+ "theoretical_loss": 3.6028497643222774,
+ "tokens_seen": 1248788480
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023390170511535,
+ "loss": 1.2558,
+ "theoretical_loss": 3.602832638859848,
+ "tokens_seen": 1248854016
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023229689067202,
+ "loss": 1.1927,
+ "theoretical_loss": 3.6028155145477054,
+ "tokens_seen": 1248919552
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005023069207622869,
+ "loss": 1.2656,
+ "theoretical_loss": 3.602798391385711,
+ "tokens_seen": 1248985088
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005022908726178536,
+ "loss": 1.3359,
+ "theoretical_loss": 3.602781269373727,
+ "tokens_seen": 1249050624
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005022748244734203,
+ "loss": 1.3055,
+ "theoretical_loss": 3.6027641485116177,
+ "tokens_seen": 1249116160
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502258776328987,
+ "loss": 1.2639,
+ "theoretical_loss": 3.602747028799244,
+ "tokens_seen": 1249181696
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005022427281845536,
+ "loss": 1.5219,
+ "theoretical_loss": 3.6027299102364685,
+ "tokens_seen": 1249247232
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005022266800401204,
+ "loss": 1.2866,
+ "theoretical_loss": 3.6027127928231537,
+ "tokens_seen": 1249312768
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502210631895687,
+ "loss": 1.2317,
+ "theoretical_loss": 3.602695676559163,
+ "tokens_seen": 1249378304
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021945837512538,
+ "loss": 1.285,
+ "theoretical_loss": 3.6026785614443577,
+ "tokens_seen": 1249443840
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021785356068204,
+ "loss": 1.251,
+ "theoretical_loss": 3.6026614474786016,
+ "tokens_seen": 1249509376
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021624874623872,
+ "loss": 1.3303,
+ "theoretical_loss": 3.6026443346617567,
+ "tokens_seen": 1249574912
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021464393179538,
+ "loss": 1.438,
+ "theoretical_loss": 3.602627222993685,
+ "tokens_seen": 1249640448
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021303911735206,
+ "loss": 1.4266,
+ "theoretical_loss": 3.6026101124742502,
+ "tokens_seen": 1249705984
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005021143430290872,
+ "loss": 1.4137,
+ "theoretical_loss": 3.6025930031033147,
+ "tokens_seen": 1249771520
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502098294884654,
+ "loss": 1.3768,
+ "theoretical_loss": 3.602575894880741,
+ "tokens_seen": 1249837056
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005020822467402206,
+ "loss": 1.3509,
+ "theoretical_loss": 3.602558787806391,
+ "tokens_seen": 1249902592
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005020661985957874,
+ "loss": 1.3474,
+ "theoretical_loss": 3.602541681880129,
+ "tokens_seen": 1249968128
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000502050150451354,
+ "loss": 1.2582,
+ "theoretical_loss": 3.602524577101817,
+ "tokens_seen": 1250033664
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005020341023069208,
+ "loss": 1.353,
+ "theoretical_loss": 3.602507473471318,
+ "tokens_seen": 1250099200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005020180541624874,
+ "loss": 1.3492,
+ "theoretical_loss": 3.602490370988494,
+ "tokens_seen": 1250164736
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005020020060180542,
+ "loss": 1.294,
+ "theoretical_loss": 3.602473269653209,
+ "tokens_seen": 1250230272
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005019859578736208,
+ "loss": 1.365,
+ "theoretical_loss": 3.602456169465325,
+ "tokens_seen": 1250295808
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005019699097291876,
+ "loss": 1.4377,
+ "theoretical_loss": 3.6024390704247056,
+ "tokens_seen": 1250361344
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005019538615847542,
+ "loss": 1.28,
+ "theoretical_loss": 3.6024219725312125,
+ "tokens_seen": 1250426880
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501937813440321,
+ "loss": 1.324,
+ "theoretical_loss": 3.6024048757847105,
+ "tokens_seen": 1250492416
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005019217652958877,
+ "loss": 1.2918,
+ "theoretical_loss": 3.6023877801850603,
+ "tokens_seen": 1250557952
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005019057171514544,
+ "loss": 1.3186,
+ "theoretical_loss": 3.6023706857321267,
+ "tokens_seen": 1250623488
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501889669007021,
+ "loss": 1.4421,
+ "theoretical_loss": 3.6023535924257724,
+ "tokens_seen": 1250689024
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005018736208625878,
+ "loss": 1.3071,
+ "theoretical_loss": 3.602336500265859,
+ "tokens_seen": 1250754560
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005018575727181545,
+ "loss": 1.4807,
+ "theoretical_loss": 3.6023194092522517,
+ "tokens_seen": 1250820096
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005018415245737212,
+ "loss": 1.3308,
+ "theoretical_loss": 3.602302319384812,
+ "tokens_seen": 1250885632
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005018254764292879,
+ "loss": 1.4066,
+ "theoretical_loss": 3.6022852306634032,
+ "tokens_seen": 1250951168
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005018094282848546,
+ "loss": 1.3101,
+ "theoretical_loss": 3.602268143087889,
+ "tokens_seen": 1251016704
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005017933801404213,
+ "loss": 1.2965,
+ "theoretical_loss": 3.6022510566581323,
+ "tokens_seen": 1251082240
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501777331995988,
+ "loss": 1.4012,
+ "theoretical_loss": 3.6022339713739964,
+ "tokens_seen": 1251147776
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005017612838515547,
+ "loss": 1.3074,
+ "theoretical_loss": 3.6022168872353437,
+ "tokens_seen": 1251213312
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005017452357071214,
+ "loss": 1.2211,
+ "theoretical_loss": 3.602199804242038,
+ "tokens_seen": 1251278848
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005017291875626881,
+ "loss": 1.3246,
+ "theoretical_loss": 3.602182722393943,
+ "tokens_seen": 1251344384
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005017131394182548,
+ "loss": 1.3118,
+ "theoretical_loss": 3.602165641690921,
+ "tokens_seen": 1251409920
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016970912738215,
+ "loss": 1.2714,
+ "theoretical_loss": 3.602148562132836,
+ "tokens_seen": 1251475456
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016810431293882,
+ "loss": 1.2469,
+ "theoretical_loss": 3.6021314837195515,
+ "tokens_seen": 1251540992
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016649949849549,
+ "loss": 1.383,
+ "theoretical_loss": 3.6021144064509296,
+ "tokens_seen": 1251606528
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016489468405216,
+ "loss": 1.3218,
+ "theoretical_loss": 3.6020973303268344,
+ "tokens_seen": 1251672064
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 420558,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4131748676300049,
+ "objective/train/theoretical_loss": 3.6020802553471296,
+ "objective/train/tokens_used": 1272197600,
+ "theoretical_loss": 3.6020802553471296,
+ "tokens_seen": 1251737600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016328986960883,
+ "loss": 1.3026,
+ "theoretical_loss": 3.6020802553471296,
+ "tokens_seen": 1251737600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501616850551655,
+ "loss": 1.3667,
+ "theoretical_loss": 3.6020631815116784,
+ "tokens_seen": 1251803136
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005016008024072217,
+ "loss": 1.4029,
+ "theoretical_loss": 3.602046108820344,
+ "tokens_seen": 1251868672
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015847542627884,
+ "loss": 1.2877,
+ "theoretical_loss": 3.60202903727299,
+ "tokens_seen": 1251934208
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015687061183551,
+ "loss": 1.3506,
+ "theoretical_loss": 3.6020119668694797,
+ "tokens_seen": 1251999744
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015526579739218,
+ "loss": 1.3731,
+ "theoretical_loss": 3.6019948976096767,
+ "tokens_seen": 1252065280
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015366098294885,
+ "loss": 1.3478,
+ "theoretical_loss": 3.6019778294934444,
+ "tokens_seen": 1252130816
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015205616850552,
+ "loss": 1.2647,
+ "theoretical_loss": 3.6019607625206467,
+ "tokens_seen": 1252196352
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005015045135406219,
+ "loss": 1.2592,
+ "theoretical_loss": 3.601943696691147,
+ "tokens_seen": 1252261888
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005014884653961886,
+ "loss": 1.3061,
+ "theoretical_loss": 3.601926632004809,
+ "tokens_seen": 1252327424
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005014724172517553,
+ "loss": 1.2887,
+ "theoretical_loss": 3.6019095684614957,
+ "tokens_seen": 1252392960
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501456369107322,
+ "loss": 1.3371,
+ "theoretical_loss": 3.601892506061071,
+ "tokens_seen": 1252458496
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005014403209628887,
+ "loss": 1.395,
+ "theoretical_loss": 3.601875444803399,
+ "tokens_seen": 1252524032
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005014242728184554,
+ "loss": 1.3225,
+ "theoretical_loss": 3.6018583846883434,
+ "tokens_seen": 1252589568
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005014082246740221,
+ "loss": 1.5075,
+ "theoretical_loss": 3.601841325715767,
+ "tokens_seen": 1252655104
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013921765295888,
+ "loss": 1.4083,
+ "theoretical_loss": 3.601824267885535,
+ "tokens_seen": 1252720640
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013761283851555,
+ "loss": 1.3404,
+ "theoretical_loss": 3.6018072111975092,
+ "tokens_seen": 1252786176
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013600802407222,
+ "loss": 1.4084,
+ "theoretical_loss": 3.6017901556515546,
+ "tokens_seen": 1252851712
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013440320962889,
+ "loss": 1.3497,
+ "theoretical_loss": 3.6017731012475354,
+ "tokens_seen": 1252917248
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013279839518556,
+ "loss": 1.3632,
+ "theoretical_loss": 3.6017560479853143,
+ "tokens_seen": 1252982784
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005013119358074223,
+ "loss": 1.3007,
+ "theoretical_loss": 3.601738995864756,
+ "tokens_seen": 1253048320
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501295887662989,
+ "loss": 1.4,
+ "theoretical_loss": 3.601721944885724,
+ "tokens_seen": 1253113856
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005012798395185557,
+ "loss": 1.2713,
+ "theoretical_loss": 3.6017048950480817,
+ "tokens_seen": 1253179392
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005012637913741224,
+ "loss": 1.4096,
+ "theoretical_loss": 3.601687846351694,
+ "tokens_seen": 1253244928
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005012477432296891,
+ "loss": 1.1946,
+ "theoretical_loss": 3.6016707987964245,
+ "tokens_seen": 1253310464
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005012316950852558,
+ "loss": 1.2877,
+ "theoretical_loss": 3.601653752382137,
+ "tokens_seen": 1253376000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005012156469408225,
+ "loss": 1.3249,
+ "theoretical_loss": 3.601636707108695,
+ "tokens_seen": 1253441536
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011995987963892,
+ "loss": 1.245,
+ "theoretical_loss": 3.601619662975964,
+ "tokens_seen": 1253507072
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011835506519559,
+ "loss": 1.4088,
+ "theoretical_loss": 3.6016026199838063,
+ "tokens_seen": 1253572608
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011675025075226,
+ "loss": 1.4695,
+ "theoretical_loss": 3.601585578132087,
+ "tokens_seen": 1253638144
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011514543630893,
+ "loss": 1.3517,
+ "theoretical_loss": 3.6015685374206696,
+ "tokens_seen": 1253703680
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000501135406218656,
+ "loss": 1.3793,
+ "theoretical_loss": 3.601551497849419,
+ "tokens_seen": 1253769216
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011193580742227,
+ "loss": 1.4154,
+ "theoretical_loss": 3.6015344594181986,
+ "tokens_seen": 1253834752
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005011033099297894,
+ "loss": 1.3332,
+ "theoretical_loss": 3.6015174221268724,
+ "tokens_seen": 1253900288
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010872617853561,
+ "loss": 1.4641,
+ "theoretical_loss": 3.601500385975305,
+ "tokens_seen": 1253965824
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010712136409228,
+ "loss": 1.3193,
+ "theoretical_loss": 3.6014833509633606,
+ "tokens_seen": 1254031360
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010551654964895,
+ "loss": 1.308,
+ "theoretical_loss": 3.6014663170909036,
+ "tokens_seen": 1254096896
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010391173520562,
+ "loss": 1.4253,
+ "theoretical_loss": 3.6014492843577974,
+ "tokens_seen": 1254162432
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010230692076229,
+ "loss": 1.394,
+ "theoretical_loss": 3.6014322527639075,
+ "tokens_seen": 1254227968
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005010070210631896,
+ "loss": 1.3337,
+ "theoretical_loss": 3.6014152223090967,
+ "tokens_seen": 1254293504
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005009909729187563,
+ "loss": 1.4764,
+ "theoretical_loss": 3.601398192993231,
+ "tokens_seen": 1254359040
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500974924774323,
+ "loss": 1.2114,
+ "theoretical_loss": 3.601381164816173,
+ "tokens_seen": 1254424576
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005009588766298897,
+ "loss": 1.2906,
+ "theoretical_loss": 3.6013641377777885,
+ "tokens_seen": 1254490112
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005009428284854564,
+ "loss": 1.3512,
+ "theoretical_loss": 3.601347111877941,
+ "tokens_seen": 1254555648
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005009267803410231,
+ "loss": 1.3272,
+ "theoretical_loss": 3.6013300871164953,
+ "tokens_seen": 1254621184
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005009107321965898,
+ "loss": 1.3745,
+ "theoretical_loss": 3.6013130634933157,
+ "tokens_seen": 1254686720
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008946840521565,
+ "loss": 1.3642,
+ "theoretical_loss": 3.6012960410082666,
+ "tokens_seen": 1254752256
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008786359077232,
+ "loss": 1.3109,
+ "theoretical_loss": 3.601279019661213,
+ "tokens_seen": 1254817792
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008625877632899,
+ "loss": 1.3514,
+ "theoretical_loss": 3.601261999452018,
+ "tokens_seen": 1254883328
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008465396188565,
+ "loss": 1.2845,
+ "theoretical_loss": 3.6012449803805477,
+ "tokens_seen": 1254948864
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 421519,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.402679443359375,
+ "objective/train/theoretical_loss": 3.601227962446665,
+ "objective/train/tokens_used": 1275474400,
+ "theoretical_loss": 3.601227962446665,
+ "tokens_seen": 1255014400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008304914744233,
+ "loss": 1.3367,
+ "theoretical_loss": 3.601227962446665,
+ "tokens_seen": 1255014400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005008144433299899,
+ "loss": 1.3501,
+ "theoretical_loss": 3.6012109456502364,
+ "tokens_seen": 1255079936
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007983951855567,
+ "loss": 1.3978,
+ "theoretical_loss": 3.6011939299911253,
+ "tokens_seen": 1255145472
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007823470411233,
+ "loss": 1.3117,
+ "theoretical_loss": 3.601176915469196,
+ "tokens_seen": 1255211008
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007662988966901,
+ "loss": 1.2574,
+ "theoretical_loss": 3.601159902084315,
+ "tokens_seen": 1255276544
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007502507522567,
+ "loss": 1.2656,
+ "theoretical_loss": 3.6011428898363445,
+ "tokens_seen": 1255342080
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007342026078235,
+ "loss": 1.3554,
+ "theoretical_loss": 3.60112587872515,
+ "tokens_seen": 1255407616
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007181544633901,
+ "loss": 1.3155,
+ "theoretical_loss": 3.6011088687505977,
+ "tokens_seen": 1255473152
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005007021063189569,
+ "loss": 1.3541,
+ "theoretical_loss": 3.6010918599125503,
+ "tokens_seen": 1255538688
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005006860581745235,
+ "loss": 1.3029,
+ "theoretical_loss": 3.601074852210873,
+ "tokens_seen": 1255604224
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005006700100300903,
+ "loss": 1.3996,
+ "theoretical_loss": 3.601057845645432,
+ "tokens_seen": 1255669760
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500653961885657,
+ "loss": 1.2697,
+ "theoretical_loss": 3.6010408402160907,
+ "tokens_seen": 1255735296
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005006379137412237,
+ "loss": 1.3648,
+ "theoretical_loss": 3.601023835922714,
+ "tokens_seen": 1255800832
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005006218655967903,
+ "loss": 1.4372,
+ "theoretical_loss": 3.601006832765167,
+ "tokens_seen": 1255866368
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005006058174523571,
+ "loss": 1.331,
+ "theoretical_loss": 3.6009898307433144,
+ "tokens_seen": 1255931904
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005897693079237,
+ "loss": 1.345,
+ "theoretical_loss": 3.6009728298570214,
+ "tokens_seen": 1255997440
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005737211634905,
+ "loss": 1.2894,
+ "theoretical_loss": 3.6009558301061526,
+ "tokens_seen": 1256062976
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005576730190571,
+ "loss": 1.3336,
+ "theoretical_loss": 3.6009388314905735,
+ "tokens_seen": 1256128512
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005416248746239,
+ "loss": 1.1996,
+ "theoretical_loss": 3.6009218340101485,
+ "tokens_seen": 1256194048
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005255767301906,
+ "loss": 1.3561,
+ "theoretical_loss": 3.600904837664743,
+ "tokens_seen": 1256259584
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005005095285857573,
+ "loss": 1.3176,
+ "theoretical_loss": 3.6008878424542208,
+ "tokens_seen": 1256325120
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500493480441324,
+ "loss": 1.3492,
+ "theoretical_loss": 3.600870848378449,
+ "tokens_seen": 1256390656
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005004774322968907,
+ "loss": 1.2025,
+ "theoretical_loss": 3.6008538554372906,
+ "tokens_seen": 1256456192
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005004613841524574,
+ "loss": 1.2839,
+ "theoretical_loss": 3.6008368636306116,
+ "tokens_seen": 1256521728
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005004453360080241,
+ "loss": 1.2362,
+ "theoretical_loss": 3.6008198729582777,
+ "tokens_seen": 1256587264
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005004292878635908,
+ "loss": 1.2907,
+ "theoretical_loss": 3.600802883420153,
+ "tokens_seen": 1256652800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005004132397191575,
+ "loss": 1.3262,
+ "theoretical_loss": 3.600785895016103,
+ "tokens_seen": 1256718336
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003971915747242,
+ "loss": 1.2668,
+ "theoretical_loss": 3.600768907745993,
+ "tokens_seen": 1256783872
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003811434302909,
+ "loss": 1.27,
+ "theoretical_loss": 3.6007519216096884,
+ "tokens_seen": 1256849408
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003650952858576,
+ "loss": 1.3873,
+ "theoretical_loss": 3.6007349366070533,
+ "tokens_seen": 1256914944
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003490471414243,
+ "loss": 1.266,
+ "theoretical_loss": 3.6007179527379547,
+ "tokens_seen": 1256980480
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500332998996991,
+ "loss": 1.3877,
+ "theoretical_loss": 3.600700970002256,
+ "tokens_seen": 1257046016
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003169508525577,
+ "loss": 1.2647,
+ "theoretical_loss": 3.6006839883998243,
+ "tokens_seen": 1257111552
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005003009027081244,
+ "loss": 1.3241,
+ "theoretical_loss": 3.6006670079305234,
+ "tokens_seen": 1257177088
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002848545636911,
+ "loss": 1.2356,
+ "theoretical_loss": 3.6006500285942193,
+ "tokens_seen": 1257242624
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002688064192578,
+ "loss": 1.3316,
+ "theoretical_loss": 3.600633050390777,
+ "tokens_seen": 1257308160
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002527582748245,
+ "loss": 1.3431,
+ "theoretical_loss": 3.6006160733200625,
+ "tokens_seen": 1257373696
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002367101303912,
+ "loss": 1.3099,
+ "theoretical_loss": 3.6005990973819406,
+ "tokens_seen": 1257439232
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002206619859579,
+ "loss": 1.3399,
+ "theoretical_loss": 3.6005821225762764,
+ "tokens_seen": 1257504768
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005002046138415246,
+ "loss": 1.387,
+ "theoretical_loss": 3.6005651489029367,
+ "tokens_seen": 1257570304
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005001885656970913,
+ "loss": 1.3326,
+ "theoretical_loss": 3.600548176361786,
+ "tokens_seen": 1257635840
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500172517552658,
+ "loss": 1.2592,
+ "theoretical_loss": 3.6005312049526896,
+ "tokens_seen": 1257701376
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005001564694082247,
+ "loss": 1.3732,
+ "theoretical_loss": 3.6005142346755132,
+ "tokens_seen": 1257766912
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005001404212637914,
+ "loss": 1.2656,
+ "theoretical_loss": 3.600497265530123,
+ "tokens_seen": 1257832448
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005001243731193581,
+ "loss": 1.2732,
+ "theoretical_loss": 3.600480297516384,
+ "tokens_seen": 1257897984
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005001083249749248,
+ "loss": 1.3049,
+ "theoretical_loss": 3.6004633306341614,
+ "tokens_seen": 1257963520
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005000922768304915,
+ "loss": 1.4004,
+ "theoretical_loss": 3.6004463648833216,
+ "tokens_seen": 1258029056
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005000762286860582,
+ "loss": 1.3171,
+ "theoretical_loss": 3.6004294002637294,
+ "tokens_seen": 1258094592
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005000601805416249,
+ "loss": 1.2923,
+ "theoretical_loss": 3.600412436775251,
+ "tokens_seen": 1258160128
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005000441323971916,
+ "loss": 1.4036,
+ "theoretical_loss": 3.600395474417752,
+ "tokens_seen": 1258225664
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 422629,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2424266338348389,
+ "objective/train/theoretical_loss": 3.600378513191098,
+ "objective/train/tokens_used": 1278751200,
+ "theoretical_loss": 3.600378513191098,
+ "tokens_seen": 1258291200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0005000280842527583,
+ "loss": 1.3259,
+ "theoretical_loss": 3.600378513191098,
+ "tokens_seen": 1258291200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000500012036108325,
+ "loss": 1.3379,
+ "theoretical_loss": 3.600361553095155,
+ "tokens_seen": 1258356736
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999959879638917,
+ "loss": 1.4047,
+ "theoretical_loss": 3.6003445941297882,
+ "tokens_seen": 1258422272
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999799398194584,
+ "loss": 1.2773,
+ "theoretical_loss": 3.600327636294864,
+ "tokens_seen": 1258487808
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999638916750251,
+ "loss": 1.3321,
+ "theoretical_loss": 3.6003106795902475,
+ "tokens_seen": 1258553344
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999478435305918,
+ "loss": 1.3351,
+ "theoretical_loss": 3.600293724015805,
+ "tokens_seen": 1258618880
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999317953861585,
+ "loss": 1.3151,
+ "theoretical_loss": 3.600276769571402,
+ "tokens_seen": 1258684416
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004999157472417252,
+ "loss": 1.2681,
+ "theoretical_loss": 3.6002598162569046,
+ "tokens_seen": 1258749952
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998996990972919,
+ "loss": 1.2733,
+ "theoretical_loss": 3.600242864072179,
+ "tokens_seen": 1258815488
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998836509528586,
+ "loss": 1.239,
+ "theoretical_loss": 3.6002259130170904,
+ "tokens_seen": 1258881024
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998676028084253,
+ "loss": 1.3811,
+ "theoretical_loss": 3.6002089630915046,
+ "tokens_seen": 1258946560
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000499851554663992,
+ "loss": 1.2456,
+ "theoretical_loss": 3.6001920142952883,
+ "tokens_seen": 1259012096
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998355065195587,
+ "loss": 1.35,
+ "theoretical_loss": 3.6001750666283074,
+ "tokens_seen": 1259077632
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998194583751254,
+ "loss": 1.2933,
+ "theoretical_loss": 3.600158120090428,
+ "tokens_seen": 1259143168
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004998034102306921,
+ "loss": 1.2378,
+ "theoretical_loss": 3.600141174681515,
+ "tokens_seen": 1259208704
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997873620862588,
+ "loss": 1.2058,
+ "theoretical_loss": 3.6001242304014354,
+ "tokens_seen": 1259274240
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997713139418255,
+ "loss": 1.3714,
+ "theoretical_loss": 3.6001072872500552,
+ "tokens_seen": 1259339776
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997552657973922,
+ "loss": 1.3448,
+ "theoretical_loss": 3.6000903452272404,
+ "tokens_seen": 1259405312
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997392176529589,
+ "loss": 1.3862,
+ "theoretical_loss": 3.600073404332856,
+ "tokens_seen": 1259470848
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997231695085256,
+ "loss": 1.3522,
+ "theoretical_loss": 3.6000564645667703,
+ "tokens_seen": 1259536384
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004997071213640923,
+ "loss": 1.1841,
+ "theoretical_loss": 3.6000395259288482,
+ "tokens_seen": 1259601920
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996910732196591,
+ "loss": 1.3151,
+ "theoretical_loss": 3.6000225884189563,
+ "tokens_seen": 1259667456
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996750250752257,
+ "loss": 1.1988,
+ "theoretical_loss": 3.60000565203696,
+ "tokens_seen": 1259732992
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996589769307925,
+ "loss": 1.2379,
+ "theoretical_loss": 3.599988716782726,
+ "tokens_seen": 1259798528
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996429287863591,
+ "loss": 1.3703,
+ "theoretical_loss": 3.5999717826561204,
+ "tokens_seen": 1259864064
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996268806419259,
+ "loss": 1.3443,
+ "theoretical_loss": 3.59995484965701,
+ "tokens_seen": 1259929600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004996108324974925,
+ "loss": 1.307,
+ "theoretical_loss": 3.5999379177852604,
+ "tokens_seen": 1259995136
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004995947843530592,
+ "loss": 1.297,
+ "theoretical_loss": 3.5999209870407385,
+ "tokens_seen": 1260060672
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004995787362086259,
+ "loss": 1.3657,
+ "theoretical_loss": 3.59990405742331,
+ "tokens_seen": 1260126208
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004995626880641926,
+ "loss": 1.3301,
+ "theoretical_loss": 3.599887128932842,
+ "tokens_seen": 1260191744
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004995466399197593,
+ "loss": 1.3567,
+ "theoretical_loss": 3.5998702015692006,
+ "tokens_seen": 1260257280
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000499530591775326,
+ "loss": 1.2652,
+ "theoretical_loss": 3.5998532753322516,
+ "tokens_seen": 1260322816
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004995145436308927,
+ "loss": 1.323,
+ "theoretical_loss": 3.599836350221862,
+ "tokens_seen": 1260388352
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994984954864594,
+ "loss": 1.2185,
+ "theoretical_loss": 3.5998194262378984,
+ "tokens_seen": 1260453888
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994824473420261,
+ "loss": 1.312,
+ "theoretical_loss": 3.599802503380227,
+ "tokens_seen": 1260519424
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994663991975928,
+ "loss": 1.2526,
+ "theoretical_loss": 3.5997855816487143,
+ "tokens_seen": 1260584960
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994503510531594,
+ "loss": 1.1733,
+ "theoretical_loss": 3.599768661043227,
+ "tokens_seen": 1260650496
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994343029087262,
+ "loss": 1.2027,
+ "theoretical_loss": 3.5997517415636313,
+ "tokens_seen": 1260716032
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994182547642928,
+ "loss": 1.2417,
+ "theoretical_loss": 3.599734823209794,
+ "tokens_seen": 1260781568
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004994022066198596,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5997179059815814,
+ "tokens_seen": 1260847104
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004993861584754262,
+ "loss": 1.2988,
+ "theoretical_loss": 3.599700989878861,
+ "tokens_seen": 1260912640
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000499370110330993,
+ "loss": 1.3054,
+ "theoretical_loss": 3.599684074901498,
+ "tokens_seen": 1260978176
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004993540621865596,
+ "loss": 1.172,
+ "theoretical_loss": 3.5996671610493602,
+ "tokens_seen": 1261043712
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004993380140421264,
+ "loss": 1.3809,
+ "theoretical_loss": 3.599650248322314,
+ "tokens_seen": 1261109248
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000499321965897693,
+ "loss": 1.2898,
+ "theoretical_loss": 3.599633336720226,
+ "tokens_seen": 1261174784
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004993059177532598,
+ "loss": 1.2499,
+ "theoretical_loss": 3.599616426242963,
+ "tokens_seen": 1261240320
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004992898696088264,
+ "loss": 1.2464,
+ "theoretical_loss": 3.5995995168903914,
+ "tokens_seen": 1261305856
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004992738214643932,
+ "loss": 1.3148,
+ "theoretical_loss": 3.599582608662378,
+ "tokens_seen": 1261371392
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004992577733199598,
+ "loss": 1.2937,
+ "theoretical_loss": 3.59956570155879,
+ "tokens_seen": 1261436928
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004992417251755266,
+ "loss": 1.2998,
+ "theoretical_loss": 3.5995487955794943,
+ "tokens_seen": 1261502464
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 423890,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3517919778823853,
+ "objective/train/theoretical_loss": 3.599531890724357,
+ "objective/train/tokens_used": 1282028000,
+ "theoretical_loss": 3.599531890724357,
+ "tokens_seen": 1261568000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004992256770310932,
+ "loss": 1.3402,
+ "theoretical_loss": 3.599531890724357,
+ "tokens_seen": 1261568000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.00049920962888666,
+ "loss": 1.2594,
+ "theoretical_loss": 3.599514986993246,
+ "tokens_seen": 1261633536
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004991935807422266,
+ "loss": 1.2545,
+ "theoretical_loss": 3.5994980843860276,
+ "tokens_seen": 1261699072
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004991775325977934,
+ "loss": 1.4063,
+ "theoretical_loss": 3.5994811829025686,
+ "tokens_seen": 1261764608
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.00049916148445336,
+ "loss": 1.342,
+ "theoretical_loss": 3.599464282542735,
+ "tokens_seen": 1261830144
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004991454363089268,
+ "loss": 1.3544,
+ "theoretical_loss": 3.599447383306396,
+ "tokens_seen": 1261895680
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004991293881644935,
+ "loss": 1.2332,
+ "theoretical_loss": 3.5994304851934174,
+ "tokens_seen": 1261961216
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004991133400200602,
+ "loss": 1.3271,
+ "theoretical_loss": 3.5994135882036655,
+ "tokens_seen": 1262026752
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990972918756269,
+ "loss": 1.1862,
+ "theoretical_loss": 3.5993966923370087,
+ "tokens_seen": 1262092288
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990812437311936,
+ "loss": 1.3249,
+ "theoretical_loss": 3.599379797593312,
+ "tokens_seen": 1262157824
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990651955867603,
+ "loss": 1.3205,
+ "theoretical_loss": 3.599362903972445,
+ "tokens_seen": 1262223360
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000499049147442327,
+ "loss": 1.3479,
+ "theoretical_loss": 3.5993460114742732,
+ "tokens_seen": 1262288896
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990330992978937,
+ "loss": 1.2175,
+ "theoretical_loss": 3.599329120098664,
+ "tokens_seen": 1262354432
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990170511534604,
+ "loss": 1.3575,
+ "theoretical_loss": 3.5993122298454843,
+ "tokens_seen": 1262419968
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004990010030090271,
+ "loss": 1.3866,
+ "theoretical_loss": 3.599295340714602,
+ "tokens_seen": 1262485504
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989849548645938,
+ "loss": 1.3307,
+ "theoretical_loss": 3.5992784527058834,
+ "tokens_seen": 1262551040
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989689067201605,
+ "loss": 1.2999,
+ "theoretical_loss": 3.5992615658191967,
+ "tokens_seen": 1262616576
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989528585757272,
+ "loss": 1.2711,
+ "theoretical_loss": 3.599244680054408,
+ "tokens_seen": 1262682112
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989368104312939,
+ "loss": 1.2886,
+ "theoretical_loss": 3.599227795411385,
+ "tokens_seen": 1262747648
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989207622868606,
+ "loss": 1.3036,
+ "theoretical_loss": 3.5992109118899958,
+ "tokens_seen": 1262813184
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004989047141424273,
+ "loss": 1.3382,
+ "theoretical_loss": 3.599194029490106,
+ "tokens_seen": 1262878720
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498888665997994,
+ "loss": 1.3411,
+ "theoretical_loss": 3.5991771482115844,
+ "tokens_seen": 1262944256
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004988726178535607,
+ "loss": 1.3098,
+ "theoretical_loss": 3.5991602680542973,
+ "tokens_seen": 1263009792
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004988565697091274,
+ "loss": 1.3073,
+ "theoretical_loss": 3.5991433890181126,
+ "tokens_seen": 1263075328
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004988405215646941,
+ "loss": 1.2819,
+ "theoretical_loss": 3.599126511102898,
+ "tokens_seen": 1263140864
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004988244734202608,
+ "loss": 1.311,
+ "theoretical_loss": 3.59910963430852,
+ "tokens_seen": 1263206400
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004988084252758275,
+ "loss": 1.3306,
+ "theoretical_loss": 3.599092758634847,
+ "tokens_seen": 1263271936
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004987923771313942,
+ "loss": 1.2372,
+ "theoretical_loss": 3.5990758840817456,
+ "tokens_seen": 1263337472
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004987763289869609,
+ "loss": 1.2025,
+ "theoretical_loss": 3.599059010649084,
+ "tokens_seen": 1263403008
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004987602808425276,
+ "loss": 1.2648,
+ "theoretical_loss": 3.5990421383367286,
+ "tokens_seen": 1263468544
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004987442326980943,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5990252671445484,
+ "tokens_seen": 1263534080
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498728184553661,
+ "loss": 1.3771,
+ "theoretical_loss": 3.5990083970724096,
+ "tokens_seen": 1263599616
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004987121364092277,
+ "loss": 1.2826,
+ "theoretical_loss": 3.5989915281201807,
+ "tokens_seen": 1263665152
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986960882647944,
+ "loss": 1.3061,
+ "theoretical_loss": 3.5989746602877286,
+ "tokens_seen": 1263730688
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986800401203611,
+ "loss": 1.2911,
+ "theoretical_loss": 3.5989577935749217,
+ "tokens_seen": 1263796224
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986639919759278,
+ "loss": 1.2631,
+ "theoretical_loss": 3.598940927981627,
+ "tokens_seen": 1263861760
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986479438314945,
+ "loss": 1.3076,
+ "theoretical_loss": 3.598924063507712,
+ "tokens_seen": 1263927296
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986318956870612,
+ "loss": 1.3675,
+ "theoretical_loss": 3.598907200153045,
+ "tokens_seen": 1263992832
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004986158475426279,
+ "loss": 1.2881,
+ "theoretical_loss": 3.598890337917493,
+ "tokens_seen": 1264058368
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985997993981946,
+ "loss": 1.3467,
+ "theoretical_loss": 3.5988734768009243,
+ "tokens_seen": 1264123904
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985837512537613,
+ "loss": 1.3733,
+ "theoretical_loss": 3.5988566168032063,
+ "tokens_seen": 1264189440
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498567703109328,
+ "loss": 1.3434,
+ "theoretical_loss": 3.598839757924207,
+ "tokens_seen": 1264254976
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985516549648947,
+ "loss": 1.2633,
+ "theoretical_loss": 3.5988229001637935,
+ "tokens_seen": 1264320512
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985356068204614,
+ "loss": 1.2668,
+ "theoretical_loss": 3.5988060435218348,
+ "tokens_seen": 1264386048
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985195586760281,
+ "loss": 1.2314,
+ "theoretical_loss": 3.598789187998198,
+ "tokens_seen": 1264451584
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004985035105315948,
+ "loss": 1.3266,
+ "theoretical_loss": 3.5987723335927506,
+ "tokens_seen": 1264517120
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004984874623871616,
+ "loss": 1.2526,
+ "theoretical_loss": 3.5987554803053614,
+ "tokens_seen": 1264582656
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004984714142427282,
+ "loss": 1.3122,
+ "theoretical_loss": 3.5987386281358975,
+ "tokens_seen": 1264648192
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498455366098295,
+ "loss": 1.3031,
+ "theoretical_loss": 3.5987217770842275,
+ "tokens_seen": 1264713728
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004984393179538616,
+ "loss": 1.1985,
+ "theoretical_loss": 3.5987049271502185,
+ "tokens_seen": 1264779264
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 424734,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4077988862991333,
+ "objective/train/theoretical_loss": 3.5986880783337387,
+ "objective/train/tokens_used": 1285304800,
+ "theoretical_loss": 3.5986880783337387,
+ "tokens_seen": 1264844800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004984232698094284,
+ "loss": 1.3125,
+ "theoretical_loss": 3.5986880783337387,
+ "tokens_seen": 1264844800
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498407221664995,
+ "loss": 1.1478,
+ "theoretical_loss": 3.598671230634657,
+ "tokens_seen": 1264910336
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983911735205618,
+ "loss": 1.2639,
+ "theoretical_loss": 3.59865438405284,
+ "tokens_seen": 1264975872
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983751253761284,
+ "loss": 1.2707,
+ "theoretical_loss": 3.5986375385881573,
+ "tokens_seen": 1265041408
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983590772316952,
+ "loss": 1.3552,
+ "theoretical_loss": 3.5986206942404757,
+ "tokens_seen": 1265106944
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983430290872618,
+ "loss": 1.272,
+ "theoretical_loss": 3.5986038510096634,
+ "tokens_seen": 1265172480
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983269809428286,
+ "loss": 1.3361,
+ "theoretical_loss": 3.5985870088955894,
+ "tokens_seen": 1265238016
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004983109327983952,
+ "loss": 1.278,
+ "theoretical_loss": 3.598570167898121,
+ "tokens_seen": 1265303552
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498294884653962,
+ "loss": 1.3644,
+ "theoretical_loss": 3.5985533280171262,
+ "tokens_seen": 1265369088
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004982788365095286,
+ "loss": 1.5033,
+ "theoretical_loss": 3.5985364892524734,
+ "tokens_seen": 1265434624
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004982627883650954,
+ "loss": 1.3121,
+ "theoretical_loss": 3.598519651604031,
+ "tokens_seen": 1265500160
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498246740220662,
+ "loss": 1.2772,
+ "theoretical_loss": 3.598502815071667,
+ "tokens_seen": 1265565696
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004982306920762288,
+ "loss": 1.3747,
+ "theoretical_loss": 3.5984859796552504,
+ "tokens_seen": 1265631232
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004982146439317954,
+ "loss": 1.3262,
+ "theoretical_loss": 3.598469145354648,
+ "tokens_seen": 1265696768
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981985957873622,
+ "loss": 1.2549,
+ "theoretical_loss": 3.598452312169729,
+ "tokens_seen": 1265762304
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981825476429288,
+ "loss": 1.2031,
+ "theoretical_loss": 3.598435480100362,
+ "tokens_seen": 1265827840
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981664994984955,
+ "loss": 1.3247,
+ "theoretical_loss": 3.5984186491464145,
+ "tokens_seen": 1265893376
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981504513540622,
+ "loss": 1.2843,
+ "theoretical_loss": 3.598401819307756,
+ "tokens_seen": 1265958912
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981344032096289,
+ "loss": 1.3527,
+ "theoretical_loss": 3.598384990584253,
+ "tokens_seen": 1266024448
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981183550651956,
+ "loss": 1.3106,
+ "theoretical_loss": 3.5983681629757753,
+ "tokens_seen": 1266089984
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004981023069207623,
+ "loss": 1.3004,
+ "theoretical_loss": 3.598351336482191,
+ "tokens_seen": 1266155520
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000498086258776329,
+ "loss": 1.3196,
+ "theoretical_loss": 3.598334511103369,
+ "tokens_seen": 1266221056
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004980702106318957,
+ "loss": 1.3494,
+ "theoretical_loss": 3.5983176868391764,
+ "tokens_seen": 1266286592
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004980541624874624,
+ "loss": 1.3617,
+ "theoretical_loss": 3.5983008636894835,
+ "tokens_seen": 1266352128
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004980381143430291,
+ "loss": 1.2671,
+ "theoretical_loss": 3.598284041654157,
+ "tokens_seen": 1266417664
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004980220661985957,
+ "loss": 1.2546,
+ "theoretical_loss": 3.598267220733067,
+ "tokens_seen": 1266483200
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004980060180541625,
+ "loss": 1.2556,
+ "theoretical_loss": 3.598250400926081,
+ "tokens_seen": 1266548736
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979899699097291,
+ "loss": 1.2886,
+ "theoretical_loss": 3.5982335822330676,
+ "tokens_seen": 1266614272
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979739217652959,
+ "loss": 1.106,
+ "theoretical_loss": 3.5982167646538956,
+ "tokens_seen": 1266679808
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979578736208625,
+ "loss": 1.347,
+ "theoretical_loss": 3.5981999481884346,
+ "tokens_seen": 1266745344
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979418254764293,
+ "loss": 1.2945,
+ "theoretical_loss": 3.5981831328365512,
+ "tokens_seen": 1266810880
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979257773319959,
+ "loss": 1.2477,
+ "theoretical_loss": 3.598166318598116,
+ "tokens_seen": 1266876416
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004979097291875627,
+ "loss": 1.3321,
+ "theoretical_loss": 3.5981495054729966,
+ "tokens_seen": 1266941952
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978936810431293,
+ "loss": 1.1598,
+ "theoretical_loss": 3.5981326934610616,
+ "tokens_seen": 1267007488
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978776328986961,
+ "loss": 1.2386,
+ "theoretical_loss": 3.5981158825621806,
+ "tokens_seen": 1267073024
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978615847542627,
+ "loss": 1.2771,
+ "theoretical_loss": 3.598099072776222,
+ "tokens_seen": 1267138560
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978455366098295,
+ "loss": 1.2073,
+ "theoretical_loss": 3.598082264103054,
+ "tokens_seen": 1267204096
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978294884653961,
+ "loss": 1.1953,
+ "theoretical_loss": 3.5980654565425456,
+ "tokens_seen": 1267269632
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004978134403209629,
+ "loss": 1.253,
+ "theoretical_loss": 3.598048650094566,
+ "tokens_seen": 1267335168
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977973921765296,
+ "loss": 1.2272,
+ "theoretical_loss": 3.598031844758984,
+ "tokens_seen": 1267400704
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977813440320963,
+ "loss": 1.3106,
+ "theoretical_loss": 3.598015040535668,
+ "tokens_seen": 1267466240
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000497765295887663,
+ "loss": 1.4032,
+ "theoretical_loss": 3.597998237424487,
+ "tokens_seen": 1267531776
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977492477432297,
+ "loss": 1.3226,
+ "theoretical_loss": 3.597981435425311,
+ "tokens_seen": 1267597312
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977331995987964,
+ "loss": 1.2629,
+ "theoretical_loss": 3.597964634538007,
+ "tokens_seen": 1267662848
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977171514543631,
+ "loss": 1.3556,
+ "theoretical_loss": 3.5979478347624454,
+ "tokens_seen": 1267728384
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004977011033099298,
+ "loss": 1.3254,
+ "theoretical_loss": 3.5979310360984944,
+ "tokens_seen": 1267793920
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004976850551654965,
+ "loss": 1.2801,
+ "theoretical_loss": 3.597914238546023,
+ "tokens_seen": 1267859456
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004976690070210632,
+ "loss": 1.2658,
+ "theoretical_loss": 3.5978974421049017,
+ "tokens_seen": 1267924992
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004976529588766299,
+ "loss": 1.217,
+ "theoretical_loss": 3.5978806467749975,
+ "tokens_seen": 1267990528
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004976369107321966,
+ "loss": 1.3581,
+ "theoretical_loss": 3.5978638525561806,
+ "tokens_seen": 1268056064
+ },
+ {
+ "epoch": 0.38,
+ "objective/train/docs_used": 425936,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3263369798660278,
+ "objective/train/theoretical_loss": 3.5978470594483194,
+ "objective/train/tokens_used": 1288581600,
+ "theoretical_loss": 3.5978470594483194,
+ "tokens_seen": 1268121600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004976208625877633,
+ "loss": 1.2524,
+ "theoretical_loss": 3.5978470594483194,
+ "tokens_seen": 1268121600
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.00049760481444333,
+ "loss": 1.2851,
+ "theoretical_loss": 3.5978302674512834,
+ "tokens_seen": 1268187136
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975887662988967,
+ "loss": 1.3585,
+ "theoretical_loss": 3.5978134765649417,
+ "tokens_seen": 1268252672
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975727181544634,
+ "loss": 1.2964,
+ "theoretical_loss": 3.597796686789164,
+ "tokens_seen": 1268318208
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975566700100301,
+ "loss": 1.2648,
+ "theoretical_loss": 3.5977798981238185,
+ "tokens_seen": 1268383744
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975406218655968,
+ "loss": 1.2545,
+ "theoretical_loss": 3.597763110568775,
+ "tokens_seen": 1268449280
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975245737211635,
+ "loss": 1.3272,
+ "theoretical_loss": 3.597746324123902,
+ "tokens_seen": 1268514816
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004975085255767302,
+ "loss": 1.265,
+ "theoretical_loss": 3.5977295387890695,
+ "tokens_seen": 1268580352
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004974924774322969,
+ "loss": 1.3263,
+ "theoretical_loss": 3.5977127545641467,
+ "tokens_seen": 1268645888
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004974764292878636,
+ "loss": 1.2467,
+ "theoretical_loss": 3.597695971449003,
+ "tokens_seen": 1268711424
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004974603811434303,
+ "loss": 1.3192,
+ "theoretical_loss": 3.5976791894435065,
+ "tokens_seen": 1268776960
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.000497444332998997,
+ "loss": 1.251,
+ "theoretical_loss": 3.597662408547528,
+ "tokens_seen": 1268842496
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004974282848545637,
+ "loss": 1.2984,
+ "theoretical_loss": 3.5976456287609357,
+ "tokens_seen": 1268908032
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004974122367101304,
+ "loss": 1.3213,
+ "theoretical_loss": 3.5976288500836002,
+ "tokens_seen": 1268973568
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973961885656971,
+ "loss": 1.3017,
+ "theoretical_loss": 3.59761207251539,
+ "tokens_seen": 1269039104
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973801404212638,
+ "loss": 1.3569,
+ "theoretical_loss": 3.5975952960561743,
+ "tokens_seen": 1269104640
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973640922768305,
+ "loss": 1.3836,
+ "theoretical_loss": 3.5975785207058237,
+ "tokens_seen": 1269170176
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973480441323972,
+ "loss": 1.3471,
+ "theoretical_loss": 3.5975617464642067,
+ "tokens_seen": 1269235712
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973319959879639,
+ "loss": 1.3619,
+ "theoretical_loss": 3.5975449733311926,
+ "tokens_seen": 1269301248
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004973159478435306,
+ "loss": 1.3563,
+ "theoretical_loss": 3.597528201306651,
+ "tokens_seen": 1269366784
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972998996990973,
+ "loss": 1.2156,
+ "theoretical_loss": 3.5975114303904525,
+ "tokens_seen": 1269432320
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972838515546641,
+ "loss": 1.2841,
+ "theoretical_loss": 3.5974946605824654,
+ "tokens_seen": 1269497856
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972678034102307,
+ "loss": 1.3345,
+ "theoretical_loss": 3.59747789188256,
+ "tokens_seen": 1269563392
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972517552657975,
+ "loss": 1.3687,
+ "theoretical_loss": 3.5974611242906054,
+ "tokens_seen": 1269628928
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972357071213641,
+ "loss": 1.3154,
+ "theoretical_loss": 3.597444357806471,
+ "tokens_seen": 1269694464
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972196589769309,
+ "loss": 1.2488,
+ "theoretical_loss": 3.5974275924300274,
+ "tokens_seen": 1269760000
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004972036108324975,
+ "loss": 1.2299,
+ "theoretical_loss": 3.597410828161144,
+ "tokens_seen": 1269825536
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971875626880643,
+ "loss": 1.3637,
+ "theoretical_loss": 3.5973940649996896,
+ "tokens_seen": 1269891072
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971715145436309,
+ "loss": 1.2875,
+ "theoretical_loss": 3.5973773029455343,
+ "tokens_seen": 1269956608
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971554663991977,
+ "loss": 1.4486,
+ "theoretical_loss": 3.5973605419985484,
+ "tokens_seen": 1270022144
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971394182547643,
+ "loss": 1.3388,
+ "theoretical_loss": 3.5973437821586014,
+ "tokens_seen": 1270087680
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971233701103311,
+ "loss": 1.4281,
+ "theoretical_loss": 3.5973270234255628,
+ "tokens_seen": 1270153216
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004971073219658977,
+ "loss": 1.3701,
+ "theoretical_loss": 3.597310265799302,
+ "tokens_seen": 1270218752
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004970912738214645,
+ "loss": 1.2282,
+ "theoretical_loss": 3.5972935092796896,
+ "tokens_seen": 1270284288
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004970752256770311,
+ "loss": 1.3315,
+ "theoretical_loss": 3.5972767538665953,
+ "tokens_seen": 1270349824
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004970591775325979,
+ "loss": 1.2759,
+ "theoretical_loss": 3.5972599995598884,
+ "tokens_seen": 1270415360
+ },
+ {
+ "epoch": 0.38,
+ "learning_rate": 0.0004970431293881645,
+ "loss": 1.3413,
+ "theoretical_loss": 3.5972432463594393,
+ "tokens_seen": 1270480896
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004970270812437313,
+ "loss": 1.3109,
+ "theoretical_loss": 3.5972264942651178,
+ "tokens_seen": 1270546432
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004970110330992979,
+ "loss": 1.3672,
+ "theoretical_loss": 3.597209743276794,
+ "tokens_seen": 1270611968
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969949849548647,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5971929933943376,
+ "tokens_seen": 1270677504
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969789368104313,
+ "loss": 1.332,
+ "theoretical_loss": 3.5971762446176183,
+ "tokens_seen": 1270743040
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969628886659981,
+ "loss": 1.3023,
+ "theoretical_loss": 3.5971594969465066,
+ "tokens_seen": 1270808576
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969468405215647,
+ "loss": 1.3514,
+ "theoretical_loss": 3.597142750380872,
+ "tokens_seen": 1270874112
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969307923771315,
+ "loss": 1.2509,
+ "theoretical_loss": 3.597126004920585,
+ "tokens_seen": 1270939648
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004969147442326981,
+ "loss": 1.15,
+ "theoretical_loss": 3.5971092605655155,
+ "tokens_seen": 1271005184
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968986960882649,
+ "loss": 1.2723,
+ "theoretical_loss": 3.597092517315533,
+ "tokens_seen": 1271070720
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968826479438315,
+ "loss": 1.1763,
+ "theoretical_loss": 3.5970757751705085,
+ "tokens_seen": 1271136256
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968665997993983,
+ "loss": 1.3644,
+ "theoretical_loss": 3.5970590341303117,
+ "tokens_seen": 1271201792
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968505516549649,
+ "loss": 1.2889,
+ "theoretical_loss": 3.5970422941948126,
+ "tokens_seen": 1271267328
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968345035105317,
+ "loss": 1.3835,
+ "theoretical_loss": 3.5970255553638815,
+ "tokens_seen": 1271332864
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 426775,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3854188919067383,
+ "objective/train/theoretical_loss": 3.5970088176373887,
+ "objective/train/tokens_used": 1291858400,
+ "theoretical_loss": 3.5970088176373887,
+ "tokens_seen": 1271398400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968184553660983,
+ "loss": 1.3204,
+ "theoretical_loss": 3.5970088176373887,
+ "tokens_seen": 1271398400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004968024072216651,
+ "loss": 1.3744,
+ "theoretical_loss": 3.5969920810152045,
+ "tokens_seen": 1271463936
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967863590772317,
+ "loss": 1.3329,
+ "theoretical_loss": 3.596975345497199,
+ "tokens_seen": 1271529472
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967703109327984,
+ "loss": 1.2875,
+ "theoretical_loss": 3.596958611083242,
+ "tokens_seen": 1271595008
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967542627883651,
+ "loss": 1.263,
+ "theoretical_loss": 3.5969418777732045,
+ "tokens_seen": 1271660544
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967382146439318,
+ "loss": 1.2913,
+ "theoretical_loss": 3.5969251455669564,
+ "tokens_seen": 1271726080
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967221664994985,
+ "loss": 1.2597,
+ "theoretical_loss": 3.5969084144643677,
+ "tokens_seen": 1271791616
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004967061183550652,
+ "loss": 1.211,
+ "theoretical_loss": 3.5968916844653096,
+ "tokens_seen": 1271857152
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496690070210632,
+ "loss": 1.2919,
+ "theoretical_loss": 3.5968749555696515,
+ "tokens_seen": 1271922688
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004966740220661986,
+ "loss": 1.349,
+ "theoretical_loss": 3.5968582277772647,
+ "tokens_seen": 1271988224
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004966579739217653,
+ "loss": 1.1978,
+ "theoretical_loss": 3.5968415010880186,
+ "tokens_seen": 1272053760
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496641925777332,
+ "loss": 1.265,
+ "theoretical_loss": 3.5968247755017844,
+ "tokens_seen": 1272119296
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004966258776328986,
+ "loss": 1.2978,
+ "theoretical_loss": 3.5968080510184324,
+ "tokens_seen": 1272184832
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004966098294884654,
+ "loss": 1.3129,
+ "theoretical_loss": 3.596791327637833,
+ "tokens_seen": 1272250368
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496593781344032,
+ "loss": 1.304,
+ "theoretical_loss": 3.596774605359857,
+ "tokens_seen": 1272315904
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004965777331995988,
+ "loss": 1.3701,
+ "theoretical_loss": 3.596757884184374,
+ "tokens_seen": 1272381440
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004965616850551654,
+ "loss": 1.2884,
+ "theoretical_loss": 3.596741164111255,
+ "tokens_seen": 1272446976
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004965456369107322,
+ "loss": 1.3437,
+ "theoretical_loss": 3.596724445140371,
+ "tokens_seen": 1272512512
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004965295887662988,
+ "loss": 1.2489,
+ "theoretical_loss": 3.596707727271592,
+ "tokens_seen": 1272578048
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004965135406218656,
+ "loss": 1.2302,
+ "theoretical_loss": 3.596691010504789,
+ "tokens_seen": 1272643584
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004964974924774322,
+ "loss": 1.3499,
+ "theoretical_loss": 3.5966742948398327,
+ "tokens_seen": 1272709120
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496481444332999,
+ "loss": 1.3188,
+ "theoretical_loss": 3.596657580276593,
+ "tokens_seen": 1272774656
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004964653961885656,
+ "loss": 1.3194,
+ "theoretical_loss": 3.5966408668149414,
+ "tokens_seen": 1272840192
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004964493480441324,
+ "loss": 1.3017,
+ "theoretical_loss": 3.5966241544547484,
+ "tokens_seen": 1272905728
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496433299899699,
+ "loss": 1.2717,
+ "theoretical_loss": 3.596607443195884,
+ "tokens_seen": 1272971264
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004964172517552658,
+ "loss": 1.32,
+ "theoretical_loss": 3.59659073303822,
+ "tokens_seen": 1273036800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004964012036108325,
+ "loss": 1.2705,
+ "theoretical_loss": 3.5965740239816264,
+ "tokens_seen": 1273102336
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004963851554663992,
+ "loss": 1.2454,
+ "theoretical_loss": 3.596557316025974,
+ "tokens_seen": 1273167872
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004963691073219659,
+ "loss": 1.209,
+ "theoretical_loss": 3.596540609171134,
+ "tokens_seen": 1273233408
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004963530591775326,
+ "loss": 1.2642,
+ "theoretical_loss": 3.596523903416977,
+ "tokens_seen": 1273298944
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004963370110330993,
+ "loss": 1.3399,
+ "theoretical_loss": 3.5965071987633745,
+ "tokens_seen": 1273364480
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496320962888666,
+ "loss": 1.2915,
+ "theoretical_loss": 3.5964904952101957,
+ "tokens_seen": 1273430016
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004963049147442327,
+ "loss": 1.1884,
+ "theoretical_loss": 3.5964737927573136,
+ "tokens_seen": 1273495552
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962888665997994,
+ "loss": 1.2608,
+ "theoretical_loss": 3.596457091404597,
+ "tokens_seen": 1273561088
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962728184553661,
+ "loss": 1.2978,
+ "theoretical_loss": 3.5964403911519183,
+ "tokens_seen": 1273626624
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962567703109328,
+ "loss": 1.3278,
+ "theoretical_loss": 3.596423691999148,
+ "tokens_seen": 1273692160
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962407221664995,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5964069939461565,
+ "tokens_seen": 1273757696
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962246740220662,
+ "loss": 1.2383,
+ "theoretical_loss": 3.596390296992816,
+ "tokens_seen": 1273823232
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004962086258776329,
+ "loss": 1.4236,
+ "theoretical_loss": 3.5963736011389966,
+ "tokens_seen": 1273888768
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004961925777331996,
+ "loss": 1.2526,
+ "theoretical_loss": 3.59635690638457,
+ "tokens_seen": 1273954304
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004961765295887663,
+ "loss": 1.3395,
+ "theoretical_loss": 3.596340212729406,
+ "tokens_seen": 1274019840
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496160481444333,
+ "loss": 1.3486,
+ "theoretical_loss": 3.596323520173377,
+ "tokens_seen": 1274085376
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004961444332998997,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5963068287163535,
+ "tokens_seen": 1274150912
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004961283851554664,
+ "loss": 1.2996,
+ "theoretical_loss": 3.5962901383582073,
+ "tokens_seen": 1274216448
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004961123370110331,
+ "loss": 1.3083,
+ "theoretical_loss": 3.596273449098809,
+ "tokens_seen": 1274281984
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960962888665998,
+ "loss": 1.2555,
+ "theoretical_loss": 3.5962567609380285,
+ "tokens_seen": 1274347520
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960802407221666,
+ "loss": 1.2913,
+ "theoretical_loss": 3.596240073875739,
+ "tokens_seen": 1274413056
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960641925777332,
+ "loss": 1.2185,
+ "theoretical_loss": 3.5962233879118113,
+ "tokens_seen": 1274478592
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960481444333,
+ "loss": 1.3612,
+ "theoretical_loss": 3.596206703046116,
+ "tokens_seen": 1274544128
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960320962888666,
+ "loss": 1.3007,
+ "theoretical_loss": 3.5961900192785246,
+ "tokens_seen": 1274609664
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 427982,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2709904909133911,
+ "objective/train/theoretical_loss": 3.5961733366089086,
+ "objective/train/tokens_used": 1295135200,
+ "theoretical_loss": 3.5961733366089086,
+ "tokens_seen": 1274675200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004960160481444334,
+ "loss": 1.2709,
+ "theoretical_loss": 3.5961733366089086,
+ "tokens_seen": 1274675200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000496,
+ "loss": 1.3176,
+ "theoretical_loss": 3.596156655037139,
+ "tokens_seen": 1274740736
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959839518555668,
+ "loss": 1.2992,
+ "theoretical_loss": 3.596139974563087,
+ "tokens_seen": 1274806272
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959679037111334,
+ "loss": 1.2785,
+ "theoretical_loss": 3.5961232951866244,
+ "tokens_seen": 1274871808
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959518555667002,
+ "loss": 1.272,
+ "theoretical_loss": 3.5961066169076226,
+ "tokens_seen": 1274937344
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959358074222668,
+ "loss": 1.2913,
+ "theoretical_loss": 3.5960899397259523,
+ "tokens_seen": 1275002880
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959197592778336,
+ "loss": 1.3332,
+ "theoretical_loss": 3.596073263641485,
+ "tokens_seen": 1275068416
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004959037111334002,
+ "loss": 1.3603,
+ "theoretical_loss": 3.596056588654093,
+ "tokens_seen": 1275133952
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495887662988967,
+ "loss": 1.2949,
+ "theoretical_loss": 3.596039914763647,
+ "tokens_seen": 1275199488
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004958716148445336,
+ "loss": 1.4417,
+ "theoretical_loss": 3.596023241970019,
+ "tokens_seen": 1275265024
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004958555667001004,
+ "loss": 1.2076,
+ "theoretical_loss": 3.5960065702730795,
+ "tokens_seen": 1275330560
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495839518555667,
+ "loss": 1.4377,
+ "theoretical_loss": 3.5959898996727007,
+ "tokens_seen": 1275396096
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004958234704112338,
+ "loss": 1.2029,
+ "theoretical_loss": 3.595973230168754,
+ "tokens_seen": 1275461632
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004958074222668004,
+ "loss": 1.4822,
+ "theoretical_loss": 3.5959565617611116,
+ "tokens_seen": 1275527168
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004957913741223672,
+ "loss": 1.3551,
+ "theoretical_loss": 3.5959398944496446,
+ "tokens_seen": 1275592704
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004957753259779338,
+ "loss": 1.3049,
+ "theoretical_loss": 3.5959232282342235,
+ "tokens_seen": 1275658240
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004957592778335006,
+ "loss": 1.3371,
+ "theoretical_loss": 3.595906563114722,
+ "tokens_seen": 1275723776
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004957432296890672,
+ "loss": 1.2368,
+ "theoretical_loss": 3.59588989909101,
+ "tokens_seen": 1275789312
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495727181544634,
+ "loss": 1.3828,
+ "theoretical_loss": 3.5958732361629604,
+ "tokens_seen": 1275854848
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004957111334002006,
+ "loss": 1.342,
+ "theoretical_loss": 3.595856574330444,
+ "tokens_seen": 1275920384
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004956950852557674,
+ "loss": 1.3444,
+ "theoretical_loss": 3.595839913593333,
+ "tokens_seen": 1275985920
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495679037111334,
+ "loss": 1.3372,
+ "theoretical_loss": 3.595823253951499,
+ "tokens_seen": 1276051456
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004956629889669008,
+ "loss": 1.2516,
+ "theoretical_loss": 3.595806595404813,
+ "tokens_seen": 1276116992
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004956469408224674,
+ "loss": 1.2169,
+ "theoretical_loss": 3.5957899379531484,
+ "tokens_seen": 1276182528
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004956308926780342,
+ "loss": 1.1793,
+ "theoretical_loss": 3.595773281596376,
+ "tokens_seen": 1276248064
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004956148445336008,
+ "loss": 1.3753,
+ "theoretical_loss": 3.595756626334367,
+ "tokens_seen": 1276313600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004955987963891676,
+ "loss": 1.3244,
+ "theoretical_loss": 3.595739972166995,
+ "tokens_seen": 1276379136
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004955827482447342,
+ "loss": 1.3931,
+ "theoretical_loss": 3.5957233190941302,
+ "tokens_seen": 1276444672
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495566700100301,
+ "loss": 1.4858,
+ "theoretical_loss": 3.595706667115645,
+ "tokens_seen": 1276510208
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004955506519558676,
+ "loss": 1.3683,
+ "theoretical_loss": 3.595690016231411,
+ "tokens_seen": 1276575744
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004955346038114344,
+ "loss": 1.2999,
+ "theoretical_loss": 3.595673366441301,
+ "tokens_seen": 1276641280
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495518555667001,
+ "loss": 1.3359,
+ "theoretical_loss": 3.595656717745187,
+ "tokens_seen": 1276706816
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004955025075225678,
+ "loss": 1.3228,
+ "theoretical_loss": 3.5956400701429394,
+ "tokens_seen": 1276772352
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004954864593781344,
+ "loss": 1.3666,
+ "theoretical_loss": 3.5956234236344313,
+ "tokens_seen": 1276837888
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004954704112337012,
+ "loss": 1.2305,
+ "theoretical_loss": 3.595606778219535,
+ "tokens_seen": 1276903424
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004954543630892678,
+ "loss": 1.233,
+ "theoretical_loss": 3.595590133898122,
+ "tokens_seen": 1276968960
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004954383149448346,
+ "loss": 1.3041,
+ "theoretical_loss": 3.5955734906700645,
+ "tokens_seen": 1277034496
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004954222668004012,
+ "loss": 1.28,
+ "theoretical_loss": 3.5955568485352343,
+ "tokens_seen": 1277100032
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495406218655968,
+ "loss": 1.2192,
+ "theoretical_loss": 3.5955402074935043,
+ "tokens_seen": 1277165568
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004953901705115346,
+ "loss": 1.2501,
+ "theoretical_loss": 3.5955235675447454,
+ "tokens_seen": 1277231104
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004953741223671013,
+ "loss": 1.342,
+ "theoretical_loss": 3.595506928688831,
+ "tokens_seen": 1277296640
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495358074222668,
+ "loss": 1.242,
+ "theoretical_loss": 3.5954902909256328,
+ "tokens_seen": 1277362176
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004953420260782347,
+ "loss": 1.2875,
+ "theoretical_loss": 3.5954736542550227,
+ "tokens_seen": 1277427712
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004953259779338014,
+ "loss": 1.3997,
+ "theoretical_loss": 3.5954570186768726,
+ "tokens_seen": 1277493248
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004953099297893681,
+ "loss": 1.2826,
+ "theoretical_loss": 3.5954403841910554,
+ "tokens_seen": 1277558784
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952938816449348,
+ "loss": 1.3569,
+ "theoretical_loss": 3.595423750797443,
+ "tokens_seen": 1277624320
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952778335005015,
+ "loss": 1.3848,
+ "theoretical_loss": 3.595407118495908,
+ "tokens_seen": 1277689856
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952617853560683,
+ "loss": 1.2403,
+ "theoretical_loss": 3.5953904872863225,
+ "tokens_seen": 1277755392
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952457372116349,
+ "loss": 1.3896,
+ "theoretical_loss": 3.5953738571685587,
+ "tokens_seen": 1277820928
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952296890672015,
+ "loss": 1.3923,
+ "theoretical_loss": 3.5953572281424893,
+ "tokens_seen": 1277886464
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 429086,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.291599988937378,
+ "objective/train/theoretical_loss": 3.595340600207986,
+ "objective/train/tokens_used": 1298412000,
+ "theoretical_loss": 3.595340600207986,
+ "tokens_seen": 1277952000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004952136409227683,
+ "loss": 1.2622,
+ "theoretical_loss": 3.595340600207986,
+ "tokens_seen": 1277952000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951975927783349,
+ "loss": 1.3463,
+ "theoretical_loss": 3.5953239733649216,
+ "tokens_seen": 1278017536
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951815446339017,
+ "loss": 1.3223,
+ "theoretical_loss": 3.595307347613169,
+ "tokens_seen": 1278083072
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951654964894683,
+ "loss": 1.3747,
+ "theoretical_loss": 3.5952907229525994,
+ "tokens_seen": 1278148608
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951494483450351,
+ "loss": 1.3882,
+ "theoretical_loss": 3.5952740993830865,
+ "tokens_seen": 1278214144
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951334002006017,
+ "loss": 1.4164,
+ "theoretical_loss": 3.595257476904502,
+ "tokens_seen": 1278279680
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951173520561685,
+ "loss": 1.3095,
+ "theoretical_loss": 3.595240855516718,
+ "tokens_seen": 1278345216
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004951013039117351,
+ "loss": 1.2501,
+ "theoretical_loss": 3.5952242352196078,
+ "tokens_seen": 1278410752
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004950852557673019,
+ "loss": 1.3951,
+ "theoretical_loss": 3.5952076160130435,
+ "tokens_seen": 1278476288
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004950692076228685,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5951909978968986,
+ "tokens_seen": 1278541824
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004950531594784353,
+ "loss": 1.2396,
+ "theoretical_loss": 3.595174380871044,
+ "tokens_seen": 1278607360
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000495037111334002,
+ "loss": 1.3792,
+ "theoretical_loss": 3.5951577649353537,
+ "tokens_seen": 1278672896
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004950210631895687,
+ "loss": 1.2545,
+ "theoretical_loss": 3.5951411500896997,
+ "tokens_seen": 1278738432
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004950050150451354,
+ "loss": 1.2397,
+ "theoretical_loss": 3.595124536333955,
+ "tokens_seen": 1278803968
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949889669007021,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5951079236679915,
+ "tokens_seen": 1278869504
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949729187562688,
+ "loss": 1.4003,
+ "theoretical_loss": 3.5950913120916823,
+ "tokens_seen": 1278935040
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949568706118355,
+ "loss": 1.4203,
+ "theoretical_loss": 3.5950747016049007,
+ "tokens_seen": 1279000576
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949408224674022,
+ "loss": 1.3769,
+ "theoretical_loss": 3.595058092207519,
+ "tokens_seen": 1279066112
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949247743229689,
+ "loss": 1.2913,
+ "theoretical_loss": 3.5950414838994087,
+ "tokens_seen": 1279131648
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004949087261785356,
+ "loss": 1.2267,
+ "theoretical_loss": 3.5950248766804447,
+ "tokens_seen": 1279197184
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948926780341023,
+ "loss": 1.3514,
+ "theoretical_loss": 3.5950082705504984,
+ "tokens_seen": 1279262720
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948766298896691,
+ "loss": 1.2558,
+ "theoretical_loss": 3.594991665509443,
+ "tokens_seen": 1279328256
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948605817452357,
+ "loss": 1.3068,
+ "theoretical_loss": 3.594975061557151,
+ "tokens_seen": 1279393792
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948445336008025,
+ "loss": 1.2278,
+ "theoretical_loss": 3.5949584586934957,
+ "tokens_seen": 1279459328
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948284854563691,
+ "loss": 1.301,
+ "theoretical_loss": 3.59494185691835,
+ "tokens_seen": 1279524864
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004948124373119359,
+ "loss": 1.3764,
+ "theoretical_loss": 3.594925256231586,
+ "tokens_seen": 1279590400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947963891675025,
+ "loss": 1.2844,
+ "theoretical_loss": 3.594908656633078,
+ "tokens_seen": 1279655936
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947803410230693,
+ "loss": 1.2771,
+ "theoretical_loss": 3.5948920581226975,
+ "tokens_seen": 1279721472
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947642928786359,
+ "loss": 1.3355,
+ "theoretical_loss": 3.594875460700319,
+ "tokens_seen": 1279787008
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947482447342027,
+ "loss": 1.3422,
+ "theoretical_loss": 3.594858864365813,
+ "tokens_seen": 1279852544
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947321965897693,
+ "loss": 1.3437,
+ "theoretical_loss": 3.594842269119055,
+ "tokens_seen": 1279918080
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947161484453361,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5948256749599174,
+ "tokens_seen": 1279983616
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004947001003009027,
+ "loss": 1.3646,
+ "theoretical_loss": 3.594809081888272,
+ "tokens_seen": 1280049152
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946840521564695,
+ "loss": 1.2377,
+ "theoretical_loss": 3.5947924899039934,
+ "tokens_seen": 1280114688
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946680040120361,
+ "loss": 1.312,
+ "theoretical_loss": 3.5947758990069536,
+ "tokens_seen": 1280180224
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946519558676029,
+ "loss": 1.4118,
+ "theoretical_loss": 3.594759309197027,
+ "tokens_seen": 1280245760
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946359077231695,
+ "loss": 1.4197,
+ "theoretical_loss": 3.594742720474085,
+ "tokens_seen": 1280311296
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946198595787363,
+ "loss": 1.3832,
+ "theoretical_loss": 3.5947261328380016,
+ "tokens_seen": 1280376832
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004946038114343029,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5947095462886502,
+ "tokens_seen": 1280442368
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945877632898697,
+ "loss": 1.2556,
+ "theoretical_loss": 3.594692960825904,
+ "tokens_seen": 1280507904
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945717151454363,
+ "loss": 1.31,
+ "theoretical_loss": 3.5946763764496357,
+ "tokens_seen": 1280573440
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945556670010031,
+ "loss": 1.4522,
+ "theoretical_loss": 3.594659793159719,
+ "tokens_seen": 1280638976
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945396188565697,
+ "loss": 1.3988,
+ "theoretical_loss": 3.5946432109560265,
+ "tokens_seen": 1280704512
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945235707121365,
+ "loss": 1.2574,
+ "theoretical_loss": 3.594626629838433,
+ "tokens_seen": 1280770048
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004945075225677031,
+ "loss": 1.3055,
+ "theoretical_loss": 3.5946100498068096,
+ "tokens_seen": 1280835584
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944914744232699,
+ "loss": 1.3813,
+ "theoretical_loss": 3.5945934708610308,
+ "tokens_seen": 1280901120
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944754262788365,
+ "loss": 1.3457,
+ "theoretical_loss": 3.59457689300097,
+ "tokens_seen": 1280966656
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944593781344033,
+ "loss": 1.3795,
+ "theoretical_loss": 3.5945603162265005,
+ "tokens_seen": 1281032192
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944433299899699,
+ "loss": 1.289,
+ "theoretical_loss": 3.594543740537496,
+ "tokens_seen": 1281097728
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944272818455367,
+ "loss": 1.2675,
+ "theoretical_loss": 3.594527165933829,
+ "tokens_seen": 1281163264
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 430126,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1517274379730225,
+ "objective/train/theoretical_loss": 3.594510592415374,
+ "objective/train/tokens_used": 1301688800,
+ "theoretical_loss": 3.594510592415374,
+ "tokens_seen": 1281228800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004944112337011033,
+ "loss": 1.3297,
+ "theoretical_loss": 3.594510592415374,
+ "tokens_seen": 1281228800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943951855566701,
+ "loss": 1.332,
+ "theoretical_loss": 3.5944940199820037,
+ "tokens_seen": 1281294336
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943791374122367,
+ "loss": 1.3252,
+ "theoretical_loss": 3.594477448633591,
+ "tokens_seen": 1281359872
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943630892678035,
+ "loss": 1.2618,
+ "theoretical_loss": 3.594460878370011,
+ "tokens_seen": 1281425408
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943470411233701,
+ "loss": 1.2576,
+ "theoretical_loss": 3.594444309191136,
+ "tokens_seen": 1281490944
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943309929789369,
+ "loss": 1.3243,
+ "theoretical_loss": 3.5944277410968395,
+ "tokens_seen": 1281556480
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004943149448345035,
+ "loss": 1.4041,
+ "theoretical_loss": 3.594411174086996,
+ "tokens_seen": 1281622016
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942988966900703,
+ "loss": 1.2448,
+ "theoretical_loss": 3.594394608161478,
+ "tokens_seen": 1281687552
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942828485456369,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5943780433201598,
+ "tokens_seen": 1281753088
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942668004012037,
+ "loss": 1.4211,
+ "theoretical_loss": 3.594361479562915,
+ "tokens_seen": 1281818624
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942507522567703,
+ "loss": 1.3106,
+ "theoretical_loss": 3.594344916889616,
+ "tokens_seen": 1281884160
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942347041123371,
+ "loss": 1.2583,
+ "theoretical_loss": 3.5943283553001386,
+ "tokens_seen": 1281949696
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942186559679037,
+ "loss": 1.4562,
+ "theoretical_loss": 3.594311794794355,
+ "tokens_seen": 1282015232
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004942026078234705,
+ "loss": 1.3874,
+ "theoretical_loss": 3.5942952353721394,
+ "tokens_seen": 1282080768
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941865596790371,
+ "loss": 1.2735,
+ "theoretical_loss": 3.5942786770333646,
+ "tokens_seen": 1282146304
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941705115346039,
+ "loss": 1.2348,
+ "theoretical_loss": 3.594262119777906,
+ "tokens_seen": 1282211840
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941544633901705,
+ "loss": 1.3056,
+ "theoretical_loss": 3.594245563605636,
+ "tokens_seen": 1282277376
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941384152457373,
+ "loss": 1.313,
+ "theoretical_loss": 3.594229008516429,
+ "tokens_seen": 1282342912
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941223671013039,
+ "loss": 1.2875,
+ "theoretical_loss": 3.594212454510158,
+ "tokens_seen": 1282408448
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004941063189568707,
+ "loss": 1.2558,
+ "theoretical_loss": 3.594195901586698,
+ "tokens_seen": 1282473984
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940902708124373,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5941793497459225,
+ "tokens_seen": 1282539520
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940742226680041,
+ "loss": 1.3036,
+ "theoretical_loss": 3.5941627989877047,
+ "tokens_seen": 1282605056
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940581745235707,
+ "loss": 1.3134,
+ "theoretical_loss": 3.594146249311919,
+ "tokens_seen": 1282670592
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940421263791375,
+ "loss": 1.4107,
+ "theoretical_loss": 3.59412970071844,
+ "tokens_seen": 1282736128
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940260782347041,
+ "loss": 1.3728,
+ "theoretical_loss": 3.59411315320714,
+ "tokens_seen": 1282801664
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004940100300902709,
+ "loss": 1.35,
+ "theoretical_loss": 3.594096606777894,
+ "tokens_seen": 1282867200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004939939819458375,
+ "loss": 1.3382,
+ "theoretical_loss": 3.594080061430576,
+ "tokens_seen": 1282932736
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004939779338014042,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5940635171650603,
+ "tokens_seen": 1282998272
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493961885656971,
+ "loss": 1.4137,
+ "theoretical_loss": 3.5940469739812198,
+ "tokens_seen": 1283063808
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004939458375125376,
+ "loss": 1.3062,
+ "theoretical_loss": 3.594030431878929,
+ "tokens_seen": 1283129344
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004939297893681043,
+ "loss": 1.2036,
+ "theoretical_loss": 3.5940138908580628,
+ "tokens_seen": 1283194880
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493913741223671,
+ "loss": 1.2865,
+ "theoretical_loss": 3.593997350918494,
+ "tokens_seen": 1283260416
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938976930792378,
+ "loss": 1.327,
+ "theoretical_loss": 3.5939808120600976,
+ "tokens_seen": 1283325952
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938816449348044,
+ "loss": 1.2471,
+ "theoretical_loss": 3.5939642742827473,
+ "tokens_seen": 1283391488
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938655967903712,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5939477375863174,
+ "tokens_seen": 1283457024
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938495486459378,
+ "loss": 1.2954,
+ "theoretical_loss": 3.593931201970682,
+ "tokens_seen": 1283522560
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938335005015044,
+ "loss": 1.3171,
+ "theoretical_loss": 3.5939146674357154,
+ "tokens_seen": 1283588096
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938174523570712,
+ "loss": 1.4015,
+ "theoretical_loss": 3.593898133981291,
+ "tokens_seen": 1283653632
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004938014042126378,
+ "loss": 1.4143,
+ "theoretical_loss": 3.5938816016072845,
+ "tokens_seen": 1283719168
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004937853560682046,
+ "loss": 1.3127,
+ "theoretical_loss": 3.5938650703135693,
+ "tokens_seen": 1283784704
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004937693079237712,
+ "loss": 1.2435,
+ "theoretical_loss": 3.5938485401000193,
+ "tokens_seen": 1283850240
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493753259779338,
+ "loss": 1.2403,
+ "theoretical_loss": 3.59383201096651,
+ "tokens_seen": 1283915776
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004937372116349046,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5938154829129143,
+ "tokens_seen": 1283981312
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004937211634904714,
+ "loss": 1.3221,
+ "theoretical_loss": 3.593798955939107,
+ "tokens_seen": 1284046848
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493705115346038,
+ "loss": 1.2331,
+ "theoretical_loss": 3.5937824300449632,
+ "tokens_seen": 1284112384
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004936890672016048,
+ "loss": 1.2967,
+ "theoretical_loss": 3.5937659052303563,
+ "tokens_seen": 1284177920
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004936730190571716,
+ "loss": 1.2442,
+ "theoretical_loss": 3.5937493814951615,
+ "tokens_seen": 1284243456
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004936569709127382,
+ "loss": 1.2047,
+ "theoretical_loss": 3.5937328588392523,
+ "tokens_seen": 1284308992
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493640922768305,
+ "loss": 1.2752,
+ "theoretical_loss": 3.593716337262504,
+ "tokens_seen": 1284374528
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004936248746238716,
+ "loss": 1.3132,
+ "theoretical_loss": 3.59369981676479,
+ "tokens_seen": 1284440064
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 431379,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4823201894760132,
+ "objective/train/theoretical_loss": 3.5936832973459865,
+ "objective/train/tokens_used": 1304965600,
+ "theoretical_loss": 3.5936832973459865,
+ "tokens_seen": 1284505600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004936088264794384,
+ "loss": 1.3724,
+ "theoretical_loss": 3.5936832973459865,
+ "tokens_seen": 1284505600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493592778335005,
+ "loss": 1.2995,
+ "theoretical_loss": 3.593666779005966,
+ "tokens_seen": 1284571136
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004935767301905718,
+ "loss": 1.3161,
+ "theoretical_loss": 3.5936502617446044,
+ "tokens_seen": 1284636672
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004935606820461384,
+ "loss": 1.2323,
+ "theoretical_loss": 3.593633745561776,
+ "tokens_seen": 1284702208
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004935446339017052,
+ "loss": 1.3966,
+ "theoretical_loss": 3.593617230457355,
+ "tokens_seen": 1284767744
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004935285857572718,
+ "loss": 1.3848,
+ "theoretical_loss": 3.593600716431216,
+ "tokens_seen": 1284833280
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004935125376128386,
+ "loss": 1.3259,
+ "theoretical_loss": 3.593584203483234,
+ "tokens_seen": 1284898816
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004934964894684052,
+ "loss": 1.4316,
+ "theoretical_loss": 3.5935676916132833,
+ "tokens_seen": 1284964352
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493480441323972,
+ "loss": 1.4408,
+ "theoretical_loss": 3.593551180821239,
+ "tokens_seen": 1285029888
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004934643931795386,
+ "loss": 1.2968,
+ "theoretical_loss": 3.593534671106975,
+ "tokens_seen": 1285095424
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004934483450351054,
+ "loss": 1.275,
+ "theoretical_loss": 3.5935181624703665,
+ "tokens_seen": 1285160960
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493432296890672,
+ "loss": 1.4033,
+ "theoretical_loss": 3.5935016549112877,
+ "tokens_seen": 1285226496
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004934162487462388,
+ "loss": 1.3205,
+ "theoretical_loss": 3.5934851484296146,
+ "tokens_seen": 1285292032
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004934002006018054,
+ "loss": 1.3356,
+ "theoretical_loss": 3.5934686430252203,
+ "tokens_seen": 1285357568
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004933841524573722,
+ "loss": 1.3887,
+ "theoretical_loss": 3.5934521386979807,
+ "tokens_seen": 1285423104
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004933681043129388,
+ "loss": 1.3009,
+ "theoretical_loss": 3.5934356354477703,
+ "tokens_seen": 1285488640
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004933520561685056,
+ "loss": 1.4457,
+ "theoretical_loss": 3.5934191332744634,
+ "tokens_seen": 1285554176
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004933360080240722,
+ "loss": 1.3302,
+ "theoretical_loss": 3.593402632177936,
+ "tokens_seen": 1285619712
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493319959879639,
+ "loss": 1.4391,
+ "theoretical_loss": 3.593386132158062,
+ "tokens_seen": 1285685248
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004933039117352056,
+ "loss": 1.2914,
+ "theoretical_loss": 3.5933696332147163,
+ "tokens_seen": 1285750784
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004932878635907724,
+ "loss": 1.4211,
+ "theoretical_loss": 3.593353135347774,
+ "tokens_seen": 1285816320
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493271815446339,
+ "loss": 1.3314,
+ "theoretical_loss": 3.59333663855711,
+ "tokens_seen": 1285881856
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004932557673019058,
+ "loss": 1.2129,
+ "theoretical_loss": 3.5933201428425994,
+ "tokens_seen": 1285947392
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004932397191574724,
+ "loss": 1.4659,
+ "theoretical_loss": 3.5933036482041167,
+ "tokens_seen": 1286012928
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004932236710130392,
+ "loss": 1.4073,
+ "theoretical_loss": 3.593287154641538,
+ "tokens_seen": 1286078464
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004932076228686058,
+ "loss": 1.3336,
+ "theoretical_loss": 3.593270662154737,
+ "tokens_seen": 1286144000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004931915747241726,
+ "loss": 1.2649,
+ "theoretical_loss": 3.593254170743589,
+ "tokens_seen": 1286209536
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004931755265797392,
+ "loss": 1.2477,
+ "theoretical_loss": 3.5932376804079698,
+ "tokens_seen": 1286275072
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493159478435306,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5932211911477534,
+ "tokens_seen": 1286340608
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004931434302908726,
+ "loss": 1.2927,
+ "theoretical_loss": 3.593204702962816,
+ "tokens_seen": 1286406144
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004931273821464394,
+ "loss": 1.3555,
+ "theoretical_loss": 3.593188215853032,
+ "tokens_seen": 1286471680
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000493111334002006,
+ "loss": 1.3249,
+ "theoretical_loss": 3.593171729818276,
+ "tokens_seen": 1286537216
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930952858575728,
+ "loss": 1.2865,
+ "theoretical_loss": 3.5931552448584245,
+ "tokens_seen": 1286602752
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930792377131394,
+ "loss": 1.3145,
+ "theoretical_loss": 3.5931387609733516,
+ "tokens_seen": 1286668288
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930631895687062,
+ "loss": 1.4134,
+ "theoretical_loss": 3.5931222781629333,
+ "tokens_seen": 1286733824
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930471414242728,
+ "loss": 1.3083,
+ "theoretical_loss": 3.593105796427044,
+ "tokens_seen": 1286799360
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930310932798396,
+ "loss": 1.3091,
+ "theoretical_loss": 3.5930893157655595,
+ "tokens_seen": 1286864896
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004930150451354062,
+ "loss": 1.3026,
+ "theoretical_loss": 3.593072836178355,
+ "tokens_seen": 1286930432
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000492998996990973,
+ "loss": 1.3357,
+ "theoretical_loss": 3.593056357665305,
+ "tokens_seen": 1286995968
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004929829488465396,
+ "loss": 1.502,
+ "theoretical_loss": 3.5930398802262857,
+ "tokens_seen": 1287061504
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004929669007021064,
+ "loss": 1.2645,
+ "theoretical_loss": 3.5930234038611726,
+ "tokens_seen": 1287127040
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000492950852557673,
+ "loss": 1.4383,
+ "theoretical_loss": 3.59300692856984,
+ "tokens_seen": 1287192576
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004929348044132398,
+ "loss": 1.4961,
+ "theoretical_loss": 3.5929904543521642,
+ "tokens_seen": 1287258112
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004929187562688064,
+ "loss": 1.3086,
+ "theoretical_loss": 3.59297398120802,
+ "tokens_seen": 1287323648
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004929027081243732,
+ "loss": 1.3873,
+ "theoretical_loss": 3.5929575091372827,
+ "tokens_seen": 1287389184
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004928866599799398,
+ "loss": 1.4399,
+ "theoretical_loss": 3.592941038139829,
+ "tokens_seen": 1287454720
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004928706118355066,
+ "loss": 1.5055,
+ "theoretical_loss": 3.5929245682155324,
+ "tokens_seen": 1287520256
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004928545636910732,
+ "loss": 1.3475,
+ "theoretical_loss": 3.5929080993642692,
+ "tokens_seen": 1287585792
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.00049283851554664,
+ "loss": 1.3472,
+ "theoretical_loss": 3.5928916315859154,
+ "tokens_seen": 1287651328
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004928224674022066,
+ "loss": 1.2471,
+ "theoretical_loss": 3.592875164880346,
+ "tokens_seen": 1287716864
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 432272,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3189022541046143,
+ "objective/train/theoretical_loss": 3.592858699247437,
+ "objective/train/tokens_used": 1308242400,
+ "theoretical_loss": 3.592858699247437,
+ "tokens_seen": 1287782400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004928064192577734,
+ "loss": 1.3203,
+ "theoretical_loss": 3.592858699247437,
+ "tokens_seen": 1287782400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.00049279037111334,
+ "loss": 1.3671,
+ "theoretical_loss": 3.592842234687063,
+ "tokens_seen": 1287847936
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004927743229689068,
+ "loss": 1.3745,
+ "theoretical_loss": 3.5928257711991005,
+ "tokens_seen": 1287913472
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004927582748244734,
+ "loss": 1.2851,
+ "theoretical_loss": 3.5928093087834245,
+ "tokens_seen": 1287979008
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004927422266800402,
+ "loss": 1.2529,
+ "theoretical_loss": 3.592792847439911,
+ "tokens_seen": 1288044544
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004927261785356068,
+ "loss": 1.3913,
+ "theoretical_loss": 3.5927763871684353,
+ "tokens_seen": 1288110080
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004927101303911736,
+ "loss": 1.3012,
+ "theoretical_loss": 3.5927599279688733,
+ "tokens_seen": 1288175616
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004926940822467402,
+ "loss": 1.3546,
+ "theoretical_loss": 3.5927434698411003,
+ "tokens_seen": 1288241152
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000492678034102307,
+ "loss": 1.318,
+ "theoretical_loss": 3.5927270127849926,
+ "tokens_seen": 1288306688
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004926619859578736,
+ "loss": 1.385,
+ "theoretical_loss": 3.592710556800425,
+ "tokens_seen": 1288372224
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004926459378134404,
+ "loss": 1.2928,
+ "theoretical_loss": 3.5926941018872744,
+ "tokens_seen": 1288437760
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000492629889669007,
+ "loss": 1.3409,
+ "theoretical_loss": 3.5926776480454157,
+ "tokens_seen": 1288503296
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004926138415245738,
+ "loss": 1.4072,
+ "theoretical_loss": 3.5926611952747245,
+ "tokens_seen": 1288568832
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925977933801404,
+ "loss": 1.3322,
+ "theoretical_loss": 3.5926447435750775,
+ "tokens_seen": 1288634368
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925817452357071,
+ "loss": 1.3475,
+ "theoretical_loss": 3.5926282929463498,
+ "tokens_seen": 1288699904
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925656970912738,
+ "loss": 1.3439,
+ "theoretical_loss": 3.5926118433884175,
+ "tokens_seen": 1288765440
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925496489468405,
+ "loss": 1.4001,
+ "theoretical_loss": 3.5925953949011564,
+ "tokens_seen": 1288830976
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925336008024073,
+ "loss": 1.307,
+ "theoretical_loss": 3.5925789474844425,
+ "tokens_seen": 1288896512
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925175526579739,
+ "loss": 1.3581,
+ "theoretical_loss": 3.5925625011381515,
+ "tokens_seen": 1288962048
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004925015045135407,
+ "loss": 1.408,
+ "theoretical_loss": 3.592546055862159,
+ "tokens_seen": 1289027584
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004924854563691073,
+ "loss": 1.3784,
+ "theoretical_loss": 3.5925296116563414,
+ "tokens_seen": 1289093120
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000492469408224674,
+ "loss": 1.4887,
+ "theoretical_loss": 3.5925131685205747,
+ "tokens_seen": 1289158656
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004924533600802407,
+ "loss": 1.4219,
+ "theoretical_loss": 3.592496726454735,
+ "tokens_seen": 1289224192
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004924373119358075,
+ "loss": 1.462,
+ "theoretical_loss": 3.5924802854586977,
+ "tokens_seen": 1289289728
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004924212637913741,
+ "loss": 1.3384,
+ "theoretical_loss": 3.592463845532339,
+ "tokens_seen": 1289355264
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004924052156469409,
+ "loss": 1.2879,
+ "theoretical_loss": 3.592447406675536,
+ "tokens_seen": 1289420800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923891675025075,
+ "loss": 1.3606,
+ "theoretical_loss": 3.592430968888163,
+ "tokens_seen": 1289486336
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923731193580743,
+ "loss": 1.2404,
+ "theoretical_loss": 3.592414532170097,
+ "tokens_seen": 1289551872
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923570712136409,
+ "loss": 1.2948,
+ "theoretical_loss": 3.5923980965212143,
+ "tokens_seen": 1289617408
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923410230692077,
+ "loss": 1.4047,
+ "theoretical_loss": 3.5923816619413906,
+ "tokens_seen": 1289682944
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923249749247743,
+ "loss": 1.4074,
+ "theoretical_loss": 3.5923652284305025,
+ "tokens_seen": 1289748480
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004923089267803411,
+ "loss": 1.3486,
+ "theoretical_loss": 3.5923487959884257,
+ "tokens_seen": 1289814016
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922928786359077,
+ "loss": 1.357,
+ "theoretical_loss": 3.5923323646150367,
+ "tokens_seen": 1289879552
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922768304914745,
+ "loss": 1.2946,
+ "theoretical_loss": 3.5923159343102116,
+ "tokens_seen": 1289945088
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922607823470411,
+ "loss": 1.3531,
+ "theoretical_loss": 3.592299505073826,
+ "tokens_seen": 1290010624
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922447342026079,
+ "loss": 1.3608,
+ "theoretical_loss": 3.5922830769057574,
+ "tokens_seen": 1290076160
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922286860581745,
+ "loss": 1.3382,
+ "theoretical_loss": 3.592266649805881,
+ "tokens_seen": 1290141696
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004922126379137413,
+ "loss": 1.3103,
+ "theoretical_loss": 3.592250223774074,
+ "tokens_seen": 1290207232
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921965897693079,
+ "loss": 1.3596,
+ "theoretical_loss": 3.592233798810212,
+ "tokens_seen": 1290272768
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921805416248747,
+ "loss": 1.4379,
+ "theoretical_loss": 3.592217374914171,
+ "tokens_seen": 1290338304
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921644934804413,
+ "loss": 1.4919,
+ "theoretical_loss": 3.5922009520858285,
+ "tokens_seen": 1290403840
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921484453360081,
+ "loss": 1.3597,
+ "theoretical_loss": 3.59218453032506,
+ "tokens_seen": 1290469376
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921323971915747,
+ "loss": 1.2389,
+ "theoretical_loss": 3.592168109631742,
+ "tokens_seen": 1290534912
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921163490471415,
+ "loss": 1.5207,
+ "theoretical_loss": 3.5921516900057515,
+ "tokens_seen": 1290600448
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004921003009027081,
+ "loss": 1.2722,
+ "theoretical_loss": 3.5921352714469643,
+ "tokens_seen": 1290665984
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920842527582749,
+ "loss": 1.3502,
+ "theoretical_loss": 3.592118853955257,
+ "tokens_seen": 1290731520
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920682046138415,
+ "loss": 1.2804,
+ "theoretical_loss": 3.592102437530506,
+ "tokens_seen": 1290797056
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920521564694083,
+ "loss": 1.2639,
+ "theoretical_loss": 3.5920860221725883,
+ "tokens_seen": 1290862592
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920361083249749,
+ "loss": 1.4943,
+ "theoretical_loss": 3.5920696078813794,
+ "tokens_seen": 1290928128
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920200601805417,
+ "loss": 1.4752,
+ "theoretical_loss": 3.592053194656757,
+ "tokens_seen": 1290993664
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 433657,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2094658613204956,
+ "objective/train/theoretical_loss": 3.592036782498597,
+ "objective/train/tokens_used": 1311519200,
+ "theoretical_loss": 3.592036782498597,
+ "tokens_seen": 1291059200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004920040120361083,
+ "loss": 1.3495,
+ "theoretical_loss": 3.592036782498597,
+ "tokens_seen": 1291059200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919879638916751,
+ "loss": 1.271,
+ "theoretical_loss": 3.5920203714067758,
+ "tokens_seen": 1291124736
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919719157472417,
+ "loss": 1.3186,
+ "theoretical_loss": 3.5920039613811703,
+ "tokens_seen": 1291190272
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919558676028085,
+ "loss": 1.3833,
+ "theoretical_loss": 3.5919875524216573,
+ "tokens_seen": 1291255808
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919398194583751,
+ "loss": 1.3534,
+ "theoretical_loss": 3.591971144528113,
+ "tokens_seen": 1291321344
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919237713139419,
+ "loss": 1.3739,
+ "theoretical_loss": 3.591954737700415,
+ "tokens_seen": 1291386880
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004919077231695085,
+ "loss": 1.3191,
+ "theoretical_loss": 3.5919383319384384,
+ "tokens_seen": 1291452416
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918916750250753,
+ "loss": 1.5903,
+ "theoretical_loss": 3.591921927242061,
+ "tokens_seen": 1291517952
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918756268806419,
+ "loss": 1.342,
+ "theoretical_loss": 3.5919055236111594,
+ "tokens_seen": 1291583488
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918595787362087,
+ "loss": 1.338,
+ "theoretical_loss": 3.5918891210456105,
+ "tokens_seen": 1291649024
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918435305917753,
+ "loss": 1.467,
+ "theoretical_loss": 3.5918727195452904,
+ "tokens_seen": 1291714560
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918274824473421,
+ "loss": 1.2794,
+ "theoretical_loss": 3.5918563191100765,
+ "tokens_seen": 1291780096
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004918114343029087,
+ "loss": 1.1941,
+ "theoretical_loss": 3.5918399197398454,
+ "tokens_seen": 1291845632
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917953861584755,
+ "loss": 1.328,
+ "theoretical_loss": 3.591823521434474,
+ "tokens_seen": 1291911168
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917793380140421,
+ "loss": 1.311,
+ "theoretical_loss": 3.5918071241938385,
+ "tokens_seen": 1291976704
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917632898696089,
+ "loss": 1.322,
+ "theoretical_loss": 3.5917907280178167,
+ "tokens_seen": 1292042240
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917472417251755,
+ "loss": 1.2841,
+ "theoretical_loss": 3.5917743329062852,
+ "tokens_seen": 1292107776
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917311935807423,
+ "loss": 1.3337,
+ "theoretical_loss": 3.5917579388591205,
+ "tokens_seen": 1292173312
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004917151454363089,
+ "loss": 1.4893,
+ "theoretical_loss": 3.5917415458762,
+ "tokens_seen": 1292238848
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916990972918757,
+ "loss": 1.3748,
+ "theoretical_loss": 3.5917251539574004,
+ "tokens_seen": 1292304384
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916830491474423,
+ "loss": 1.3087,
+ "theoretical_loss": 3.5917087631025986,
+ "tokens_seen": 1292369920
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916670010030091,
+ "loss": 1.3229,
+ "theoretical_loss": 3.591692373311672,
+ "tokens_seen": 1292435456
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916509528585757,
+ "loss": 1.2367,
+ "theoretical_loss": 3.591675984584497,
+ "tokens_seen": 1292500992
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916349047141425,
+ "loss": 1.3359,
+ "theoretical_loss": 3.591659596920951,
+ "tokens_seen": 1292566528
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916188565697091,
+ "loss": 1.4251,
+ "theoretical_loss": 3.5916432103209113,
+ "tokens_seen": 1292632064
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004916028084252759,
+ "loss": 1.3154,
+ "theoretical_loss": 3.591626824784255,
+ "tokens_seen": 1292697600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915867602808425,
+ "loss": 1.4047,
+ "theoretical_loss": 3.5916104403108577,
+ "tokens_seen": 1292763136
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915707121364093,
+ "loss": 1.3264,
+ "theoretical_loss": 3.5915940569005986,
+ "tokens_seen": 1292828672
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915546639919759,
+ "loss": 1.4677,
+ "theoretical_loss": 3.5915776745533536,
+ "tokens_seen": 1292894208
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915386158475427,
+ "loss": 1.2761,
+ "theoretical_loss": 3.5915612932689998,
+ "tokens_seen": 1292959744
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915225677031093,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5915449130474153,
+ "tokens_seen": 1293025280
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004915065195586761,
+ "loss": 1.3004,
+ "theoretical_loss": 3.591528533888476,
+ "tokens_seen": 1293090816
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914904714142427,
+ "loss": 1.3187,
+ "theoretical_loss": 3.5915121557920604,
+ "tokens_seen": 1293156352
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914744232698095,
+ "loss": 1.3155,
+ "theoretical_loss": 3.591495778758045,
+ "tokens_seen": 1293221888
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914583751253761,
+ "loss": 1.3298,
+ "theoretical_loss": 3.591479402786307,
+ "tokens_seen": 1293287424
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914423269809429,
+ "loss": 1.3217,
+ "theoretical_loss": 3.591463027876724,
+ "tokens_seen": 1293352960
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914262788365095,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5914466540291734,
+ "tokens_seen": 1293418496
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004914102306920763,
+ "loss": 1.3561,
+ "theoretical_loss": 3.5914302812435315,
+ "tokens_seen": 1293484032
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913941825476429,
+ "loss": 1.3903,
+ "theoretical_loss": 3.591413909519677,
+ "tokens_seen": 1293549568
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913781344032097,
+ "loss": 1.4476,
+ "theoretical_loss": 3.5913975388574864,
+ "tokens_seen": 1293615104
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913620862587763,
+ "loss": 1.3465,
+ "theoretical_loss": 3.591381169256838,
+ "tokens_seen": 1293680640
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913460381143431,
+ "loss": 1.3507,
+ "theoretical_loss": 3.5913648007176078,
+ "tokens_seen": 1293746176
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913299899699097,
+ "loss": 1.2841,
+ "theoretical_loss": 3.5913484332396743,
+ "tokens_seen": 1293811712
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004913139418254765,
+ "loss": 1.3726,
+ "theoretical_loss": 3.5913320668229143,
+ "tokens_seen": 1293877248
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912978936810431,
+ "loss": 1.2302,
+ "theoretical_loss": 3.5913157014672055,
+ "tokens_seen": 1293942784
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912818455366099,
+ "loss": 1.3879,
+ "theoretical_loss": 3.5912993371724258,
+ "tokens_seen": 1294008320
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912657973921765,
+ "loss": 1.3029,
+ "theoretical_loss": 3.591282973938452,
+ "tokens_seen": 1294073856
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912497492477433,
+ "loss": 1.2827,
+ "theoretical_loss": 3.591266611765162,
+ "tokens_seen": 1294139392
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.00049123370110331,
+ "loss": 1.3735,
+ "theoretical_loss": 3.5912502506524335,
+ "tokens_seen": 1294204928
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912176529588767,
+ "loss": 1.4038,
+ "theoretical_loss": 3.5912338906001433,
+ "tokens_seen": 1294270464
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 434508,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4272451400756836,
+ "objective/train/theoretical_loss": 3.5912175316081694,
+ "objective/train/tokens_used": 1314796000,
+ "theoretical_loss": 3.5912175316081694,
+ "tokens_seen": 1294336000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004912016048144433,
+ "loss": 1.45,
+ "theoretical_loss": 3.5912175316081694,
+ "tokens_seen": 1294336000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911855566700101,
+ "loss": 1.3144,
+ "theoretical_loss": 3.59120117367639,
+ "tokens_seen": 1294401536
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911695085255767,
+ "loss": 1.2642,
+ "theoretical_loss": 3.591184816804682,
+ "tokens_seen": 1294467072
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911534603811434,
+ "loss": 1.376,
+ "theoretical_loss": 3.5911684609929235,
+ "tokens_seen": 1294532608
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911374122367102,
+ "loss": 1.4699,
+ "theoretical_loss": 3.5911521062409912,
+ "tokens_seen": 1294598144
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911213640922768,
+ "loss": 1.2568,
+ "theoretical_loss": 3.591135752548764,
+ "tokens_seen": 1294663680
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004911053159478436,
+ "loss": 1.3599,
+ "theoretical_loss": 3.591119399916119,
+ "tokens_seen": 1294729216
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004910892678034102,
+ "loss": 1.324,
+ "theoretical_loss": 3.5911030483429345,
+ "tokens_seen": 1294794752
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000491073219658977,
+ "loss": 1.3569,
+ "theoretical_loss": 3.5910866978290867,
+ "tokens_seen": 1294860288
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004910571715145436,
+ "loss": 1.2693,
+ "theoretical_loss": 3.5910703483744553,
+ "tokens_seen": 1294925824
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004910411233701104,
+ "loss": 1.4145,
+ "theoretical_loss": 3.591053999978917,
+ "tokens_seen": 1294991360
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000491025075225677,
+ "loss": 1.2988,
+ "theoretical_loss": 3.591037652642349,
+ "tokens_seen": 1295056896
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004910090270812438,
+ "loss": 1.2822,
+ "theoretical_loss": 3.591021306364631,
+ "tokens_seen": 1295122432
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004909929789368104,
+ "loss": 1.2971,
+ "theoretical_loss": 3.5910049611456394,
+ "tokens_seen": 1295187968
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004909769307923772,
+ "loss": 1.2812,
+ "theoretical_loss": 3.590988616985252,
+ "tokens_seen": 1295253504
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004909608826479438,
+ "loss": 1.3704,
+ "theoretical_loss": 3.5909722738833474,
+ "tokens_seen": 1295319040
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004909448345035106,
+ "loss": 1.4753,
+ "theoretical_loss": 3.590955931839803,
+ "tokens_seen": 1295384576
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004909287863590772,
+ "loss": 1.4235,
+ "theoretical_loss": 3.5909395908544974,
+ "tokens_seen": 1295450112
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490912738214644,
+ "loss": 1.356,
+ "theoretical_loss": 3.5909232509273075,
+ "tokens_seen": 1295515648
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908966900702106,
+ "loss": 1.3655,
+ "theoretical_loss": 3.590906912058112,
+ "tokens_seen": 1295581184
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908806419257774,
+ "loss": 1.335,
+ "theoretical_loss": 3.5908905742467887,
+ "tokens_seen": 1295646720
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490864593781344,
+ "loss": 1.31,
+ "theoretical_loss": 3.5908742374932157,
+ "tokens_seen": 1295712256
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908485456369108,
+ "loss": 1.5038,
+ "theoretical_loss": 3.590857901797271,
+ "tokens_seen": 1295777792
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908324974924774,
+ "loss": 1.4252,
+ "theoretical_loss": 3.590841567158832,
+ "tokens_seen": 1295843328
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908164493480442,
+ "loss": 1.3494,
+ "theoretical_loss": 3.590825233577778,
+ "tokens_seen": 1295908864
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004908004012036108,
+ "loss": 1.4036,
+ "theoretical_loss": 3.590808901053986,
+ "tokens_seen": 1295974400
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004907843530591776,
+ "loss": 1.3434,
+ "theoretical_loss": 3.590792569587335,
+ "tokens_seen": 1296039936
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004907683049147442,
+ "loss": 1.3556,
+ "theoretical_loss": 3.5907762391777025,
+ "tokens_seen": 1296105472
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490752256770311,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5907599098249667,
+ "tokens_seen": 1296171008
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004907362086258776,
+ "loss": 1.2789,
+ "theoretical_loss": 3.5907435815290056,
+ "tokens_seen": 1296236544
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004907201604814444,
+ "loss": 1.2265,
+ "theoretical_loss": 3.590727254289698,
+ "tokens_seen": 1296302080
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490704112337011,
+ "loss": 1.5031,
+ "theoretical_loss": 3.5907109281069216,
+ "tokens_seen": 1296367616
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906880641925778,
+ "loss": 1.27,
+ "theoretical_loss": 3.5906946029805544,
+ "tokens_seen": 1296433152
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906720160481444,
+ "loss": 1.3972,
+ "theoretical_loss": 3.5906782789104756,
+ "tokens_seen": 1296498688
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906559679037112,
+ "loss": 1.3149,
+ "theoretical_loss": 3.5906619558965627,
+ "tokens_seen": 1296564224
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906399197592778,
+ "loss": 1.2508,
+ "theoretical_loss": 3.5906456339386943,
+ "tokens_seen": 1296629760
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906238716148446,
+ "loss": 1.3477,
+ "theoretical_loss": 3.5906293130367484,
+ "tokens_seen": 1296695296
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004906078234704112,
+ "loss": 1.4802,
+ "theoretical_loss": 3.5906129931906037,
+ "tokens_seen": 1296760832
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490591775325978,
+ "loss": 1.2791,
+ "theoretical_loss": 3.590596674400138,
+ "tokens_seen": 1296826368
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004905757271815446,
+ "loss": 1.2481,
+ "theoretical_loss": 3.590580356665231,
+ "tokens_seen": 1296891904
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004905596790371114,
+ "loss": 1.3494,
+ "theoretical_loss": 3.5905640399857583,
+ "tokens_seen": 1296957440
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490543630892678,
+ "loss": 1.3892,
+ "theoretical_loss": 3.5905477243616017,
+ "tokens_seen": 1297022976
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004905275827482448,
+ "loss": 1.257,
+ "theoretical_loss": 3.5905314097926375,
+ "tokens_seen": 1297088512
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004905115346038114,
+ "loss": 1.3527,
+ "theoretical_loss": 3.590515096278745,
+ "tokens_seen": 1297154048
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004904954864593782,
+ "loss": 1.4653,
+ "theoretical_loss": 3.5904987838198017,
+ "tokens_seen": 1297219584
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004904794383149448,
+ "loss": 1.4053,
+ "theoretical_loss": 3.5904824724156876,
+ "tokens_seen": 1297285120
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004904633901705116,
+ "loss": 1.2649,
+ "theoretical_loss": 3.5904661620662797,
+ "tokens_seen": 1297350656
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004904473420260782,
+ "loss": 1.3664,
+ "theoretical_loss": 3.5904498527714575,
+ "tokens_seen": 1297416192
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490431293881645,
+ "loss": 1.4184,
+ "theoretical_loss": 3.590433544531099,
+ "tokens_seen": 1297481728
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004904152457372116,
+ "loss": 1.3453,
+ "theoretical_loss": 3.590417237345083,
+ "tokens_seen": 1297547264
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 435770,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4622042179107666,
+ "objective/train/theoretical_loss": 3.5904009312132885,
+ "objective/train/tokens_used": 1318072800,
+ "theoretical_loss": 3.5904009312132885,
+ "tokens_seen": 1297612800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903991975927784,
+ "loss": 1.3757,
+ "theoretical_loss": 3.5904009312132885,
+ "tokens_seen": 1297612800
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490383149448345,
+ "loss": 1.449,
+ "theoretical_loss": 3.5903846261355934,
+ "tokens_seen": 1297678336
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903671013039118,
+ "loss": 1.3038,
+ "theoretical_loss": 3.5903683221118765,
+ "tokens_seen": 1297743872
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903510531594784,
+ "loss": 1.3635,
+ "theoretical_loss": 3.5903520191420166,
+ "tokens_seen": 1297809408
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903350050150452,
+ "loss": 1.4401,
+ "theoretical_loss": 3.5903357172258925,
+ "tokens_seen": 1297874944
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903189568706118,
+ "loss": 1.2792,
+ "theoretical_loss": 3.590319416363383,
+ "tokens_seen": 1297940480
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004903029087261786,
+ "loss": 1.4366,
+ "theoretical_loss": 3.5903031165543657,
+ "tokens_seen": 1298006016
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004902868605817452,
+ "loss": 1.3782,
+ "theoretical_loss": 3.590286817798721,
+ "tokens_seen": 1298071552
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490270812437312,
+ "loss": 1.376,
+ "theoretical_loss": 3.5902705200963263,
+ "tokens_seen": 1298137088
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004902547642928786,
+ "loss": 1.428,
+ "theoretical_loss": 3.5902542234470616,
+ "tokens_seen": 1298202624
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004902387161484454,
+ "loss": 1.3626,
+ "theoretical_loss": 3.5902379278508043,
+ "tokens_seen": 1298268160
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490222668004012,
+ "loss": 1.3293,
+ "theoretical_loss": 3.590221633307434,
+ "tokens_seen": 1298333696
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004902066198595788,
+ "loss": 1.2862,
+ "theoretical_loss": 3.59020533981683,
+ "tokens_seen": 1298399232
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004901905717151454,
+ "loss": 1.2682,
+ "theoretical_loss": 3.59018904737887,
+ "tokens_seen": 1298464768
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004901745235707122,
+ "loss": 1.3294,
+ "theoretical_loss": 3.590172755993433,
+ "tokens_seen": 1298530304
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004901584754262788,
+ "loss": 1.3853,
+ "theoretical_loss": 3.5901564656603995,
+ "tokens_seen": 1298595840
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004901424272818456,
+ "loss": 1.4211,
+ "theoretical_loss": 3.5901401763796468,
+ "tokens_seen": 1298661376
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004901263791374122,
+ "loss": 1.4769,
+ "theoretical_loss": 3.5901238881510538,
+ "tokens_seen": 1298726912
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000490110330992979,
+ "loss": 1.4099,
+ "theoretical_loss": 3.5901076009745,
+ "tokens_seen": 1298792448
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900942828485456,
+ "loss": 1.3624,
+ "theoretical_loss": 3.5900913148498645,
+ "tokens_seen": 1298857984
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900782347041124,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5900750297770267,
+ "tokens_seen": 1298923520
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900621865596791,
+ "loss": 1.4171,
+ "theoretical_loss": 3.590058745755864,
+ "tokens_seen": 1298989056
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900461384152458,
+ "loss": 1.2757,
+ "theoretical_loss": 3.590042462786257,
+ "tokens_seen": 1299054592
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900300902708125,
+ "loss": 1.4112,
+ "theoretical_loss": 3.590026180868084,
+ "tokens_seen": 1299120128
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004900140421263792,
+ "loss": 1.2693,
+ "theoretical_loss": 3.5900099000012244,
+ "tokens_seen": 1299185664
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000489997993981946,
+ "loss": 1.3372,
+ "theoretical_loss": 3.589993620185557,
+ "tokens_seen": 1299251200
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004899819458375126,
+ "loss": 1.3898,
+ "theoretical_loss": 3.589977341420961,
+ "tokens_seen": 1299316736
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004899658976930794,
+ "loss": 1.3904,
+ "theoretical_loss": 3.5899610637073156,
+ "tokens_seen": 1299382272
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000489949849548646,
+ "loss": 1.3679,
+ "theoretical_loss": 3.5899447870445,
+ "tokens_seen": 1299447808
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004899338014042128,
+ "loss": 1.2966,
+ "theoretical_loss": 3.589928511432393,
+ "tokens_seen": 1299513344
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004899177532597794,
+ "loss": 1.333,
+ "theoretical_loss": 3.5899122368708745,
+ "tokens_seen": 1299578880
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000489901705115346,
+ "loss": 1.2978,
+ "theoretical_loss": 3.589895963359823,
+ "tokens_seen": 1299644416
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898856569709128,
+ "loss": 1.4422,
+ "theoretical_loss": 3.589879690899118,
+ "tokens_seen": 1299709952
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898696088264794,
+ "loss": 1.3692,
+ "theoretical_loss": 3.5898634194886387,
+ "tokens_seen": 1299775488
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898535606820462,
+ "loss": 1.3866,
+ "theoretical_loss": 3.589847149128265,
+ "tokens_seen": 1299841024
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898375125376128,
+ "loss": 1.4209,
+ "theoretical_loss": 3.5898308798178755,
+ "tokens_seen": 1299906560
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898214643931796,
+ "loss": 1.4354,
+ "theoretical_loss": 3.5898146115573493,
+ "tokens_seen": 1299972096
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004898054162487462,
+ "loss": 1.359,
+ "theoretical_loss": 3.5897983443465664,
+ "tokens_seen": 1300037632
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000489789368104313,
+ "loss": 1.4943,
+ "theoretical_loss": 3.5897820781854053,
+ "tokens_seen": 1300103168
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004897733199598797,
+ "loss": 1.3495,
+ "theoretical_loss": 3.589765813073746,
+ "tokens_seen": 1300168704
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004897572718154463,
+ "loss": 1.3841,
+ "theoretical_loss": 3.5897495490114686,
+ "tokens_seen": 1300234240
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.000489741223671013,
+ "loss": 1.4819,
+ "theoretical_loss": 3.5897332859984505,
+ "tokens_seen": 1300299776
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004897251755265797,
+ "loss": 1.2692,
+ "theoretical_loss": 3.5897170240345733,
+ "tokens_seen": 1300365312
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004897091273821465,
+ "loss": 1.3227,
+ "theoretical_loss": 3.589700763119715,
+ "tokens_seen": 1300430848
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896930792377131,
+ "loss": 1.2962,
+ "theoretical_loss": 3.589684503253755,
+ "tokens_seen": 1300496384
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896770310932799,
+ "loss": 1.3879,
+ "theoretical_loss": 3.5896682444365746,
+ "tokens_seen": 1300561920
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896609829488465,
+ "loss": 1.3779,
+ "theoretical_loss": 3.589651986668051,
+ "tokens_seen": 1300627456
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896449348044133,
+ "loss": 1.3759,
+ "theoretical_loss": 3.5896357299480655,
+ "tokens_seen": 1300692992
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896288866599799,
+ "loss": 1.2914,
+ "theoretical_loss": 3.5896194742764966,
+ "tokens_seen": 1300758528
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004896128385155467,
+ "loss": 1.3408,
+ "theoretical_loss": 3.5896032196532244,
+ "tokens_seen": 1300824064
+ },
+ {
+ "epoch": 0.39,
+ "objective/train/docs_used": 436727,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.392486810684204,
+ "objective/train/theoretical_loss": 3.589586966078128,
+ "objective/train/tokens_used": 1321349600,
+ "theoretical_loss": 3.589586966078128,
+ "tokens_seen": 1300889600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895967903711133,
+ "loss": 1.2775,
+ "theoretical_loss": 3.589586966078128,
+ "tokens_seen": 1300889600
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895807422266801,
+ "loss": 1.3813,
+ "theoretical_loss": 3.5895707135510873,
+ "tokens_seen": 1300955136
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895646940822467,
+ "loss": 1.3381,
+ "theoretical_loss": 3.589554462071982,
+ "tokens_seen": 1301020672
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895486459378135,
+ "loss": 1.3777,
+ "theoretical_loss": 3.5895382116406918,
+ "tokens_seen": 1301086208
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895325977933801,
+ "loss": 1.4776,
+ "theoretical_loss": 3.5895219622570957,
+ "tokens_seen": 1301151744
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895165496489469,
+ "loss": 1.3612,
+ "theoretical_loss": 3.589505713921074,
+ "tokens_seen": 1301217280
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004895005015045135,
+ "loss": 1.358,
+ "theoretical_loss": 3.5894894666325072,
+ "tokens_seen": 1301282816
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894844533600803,
+ "loss": 1.3097,
+ "theoretical_loss": 3.5894732203912736,
+ "tokens_seen": 1301348352
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894684052156469,
+ "loss": 1.4728,
+ "theoretical_loss": 3.5894569751972534,
+ "tokens_seen": 1301413888
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894523570712137,
+ "loss": 1.5459,
+ "theoretical_loss": 3.5894407310503267,
+ "tokens_seen": 1301479424
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894363089267803,
+ "loss": 1.2955,
+ "theoretical_loss": 3.5894244879503727,
+ "tokens_seen": 1301544960
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894202607823471,
+ "loss": 1.3677,
+ "theoretical_loss": 3.589408245897272,
+ "tokens_seen": 1301610496
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004894042126379137,
+ "loss": 1.3244,
+ "theoretical_loss": 3.589392004890904,
+ "tokens_seen": 1301676032
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893881644934805,
+ "loss": 1.358,
+ "theoretical_loss": 3.589375764931148,
+ "tokens_seen": 1301741568
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893721163490471,
+ "loss": 1.3236,
+ "theoretical_loss": 3.5893595260178843,
+ "tokens_seen": 1301807104
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893560682046139,
+ "loss": 1.4503,
+ "theoretical_loss": 3.5893432881509937,
+ "tokens_seen": 1301872640
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893400200601805,
+ "loss": 1.2277,
+ "theoretical_loss": 3.589327051330355,
+ "tokens_seen": 1301938176
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893239719157473,
+ "loss": 1.2936,
+ "theoretical_loss": 3.589310815555848,
+ "tokens_seen": 1302003712
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004893079237713139,
+ "loss": 1.2937,
+ "theoretical_loss": 3.589294580827354,
+ "tokens_seen": 1302069248
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892918756268807,
+ "loss": 1.3151,
+ "theoretical_loss": 3.5892783471447514,
+ "tokens_seen": 1302134784
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892758274824473,
+ "loss": 1.3537,
+ "theoretical_loss": 3.5892621145079207,
+ "tokens_seen": 1302200320
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892597793380141,
+ "loss": 1.2823,
+ "theoretical_loss": 3.5892458829167424,
+ "tokens_seen": 1302265856
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892437311935807,
+ "loss": 1.4119,
+ "theoretical_loss": 3.589229652371096,
+ "tokens_seen": 1302331392
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892276830491475,
+ "loss": 1.3613,
+ "theoretical_loss": 3.589213422870862,
+ "tokens_seen": 1302396928
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004892116349047141,
+ "loss": 1.3079,
+ "theoretical_loss": 3.5891971944159193,
+ "tokens_seen": 1302462464
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891955867602809,
+ "loss": 1.2965,
+ "theoretical_loss": 3.5891809670061496,
+ "tokens_seen": 1302528000
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891795386158475,
+ "loss": 1.271,
+ "theoretical_loss": 3.589164740641432,
+ "tokens_seen": 1302593536
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891634904714143,
+ "loss": 1.2464,
+ "theoretical_loss": 3.589148515321647,
+ "tokens_seen": 1302659072
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891474423269809,
+ "loss": 1.3946,
+ "theoretical_loss": 3.589132291046675,
+ "tokens_seen": 1302724608
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891313941825477,
+ "loss": 1.3117,
+ "theoretical_loss": 3.589116067816395,
+ "tokens_seen": 1302790144
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004891153460381143,
+ "loss": 1.3672,
+ "theoretical_loss": 3.589099845630688,
+ "tokens_seen": 1302855680
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890992978936811,
+ "loss": 1.3606,
+ "theoretical_loss": 3.589083624489435,
+ "tokens_seen": 1302921216
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890832497492477,
+ "loss": 1.3398,
+ "theoretical_loss": 3.589067404392515,
+ "tokens_seen": 1302986752
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890672016048145,
+ "loss": 1.3625,
+ "theoretical_loss": 3.589051185339809,
+ "tokens_seen": 1303052288
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890511534603811,
+ "loss": 1.4311,
+ "theoretical_loss": 3.5890349673311963,
+ "tokens_seen": 1303117824
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890351053159479,
+ "loss": 1.2492,
+ "theoretical_loss": 3.5890187503665576,
+ "tokens_seen": 1303183360
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890190571715145,
+ "loss": 1.3321,
+ "theoretical_loss": 3.589002534445774,
+ "tokens_seen": 1303248896
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004890030090270813,
+ "loss": 1.3745,
+ "theoretical_loss": 3.588986319568725,
+ "tokens_seen": 1303314432
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004889869608826479,
+ "loss": 1.4623,
+ "theoretical_loss": 3.5889701057352914,
+ "tokens_seen": 1303379968
+ },
+ {
+ "epoch": 0.39,
+ "learning_rate": 0.0004889709127382147,
+ "loss": 1.3252,
+ "theoretical_loss": 3.5889538929453533,
+ "tokens_seen": 1303445504
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004889548645937813,
+ "loss": 1.3577,
+ "theoretical_loss": 3.5889376811987908,
+ "tokens_seen": 1303511040
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004889388164493481,
+ "loss": 1.377,
+ "theoretical_loss": 3.588921470495485,
+ "tokens_seen": 1303576576
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004889227683049147,
+ "loss": 1.3029,
+ "theoretical_loss": 3.5889052608353156,
+ "tokens_seen": 1303642112
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004889067201604815,
+ "loss": 1.2623,
+ "theoretical_loss": 3.588889052218163,
+ "tokens_seen": 1303707648
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004888906720160481,
+ "loss": 1.2913,
+ "theoretical_loss": 3.588872844643909,
+ "tokens_seen": 1303773184
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004888746238716149,
+ "loss": 1.2814,
+ "theoretical_loss": 3.588856638112432,
+ "tokens_seen": 1303838720
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004888585757271816,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5888404326236145,
+ "tokens_seen": 1303904256
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004888425275827483,
+ "loss": 1.3847,
+ "theoretical_loss": 3.5888242281773355,
+ "tokens_seen": 1303969792
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488826479438315,
+ "loss": 1.4997,
+ "theoretical_loss": 3.588808024773477,
+ "tokens_seen": 1304035328
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004888104312938817,
+ "loss": 1.2831,
+ "theoretical_loss": 3.5887918224119177,
+ "tokens_seen": 1304100864
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 437838,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.382591962814331,
+ "objective/train/theoretical_loss": 3.58877562109254,
+ "objective/train/tokens_used": 1324626400,
+ "theoretical_loss": 3.58877562109254,
+ "tokens_seen": 1304166400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887943831494484,
+ "loss": 1.4017,
+ "theoretical_loss": 3.58877562109254,
+ "tokens_seen": 1304166400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887783350050151,
+ "loss": 1.3708,
+ "theoretical_loss": 3.588759420815223,
+ "tokens_seen": 1304231936
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887622868605818,
+ "loss": 1.3601,
+ "theoretical_loss": 3.5887432215798487,
+ "tokens_seen": 1304297472
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887462387161485,
+ "loss": 1.2826,
+ "theoretical_loss": 3.588727023386297,
+ "tokens_seen": 1304363008
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887301905717152,
+ "loss": 1.4542,
+ "theoretical_loss": 3.588710826234448,
+ "tokens_seen": 1304428544
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004887141424272819,
+ "loss": 1.2902,
+ "theoretical_loss": 3.5886946301241833,
+ "tokens_seen": 1304494080
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886980942828486,
+ "loss": 1.3722,
+ "theoretical_loss": 3.5886784350553835,
+ "tokens_seen": 1304559616
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886820461384153,
+ "loss": 1.281,
+ "theoretical_loss": 3.5886622410279294,
+ "tokens_seen": 1304625152
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488665997993982,
+ "loss": 1.2987,
+ "theoretical_loss": 3.588646048041701,
+ "tokens_seen": 1304690688
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886499498495487,
+ "loss": 1.2728,
+ "theoretical_loss": 3.5886298560965795,
+ "tokens_seen": 1304756224
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886339017051155,
+ "loss": 1.2925,
+ "theoretical_loss": 3.5886136651924456,
+ "tokens_seen": 1304821760
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886178535606821,
+ "loss": 1.3501,
+ "theoretical_loss": 3.58859747532918,
+ "tokens_seen": 1304887296
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004886018054162489,
+ "loss": 1.3987,
+ "theoretical_loss": 3.588581286506664,
+ "tokens_seen": 1304952832
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004885857572718155,
+ "loss": 1.2724,
+ "theoretical_loss": 3.5885650987247777,
+ "tokens_seen": 1305018368
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004885697091273823,
+ "loss": 1.4087,
+ "theoretical_loss": 3.588548911983403,
+ "tokens_seen": 1305083904
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004885536609829489,
+ "loss": 1.3,
+ "theoretical_loss": 3.58853272628242,
+ "tokens_seen": 1305149440
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004885376128385157,
+ "loss": 1.4646,
+ "theoretical_loss": 3.5885165416217095,
+ "tokens_seen": 1305214976
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004885215646940823,
+ "loss": 1.2494,
+ "theoretical_loss": 3.588500358001153,
+ "tokens_seen": 1305280512
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488505516549649,
+ "loss": 1.3871,
+ "theoretical_loss": 3.5884841754206303,
+ "tokens_seen": 1305346048
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884894684052157,
+ "loss": 1.236,
+ "theoretical_loss": 3.588467993880024,
+ "tokens_seen": 1305411584
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884734202607823,
+ "loss": 1.4106,
+ "theoretical_loss": 3.588451813379214,
+ "tokens_seen": 1305477120
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884573721163491,
+ "loss": 1.2457,
+ "theoretical_loss": 3.588435633918081,
+ "tokens_seen": 1305542656
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884413239719157,
+ "loss": 1.3543,
+ "theoretical_loss": 3.5884194554965068,
+ "tokens_seen": 1305608192
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884252758274825,
+ "loss": 1.3426,
+ "theoretical_loss": 3.588403278114372,
+ "tokens_seen": 1305673728
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004884092276830492,
+ "loss": 1.2778,
+ "theoretical_loss": 3.5883871017715583,
+ "tokens_seen": 1305739264
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004883931795386159,
+ "loss": 1.2656,
+ "theoretical_loss": 3.5883709264679458,
+ "tokens_seen": 1305804800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004883771313941826,
+ "loss": 1.2422,
+ "theoretical_loss": 3.588354752203416,
+ "tokens_seen": 1305870336
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004883610832497492,
+ "loss": 1.2721,
+ "theoretical_loss": 3.5883385789778504,
+ "tokens_seen": 1305935872
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488345035105316,
+ "loss": 1.3752,
+ "theoretical_loss": 3.5883224067911295,
+ "tokens_seen": 1306001408
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004883289869608826,
+ "loss": 1.3963,
+ "theoretical_loss": 3.5883062356431346,
+ "tokens_seen": 1306066944
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004883129388164494,
+ "loss": 1.3193,
+ "theoretical_loss": 3.588290065533747,
+ "tokens_seen": 1306132480
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488296890672016,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5882738964628484,
+ "tokens_seen": 1306198016
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004882808425275828,
+ "loss": 1.2814,
+ "theoretical_loss": 3.588257728430319,
+ "tokens_seen": 1306263552
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048826479438314946,
+ "loss": 1.3708,
+ "theoretical_loss": 3.588241561436041,
+ "tokens_seen": 1306329088
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004882487462387162,
+ "loss": 1.2912,
+ "theoretical_loss": 3.5882253954798946,
+ "tokens_seen": 1306394624
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048823269809428286,
+ "loss": 1.3338,
+ "theoretical_loss": 3.5882092305617617,
+ "tokens_seen": 1306460160
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004882166499498496,
+ "loss": 1.3748,
+ "theoretical_loss": 3.588193066681524,
+ "tokens_seen": 1306525696
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048820060180541626,
+ "loss": 1.3714,
+ "theoretical_loss": 3.588176903839062,
+ "tokens_seen": 1306591232
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000488184553660983,
+ "loss": 1.2621,
+ "theoretical_loss": 3.5881607420342574,
+ "tokens_seen": 1306656768
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048816850551654966,
+ "loss": 1.3108,
+ "theoretical_loss": 3.588144581266991,
+ "tokens_seen": 1306722304
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004881524573721164,
+ "loss": 1.4491,
+ "theoretical_loss": 3.588128421537145,
+ "tokens_seen": 1306787840
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048813640922768306,
+ "loss": 1.3293,
+ "theoretical_loss": 3.5881122628446005,
+ "tokens_seen": 1306853376
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048812036108324976,
+ "loss": 1.4123,
+ "theoretical_loss": 3.588096105189239,
+ "tokens_seen": 1306918912
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048810431293881646,
+ "loss": 1.3747,
+ "theoretical_loss": 3.5880799485709414,
+ "tokens_seen": 1306984448
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048808826479438316,
+ "loss": 1.3194,
+ "theoretical_loss": 3.5880637929895896,
+ "tokens_seen": 1307049984
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048807221664994986,
+ "loss": 1.4002,
+ "theoretical_loss": 3.588047638445065,
+ "tokens_seen": 1307115520
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048805616850551656,
+ "loss": 1.3388,
+ "theoretical_loss": 3.5880314849372485,
+ "tokens_seen": 1307181056
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048804012036108327,
+ "loss": 1.3694,
+ "theoretical_loss": 3.588015332466023,
+ "tokens_seen": 1307246592
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048802407221664997,
+ "loss": 1.3555,
+ "theoretical_loss": 3.5879991810312686,
+ "tokens_seen": 1307312128
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048800802407221667,
+ "loss": 1.398,
+ "theoretical_loss": 3.5879830306328673,
+ "tokens_seen": 1307377664
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 438740,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2633261680603027,
+ "objective/train/theoretical_loss": 3.587966881270701,
+ "objective/train/tokens_used": 1327903200,
+ "theoretical_loss": 3.587966881270701,
+ "tokens_seen": 1307443200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048799197592778337,
+ "loss": 1.2872,
+ "theoretical_loss": 3.587966881270701,
+ "tokens_seen": 1307443200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048797592778335,
+ "loss": 1.486,
+ "theoretical_loss": 3.587950732944651,
+ "tokens_seen": 1307508736
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048795987963891677,
+ "loss": 1.3327,
+ "theoretical_loss": 3.5879345856545988,
+ "tokens_seen": 1307574272
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004879438314944834,
+ "loss": 1.3837,
+ "theoretical_loss": 3.5879184394004264,
+ "tokens_seen": 1307639808
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048792778335005017,
+ "loss": 1.3712,
+ "theoretical_loss": 3.587902294182015,
+ "tokens_seen": 1307705344
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004879117352056168,
+ "loss": 1.2574,
+ "theoretical_loss": 3.587886149999246,
+ "tokens_seen": 1307770880
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048789568706118357,
+ "loss": 1.4947,
+ "theoretical_loss": 3.587870006852002,
+ "tokens_seen": 1307836416
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004878796389167502,
+ "loss": 1.3119,
+ "theoretical_loss": 3.587853864740164,
+ "tokens_seen": 1307901952
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048786359077231697,
+ "loss": 1.3567,
+ "theoretical_loss": 3.587837723663614,
+ "tokens_seen": 1307967488
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004878475426278836,
+ "loss": 1.3144,
+ "theoretical_loss": 3.587821583622233,
+ "tokens_seen": 1308033024
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004878314944834504,
+ "loss": 1.3094,
+ "theoretical_loss": 3.5878054446159044,
+ "tokens_seen": 1308098560
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000487815446339017,
+ "loss": 1.3086,
+ "theoretical_loss": 3.587789306644509,
+ "tokens_seen": 1308164096
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877993981945838,
+ "loss": 1.4832,
+ "theoretical_loss": 3.5877731697079276,
+ "tokens_seen": 1308229632
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877833500501504,
+ "loss": 1.3206,
+ "theoretical_loss": 3.587757033806043,
+ "tokens_seen": 1308295168
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877673019057172,
+ "loss": 1.4024,
+ "theoretical_loss": 3.587740898938738,
+ "tokens_seen": 1308360704
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877512537612838,
+ "loss": 1.248,
+ "theoretical_loss": 3.5877247651058926,
+ "tokens_seen": 1308426240
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877352056168506,
+ "loss": 1.2777,
+ "theoretical_loss": 3.5877086323073897,
+ "tokens_seen": 1308491776
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004877191574724172,
+ "loss": 1.3802,
+ "theoretical_loss": 3.587692500543111,
+ "tokens_seen": 1308557312
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000487703109327984,
+ "loss": 1.4371,
+ "theoretical_loss": 3.5876763698129386,
+ "tokens_seen": 1308622848
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048768706118355073,
+ "loss": 1.4213,
+ "theoretical_loss": 3.5876602401167537,
+ "tokens_seen": 1308688384
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004876710130391174,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5876441114544395,
+ "tokens_seen": 1308753920
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048765496489468413,
+ "loss": 1.4607,
+ "theoretical_loss": 3.587627983825876,
+ "tokens_seen": 1308819456
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004876389167502508,
+ "loss": 1.3423,
+ "theoretical_loss": 3.587611857230948,
+ "tokens_seen": 1308884992
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048762286860581754,
+ "loss": 1.3082,
+ "theoretical_loss": 3.587595731669535,
+ "tokens_seen": 1308950528
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004876068204613842,
+ "loss": 1.3249,
+ "theoretical_loss": 3.58757960714152,
+ "tokens_seen": 1309016064
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048759077231695094,
+ "loss": 1.3357,
+ "theoretical_loss": 3.587563483646785,
+ "tokens_seen": 1309081600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004875747241725176,
+ "loss": 1.3371,
+ "theoretical_loss": 3.5875473611852122,
+ "tokens_seen": 1309147136
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048755867602808434,
+ "loss": 1.3521,
+ "theoretical_loss": 3.5875312397566836,
+ "tokens_seen": 1309212672
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000487542627883651,
+ "loss": 1.415,
+ "theoretical_loss": 3.5875151193610817,
+ "tokens_seen": 1309278208
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048752657973921774,
+ "loss": 1.3108,
+ "theoretical_loss": 3.587498999998287,
+ "tokens_seen": 1309343744
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004875105315947844,
+ "loss": 1.2217,
+ "theoretical_loss": 3.5874828816681834,
+ "tokens_seen": 1309409280
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874944834503511,
+ "loss": 1.3103,
+ "theoretical_loss": 3.5874667643706526,
+ "tokens_seen": 1309474816
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874784353059178,
+ "loss": 1.2793,
+ "theoretical_loss": 3.587450648105577,
+ "tokens_seen": 1309540352
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874623871614845,
+ "loss": 1.3311,
+ "theoretical_loss": 3.5874345328728374,
+ "tokens_seen": 1309605888
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874463390170512,
+ "loss": 1.3557,
+ "theoretical_loss": 3.5874184186723177,
+ "tokens_seen": 1309671424
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874302908726179,
+ "loss": 1.425,
+ "theoretical_loss": 3.5874023055038995,
+ "tokens_seen": 1309736960
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004874142427281846,
+ "loss": 1.2403,
+ "theoretical_loss": 3.5873861933674647,
+ "tokens_seen": 1309802496
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004873981945837513,
+ "loss": 1.384,
+ "theoretical_loss": 3.5873700822628964,
+ "tokens_seen": 1309868032
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000487382146439318,
+ "loss": 1.3239,
+ "theoretical_loss": 3.5873539721900762,
+ "tokens_seen": 1309933568
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004873660982948847,
+ "loss": 1.3367,
+ "theoretical_loss": 3.5873378631488864,
+ "tokens_seen": 1309999104
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048735005015045134,
+ "loss": 1.4561,
+ "theoretical_loss": 3.58732175513921,
+ "tokens_seen": 1310064640
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004873340020060181,
+ "loss": 1.3727,
+ "theoretical_loss": 3.5873056481609282,
+ "tokens_seen": 1310130176
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048731795386158474,
+ "loss": 1.3093,
+ "theoretical_loss": 3.5872895422139246,
+ "tokens_seen": 1310195712
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004873019057171515,
+ "loss": 1.4342,
+ "theoretical_loss": 3.587273437298081,
+ "tokens_seen": 1310261248
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048728585757271814,
+ "loss": 1.3079,
+ "theoretical_loss": 3.5872573334132802,
+ "tokens_seen": 1310326784
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004872698094282849,
+ "loss": 1.4567,
+ "theoretical_loss": 3.587241230559404,
+ "tokens_seen": 1310392320
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048725376128385154,
+ "loss": 1.2264,
+ "theoretical_loss": 3.5872251287363355,
+ "tokens_seen": 1310457856
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004872377131394183,
+ "loss": 1.3824,
+ "theoretical_loss": 3.587209027943956,
+ "tokens_seen": 1310523392
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048722166499498494,
+ "loss": 1.2255,
+ "theoretical_loss": 3.5871929281821497,
+ "tokens_seen": 1310588928
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004872056168505517,
+ "loss": 1.3629,
+ "theoretical_loss": 3.5871768294507973,
+ "tokens_seen": 1310654464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.6869446880300378,
+ "debugging/entropy-1-grams": 4.904166514984576,
+ "debugging/length": 583.2222222222222,
+ "debugging/num_segments": 9,
+ "debugging/score": 0.07426709959248204,
+ "debugging/score_std": 0.021458313531614225,
+ "epoch": 0.4,
+ "objective/train/docs_used": 440076,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2848517894744873,
+ "objective/train/theoretical_loss": 3.5871607317497833,
+ "objective/train/tokens_used": 1331180000,
+ "theoretical_loss": 3.5871607317497833,
+ "tokens_seen": 1310720000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048718956870611834,
+ "loss": 1.3497,
+ "theoretical_loss": 3.5871607317497833,
+ "tokens_seen": 1310720000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004871735205616851,
+ "loss": 1.4093,
+ "theoretical_loss": 3.587144635078989,
+ "tokens_seen": 1310785536
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048715747241725175,
+ "loss": 1.3719,
+ "theoretical_loss": 3.5871285394382966,
+ "tokens_seen": 1310851072
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004871414242728185,
+ "loss": 1.3175,
+ "theoretical_loss": 3.5871124448275893,
+ "tokens_seen": 1310916608
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048712537612838515,
+ "loss": 1.3838,
+ "theoretical_loss": 3.5870963512467497,
+ "tokens_seen": 1310982144
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004871093279839519,
+ "loss": 1.3672,
+ "theoretical_loss": 3.5870802586956607,
+ "tokens_seen": 1311047680
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048709327983951855,
+ "loss": 1.2974,
+ "theoretical_loss": 3.5870641671742045,
+ "tokens_seen": 1311113216
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004870772316950853,
+ "loss": 1.4125,
+ "theoretical_loss": 3.587048076682264,
+ "tokens_seen": 1311178752
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048706118355065195,
+ "loss": 1.4156,
+ "theoretical_loss": 3.5870319872197216,
+ "tokens_seen": 1311244288
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004870451354062187,
+ "loss": 1.2327,
+ "theoretical_loss": 3.58701589878646,
+ "tokens_seen": 1311309824
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048702908726178535,
+ "loss": 1.2803,
+ "theoretical_loss": 3.5869998113823622,
+ "tokens_seen": 1311375360
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004870130391173521,
+ "loss": 1.4842,
+ "theoretical_loss": 3.586983725007311,
+ "tokens_seen": 1311440896
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048699699097291875,
+ "loss": 1.273,
+ "theoretical_loss": 3.5869676396611885,
+ "tokens_seen": 1311506432
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004869809428284855,
+ "loss": 1.365,
+ "theoretical_loss": 3.5869515553438784,
+ "tokens_seen": 1311571968
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048696489468405215,
+ "loss": 1.3436,
+ "theoretical_loss": 3.586935472055263,
+ "tokens_seen": 1311637504
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004869488465396189,
+ "loss": 1.3666,
+ "theoretical_loss": 3.5869193897952245,
+ "tokens_seen": 1311703040
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048693279839518555,
+ "loss": 1.3589,
+ "theoretical_loss": 3.586903308563647,
+ "tokens_seen": 1311768576
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004869167502507523,
+ "loss": 1.3215,
+ "theoretical_loss": 3.586887228360413,
+ "tokens_seen": 1311834112
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048690070210631896,
+ "loss": 1.2588,
+ "theoretical_loss": 3.586871149185405,
+ "tokens_seen": 1311899648
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004868846539618857,
+ "loss": 1.4483,
+ "theoretical_loss": 3.5868550710385056,
+ "tokens_seen": 1311965184
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048686860581745236,
+ "loss": 1.3977,
+ "theoretical_loss": 3.586838993919598,
+ "tokens_seen": 1312030720
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004868525576730191,
+ "loss": 1.3464,
+ "theoretical_loss": 3.5868229178285658,
+ "tokens_seen": 1312096256
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048683650952858576,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5868068427652906,
+ "tokens_seen": 1312161792
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004868204613841525,
+ "loss": 1.264,
+ "theoretical_loss": 3.586790768729657,
+ "tokens_seen": 1312227328
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048680441323971916,
+ "loss": 1.2785,
+ "theoretical_loss": 3.586774695721547,
+ "tokens_seen": 1312292864
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004867883650952859,
+ "loss": 1.3954,
+ "theoretical_loss": 3.5867586237408435,
+ "tokens_seen": 1312358400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048677231695085256,
+ "loss": 1.4794,
+ "theoretical_loss": 3.5867425527874297,
+ "tokens_seen": 1312423936
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004867562688064193,
+ "loss": 1.3471,
+ "theoretical_loss": 3.586726482861189,
+ "tokens_seen": 1312489472
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048674022066198596,
+ "loss": 1.3096,
+ "theoretical_loss": 3.586710413962004,
+ "tokens_seen": 1312555008
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048672417251755266,
+ "loss": 1.3247,
+ "theoretical_loss": 3.5866943460897582,
+ "tokens_seen": 1312620544
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048670812437311936,
+ "loss": 1.4049,
+ "theoretical_loss": 3.5866782792443344,
+ "tokens_seen": 1312686080
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048669207622868606,
+ "loss": 1.3004,
+ "theoretical_loss": 3.5866622134256154,
+ "tokens_seen": 1312751616
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048667602808425277,
+ "loss": 1.4206,
+ "theoretical_loss": 3.586646148633485,
+ "tokens_seen": 1312817152
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048665997993981947,
+ "loss": 1.223,
+ "theoretical_loss": 3.586630084867826,
+ "tokens_seen": 1312882688
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048664393179538617,
+ "loss": 1.3097,
+ "theoretical_loss": 3.586614022128522,
+ "tokens_seen": 1312948224
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048662788365095287,
+ "loss": 1.3496,
+ "theoretical_loss": 3.586597960415456,
+ "tokens_seen": 1313013760
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048661183550651957,
+ "loss": 1.3372,
+ "theoretical_loss": 3.5865818997285106,
+ "tokens_seen": 1313079296
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048659578736208627,
+ "loss": 1.3366,
+ "theoretical_loss": 3.5865658400675695,
+ "tokens_seen": 1313144832
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004865797392176529,
+ "loss": 1.1644,
+ "theoretical_loss": 3.5865497814325162,
+ "tokens_seen": 1313210368
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048656369107321967,
+ "loss": 1.2771,
+ "theoretical_loss": 3.5865337238232335,
+ "tokens_seen": 1313275904
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004865476429287863,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5865176672396055,
+ "tokens_seen": 1313341440
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048653159478435307,
+ "loss": 1.4803,
+ "theoretical_loss": 3.5865016116815145,
+ "tokens_seen": 1313406976
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004865155466399197,
+ "loss": 1.2962,
+ "theoretical_loss": 3.5864855571488445,
+ "tokens_seen": 1313472512
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048649949849548647,
+ "loss": 1.3954,
+ "theoretical_loss": 3.586469503641479,
+ "tokens_seen": 1313538048
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004864834503510532,
+ "loss": 1.3354,
+ "theoretical_loss": 3.5864534511593003,
+ "tokens_seen": 1313603584
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004864674022066199,
+ "loss": 1.3211,
+ "theoretical_loss": 3.5864373997021928,
+ "tokens_seen": 1313669120
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048645135406218663,
+ "loss": 1.3154,
+ "theoretical_loss": 3.5864213492700396,
+ "tokens_seen": 1313734656
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004864353059177533,
+ "loss": 1.3031,
+ "theoretical_loss": 3.586405299862724,
+ "tokens_seen": 1313800192
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048641925777332003,
+ "loss": 1.2972,
+ "theoretical_loss": 3.5863892514801297,
+ "tokens_seen": 1313865728
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004864032096288867,
+ "loss": 1.274,
+ "theoretical_loss": 3.58637320412214,
+ "tokens_seen": 1313931264
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 441061,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4314746856689453,
+ "objective/train/theoretical_loss": 3.5863571577886386,
+ "objective/train/tokens_used": 1334456800,
+ "theoretical_loss": 3.5863571577886386,
+ "tokens_seen": 1313996800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048638716148445343,
+ "loss": 1.4004,
+ "theoretical_loss": 3.5863571577886386,
+ "tokens_seen": 1313996800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004863711133400201,
+ "loss": 1.4439,
+ "theoretical_loss": 3.586341112479509,
+ "tokens_seen": 1314062336
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048635506519558683,
+ "loss": 1.3404,
+ "theoretical_loss": 3.5863250681946344,
+ "tokens_seen": 1314127872
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004863390170511535,
+ "loss": 1.2621,
+ "theoretical_loss": 3.586309024933898,
+ "tokens_seen": 1314193408
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048632296890672023,
+ "loss": 1.3316,
+ "theoretical_loss": 3.5862929826971843,
+ "tokens_seen": 1314258944
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004863069207622869,
+ "loss": 1.2931,
+ "theoretical_loss": 3.5862769414843765,
+ "tokens_seen": 1314324480
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048629087261785363,
+ "loss": 1.4122,
+ "theoretical_loss": 3.5862609012953577,
+ "tokens_seen": 1314390016
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004862748244734203,
+ "loss": 1.4399,
+ "theoretical_loss": 3.5862448621300125,
+ "tokens_seen": 1314455552
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048625877632898704,
+ "loss": 1.293,
+ "theoretical_loss": 3.5862288239882236,
+ "tokens_seen": 1314521088
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004862427281845537,
+ "loss": 1.2831,
+ "theoretical_loss": 3.5862127868698757,
+ "tokens_seen": 1314586624
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048622668004012044,
+ "loss": 1.3394,
+ "theoretical_loss": 3.5861967507748513,
+ "tokens_seen": 1314652160
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004862106318956871,
+ "loss": 1.3394,
+ "theoretical_loss": 3.5861807157030343,
+ "tokens_seen": 1314717696
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048619458375125384,
+ "loss": 1.3589,
+ "theoretical_loss": 3.586164681654309,
+ "tokens_seen": 1314783232
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004861785356068205,
+ "loss": 1.3801,
+ "theoretical_loss": 3.5861486486285594,
+ "tokens_seen": 1314848768
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048616248746238724,
+ "loss": 1.2873,
+ "theoretical_loss": 3.586132616625668,
+ "tokens_seen": 1314914304
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004861464393179539,
+ "loss": 1.3393,
+ "theoretical_loss": 3.5861165856455197,
+ "tokens_seen": 1314979840
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048613039117352064,
+ "loss": 1.3433,
+ "theoretical_loss": 3.586100555687998,
+ "tokens_seen": 1315045376
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004861143430290873,
+ "loss": 1.3301,
+ "theoretical_loss": 3.5860845267529857,
+ "tokens_seen": 1315110912
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048609829488465404,
+ "loss": 1.3748,
+ "theoretical_loss": 3.586068498840368,
+ "tokens_seen": 1315176448
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860822467402207,
+ "loss": 1.251,
+ "theoretical_loss": 3.586052471950029,
+ "tokens_seen": 1315241984
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860661985957874,
+ "loss": 1.3338,
+ "theoretical_loss": 3.5860364460818506,
+ "tokens_seen": 1315307520
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860501504513541,
+ "loss": 1.3986,
+ "theoretical_loss": 3.5860204212357187,
+ "tokens_seen": 1315373056
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860341023069208,
+ "loss": 1.3436,
+ "theoretical_loss": 3.586004397411516,
+ "tokens_seen": 1315438592
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860180541624875,
+ "loss": 1.266,
+ "theoretical_loss": 3.5859883746091272,
+ "tokens_seen": 1315504128
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004860020060180542,
+ "loss": 1.2852,
+ "theoretical_loss": 3.5859723528284357,
+ "tokens_seen": 1315569664
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004859859578736209,
+ "loss": 1.4979,
+ "theoretical_loss": 3.5859563320693253,
+ "tokens_seen": 1315635200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004859699097291876,
+ "loss": 1.246,
+ "theoretical_loss": 3.585940312331681,
+ "tokens_seen": 1315700736
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048595386158475424,
+ "loss": 1.3416,
+ "theoretical_loss": 3.5859242936153857,
+ "tokens_seen": 1315766272
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000485937813440321,
+ "loss": 1.407,
+ "theoretical_loss": 3.5859082759203234,
+ "tokens_seen": 1315831808
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048592176529588764,
+ "loss": 1.2157,
+ "theoretical_loss": 3.585892259246379,
+ "tokens_seen": 1315897344
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004859057171514544,
+ "loss": 1.3887,
+ "theoretical_loss": 3.585876243593436,
+ "tokens_seen": 1315962880
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048588966900702104,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5858602289613786,
+ "tokens_seen": 1316028416
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004858736208625878,
+ "loss": 1.3796,
+ "theoretical_loss": 3.585844215350091,
+ "tokens_seen": 1316093952
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048585757271815444,
+ "loss": 1.3373,
+ "theoretical_loss": 3.585828202759457,
+ "tokens_seen": 1316159488
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004858415245737212,
+ "loss": 1.3937,
+ "theoretical_loss": 3.585812191189361,
+ "tokens_seen": 1316225024
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048582547642928784,
+ "loss": 1.4095,
+ "theoretical_loss": 3.5857961806396874,
+ "tokens_seen": 1316290560
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004858094282848546,
+ "loss": 1.373,
+ "theoretical_loss": 3.5857801711103194,
+ "tokens_seen": 1316356096
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048579338014042125,
+ "loss": 1.3198,
+ "theoretical_loss": 3.585764162601142,
+ "tokens_seen": 1316421632
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000485777331995988,
+ "loss": 1.3021,
+ "theoretical_loss": 3.5857481551120394,
+ "tokens_seen": 1316487168
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048576128385155465,
+ "loss": 1.2573,
+ "theoretical_loss": 3.5857321486428955,
+ "tokens_seen": 1316552704
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004857452357071214,
+ "loss": 1.3368,
+ "theoretical_loss": 3.585716143193595,
+ "tokens_seen": 1316618240
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048572918756268805,
+ "loss": 1.395,
+ "theoretical_loss": 3.5857001387640213,
+ "tokens_seen": 1316683776
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004857131394182548,
+ "loss": 1.3204,
+ "theoretical_loss": 3.5856841353540596,
+ "tokens_seen": 1316749312
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048569709127382145,
+ "loss": 1.2323,
+ "theoretical_loss": 3.585668132963594,
+ "tokens_seen": 1316814848
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004856810431293882,
+ "loss": 1.442,
+ "theoretical_loss": 3.585652131592508,
+ "tokens_seen": 1316880384
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048566499498495485,
+ "loss": 1.244,
+ "theoretical_loss": 3.585636131240687,
+ "tokens_seen": 1316945920
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004856489468405216,
+ "loss": 1.3468,
+ "theoretical_loss": 3.5856201319080148,
+ "tokens_seen": 1317011456
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048563289869608825,
+ "loss": 1.3423,
+ "theoretical_loss": 3.5856041335943765,
+ "tokens_seen": 1317076992
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000485616850551655,
+ "loss": 1.2923,
+ "theoretical_loss": 3.585588136299655,
+ "tokens_seen": 1317142528
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048560080240722165,
+ "loss": 1.4071,
+ "theoretical_loss": 3.5855721400237366,
+ "tokens_seen": 1317208064
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 442341,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3773832321166992,
+ "objective/train/theoretical_loss": 3.585556144766504,
+ "objective/train/tokens_used": 1337733600,
+ "theoretical_loss": 3.585556144766504,
+ "tokens_seen": 1317273600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004855847542627884,
+ "loss": 1.4276,
+ "theoretical_loss": 3.585556144766504,
+ "tokens_seen": 1317273600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048556870611835505,
+ "loss": 1.3344,
+ "theoretical_loss": 3.5855401505278426,
+ "tokens_seen": 1317339136
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004855526579739218,
+ "loss": 1.4097,
+ "theoretical_loss": 3.5855241573076366,
+ "tokens_seen": 1317404672
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048553660982948846,
+ "loss": 1.2924,
+ "theoretical_loss": 3.5855081651057708,
+ "tokens_seen": 1317470208
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004855205616850552,
+ "loss": 1.3379,
+ "theoretical_loss": 3.5854921739221295,
+ "tokens_seen": 1317535744
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048550451354062186,
+ "loss": 1.3209,
+ "theoretical_loss": 3.5854761837565974,
+ "tokens_seen": 1317601280
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004854884653961886,
+ "loss": 1.3033,
+ "theoretical_loss": 3.5854601946090585,
+ "tokens_seen": 1317666816
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048547241725175526,
+ "loss": 1.3764,
+ "theoretical_loss": 3.585444206479398,
+ "tokens_seen": 1317732352
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000485456369107322,
+ "loss": 1.3218,
+ "theoretical_loss": 3.5854282193674996,
+ "tokens_seen": 1317797888
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048544032096288866,
+ "loss": 1.4137,
+ "theoretical_loss": 3.585412233273249,
+ "tokens_seen": 1317863424
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004854242728184554,
+ "loss": 1.3492,
+ "theoretical_loss": 3.5853962481965302,
+ "tokens_seen": 1317928960
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048540822467402206,
+ "loss": 1.2507,
+ "theoretical_loss": 3.5853802641372283,
+ "tokens_seen": 1317994496
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004853921765295888,
+ "loss": 1.243,
+ "theoretical_loss": 3.5853642810952273,
+ "tokens_seen": 1318060032
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048537612838515546,
+ "loss": 1.3568,
+ "theoretical_loss": 3.5853482990704117,
+ "tokens_seen": 1318125568
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004853600802407222,
+ "loss": 1.2306,
+ "theoretical_loss": 3.5853323180626675,
+ "tokens_seen": 1318191104
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048534403209628886,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5853163380718787,
+ "tokens_seen": 1318256640
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004853279839518556,
+ "loss": 1.2497,
+ "theoretical_loss": 3.5853003590979293,
+ "tokens_seen": 1318322176
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048531193580742226,
+ "loss": 1.2938,
+ "theoretical_loss": 3.585284381140705,
+ "tokens_seen": 1318387712
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048529588766298897,
+ "loss": 1.2727,
+ "theoretical_loss": 3.5852684042000904,
+ "tokens_seen": 1318453248
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004852798395185557,
+ "loss": 1.3531,
+ "theoretical_loss": 3.58525242827597,
+ "tokens_seen": 1318518784
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048526379137412237,
+ "loss": 1.4274,
+ "theoretical_loss": 3.5852364533682284,
+ "tokens_seen": 1318584320
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004852477432296891,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5852204794767513,
+ "tokens_seen": 1318649856
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048523169508525577,
+ "loss": 1.3909,
+ "theoretical_loss": 3.5852045066014226,
+ "tokens_seen": 1318715392
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004852156469408225,
+ "loss": 1.2567,
+ "theoretical_loss": 3.585188534742128,
+ "tokens_seen": 1318780928
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048519959879638917,
+ "loss": 1.2556,
+ "theoretical_loss": 3.5851725638987517,
+ "tokens_seen": 1318846464
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004851835506519559,
+ "loss": 1.3358,
+ "theoretical_loss": 3.5851565940711794,
+ "tokens_seen": 1318912000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048516750250752257,
+ "loss": 1.2443,
+ "theoretical_loss": 3.585140625259295,
+ "tokens_seen": 1318977536
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004851514543630893,
+ "loss": 1.4086,
+ "theoretical_loss": 3.585124657462984,
+ "tokens_seen": 1319043072
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048513540621865597,
+ "loss": 1.293,
+ "theoretical_loss": 3.585108690682131,
+ "tokens_seen": 1319108608
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004851193580742227,
+ "loss": 1.3807,
+ "theoretical_loss": 3.5850927249166222,
+ "tokens_seen": 1319174144
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004851033099297894,
+ "loss": 1.2991,
+ "theoretical_loss": 3.585076760166341,
+ "tokens_seen": 1319239680
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048508726178535613,
+ "loss": 1.3025,
+ "theoretical_loss": 3.5850607964311734,
+ "tokens_seen": 1319305216
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004850712136409228,
+ "loss": 1.2968,
+ "theoretical_loss": 3.585044833711004,
+ "tokens_seen": 1319370752
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048505516549648953,
+ "loss": 1.2663,
+ "theoretical_loss": 3.5850288720057177,
+ "tokens_seen": 1319436288
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004850391173520562,
+ "loss": 1.2713,
+ "theoretical_loss": 3.5850129113152,
+ "tokens_seen": 1319501824
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048502306920762293,
+ "loss": 1.2781,
+ "theoretical_loss": 3.5849969516393365,
+ "tokens_seen": 1319567360
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004850070210631896,
+ "loss": 1.2317,
+ "theoretical_loss": 3.5849809929780108,
+ "tokens_seen": 1319632896
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048499097291875633,
+ "loss": 1.306,
+ "theoretical_loss": 3.5849650353311096,
+ "tokens_seen": 1319698432
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000484974924774323,
+ "loss": 1.3761,
+ "theoretical_loss": 3.584949078698517,
+ "tokens_seen": 1319763968
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048495887662988973,
+ "loss": 1.2449,
+ "theoretical_loss": 3.584933123080118,
+ "tokens_seen": 1319829504
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004849428284854564,
+ "loss": 1.2852,
+ "theoretical_loss": 3.584917168475799,
+ "tokens_seen": 1319895040
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048492678034102313,
+ "loss": 1.3378,
+ "theoretical_loss": 3.584901214885444,
+ "tokens_seen": 1319960576
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004849107321965898,
+ "loss": 1.4203,
+ "theoretical_loss": 3.584885262308939,
+ "tokens_seen": 1320026112
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048489468405215654,
+ "loss": 1.3073,
+ "theoretical_loss": 3.5848693107461687,
+ "tokens_seen": 1320091648
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004848786359077232,
+ "loss": 1.3765,
+ "theoretical_loss": 3.5848533601970187,
+ "tokens_seen": 1320157184
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048486258776328994,
+ "loss": 1.3058,
+ "theoretical_loss": 3.5848374106613745,
+ "tokens_seen": 1320222720
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004848465396188566,
+ "loss": 1.2311,
+ "theoretical_loss": 3.5848214621391206,
+ "tokens_seen": 1320288256
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048483049147442334,
+ "loss": 1.3319,
+ "theoretical_loss": 3.584805514630143,
+ "tokens_seen": 1320353792
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048481444332999,
+ "loss": 1.3171,
+ "theoretical_loss": 3.584789568134327,
+ "tokens_seen": 1320419328
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048479839518555674,
+ "loss": 1.3266,
+ "theoretical_loss": 3.584773622651557,
+ "tokens_seen": 1320484864
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 443211,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4416449069976807,
+ "objective/train/theoretical_loss": 3.5847576781817203,
+ "objective/train/tokens_used": 1341010400,
+ "theoretical_loss": 3.5847576781817203,
+ "tokens_seen": 1320550400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004847823470411234,
+ "loss": 1.3684,
+ "theoretical_loss": 3.5847576781817203,
+ "tokens_seen": 1320550400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048476629889669014,
+ "loss": 1.2597,
+ "theoretical_loss": 3.5847417347247004,
+ "tokens_seen": 1320615936
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004847502507522568,
+ "loss": 1.3509,
+ "theoretical_loss": 3.5847257922803832,
+ "tokens_seen": 1320681472
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048473420260782354,
+ "loss": 1.2616,
+ "theoretical_loss": 3.584709850848655,
+ "tokens_seen": 1320747008
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004847181544633902,
+ "loss": 1.3003,
+ "theoretical_loss": 3.5846939104294,
+ "tokens_seen": 1320812544
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048470210631895694,
+ "loss": 1.3852,
+ "theoretical_loss": 3.584677971022505,
+ "tokens_seen": 1320878080
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004846860581745236,
+ "loss": 1.3603,
+ "theoretical_loss": 3.5846620326278544,
+ "tokens_seen": 1320943616
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004846700100300903,
+ "loss": 1.3746,
+ "theoretical_loss": 3.5846460952453336,
+ "tokens_seen": 1321009152
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000484653961885657,
+ "loss": 1.263,
+ "theoretical_loss": 3.5846301588748295,
+ "tokens_seen": 1321074688
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004846379137412237,
+ "loss": 1.3042,
+ "theoretical_loss": 3.584614223516226,
+ "tokens_seen": 1321140224
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004846218655967904,
+ "loss": 1.3297,
+ "theoretical_loss": 3.58459828916941,
+ "tokens_seen": 1321205760
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004846058174523571,
+ "loss": 1.1947,
+ "theoretical_loss": 3.5845823558342653,
+ "tokens_seen": 1321271296
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004845897693079238,
+ "loss": 1.2345,
+ "theoretical_loss": 3.5845664235106796,
+ "tokens_seen": 1321336832
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004845737211634905,
+ "loss": 1.5226,
+ "theoretical_loss": 3.584550492198537,
+ "tokens_seen": 1321402368
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004845576730190572,
+ "loss": 1.3637,
+ "theoretical_loss": 3.5845345618977245,
+ "tokens_seen": 1321467904
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004845416248746239,
+ "loss": 1.3617,
+ "theoretical_loss": 3.584518632608126,
+ "tokens_seen": 1321533440
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048452557673019054,
+ "loss": 1.2552,
+ "theoretical_loss": 3.584502704329629,
+ "tokens_seen": 1321598976
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004845095285857573,
+ "loss": 1.3647,
+ "theoretical_loss": 3.5844867770621174,
+ "tokens_seen": 1321664512
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048449348044132394,
+ "loss": 1.3693,
+ "theoretical_loss": 3.584470850805478,
+ "tokens_seen": 1321730048
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004844774322968907,
+ "loss": 1.2726,
+ "theoretical_loss": 3.584454925559597,
+ "tokens_seen": 1321795584
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048446138415245734,
+ "loss": 1.3556,
+ "theoretical_loss": 3.584439001324359,
+ "tokens_seen": 1321861120
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004844453360080241,
+ "loss": 1.3298,
+ "theoretical_loss": 3.58442307809965,
+ "tokens_seen": 1321926656
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048442928786359075,
+ "loss": 1.3846,
+ "theoretical_loss": 3.584407155885356,
+ "tokens_seen": 1321992192
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004844132397191575,
+ "loss": 1.2629,
+ "theoretical_loss": 3.5843912346813633,
+ "tokens_seen": 1322057728
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048439719157472415,
+ "loss": 1.3087,
+ "theoretical_loss": 3.5843753144875565,
+ "tokens_seen": 1322123264
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004843811434302909,
+ "loss": 1.2942,
+ "theoretical_loss": 3.5843593953038226,
+ "tokens_seen": 1322188800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048436509528585755,
+ "loss": 1.3796,
+ "theoretical_loss": 3.584343477130047,
+ "tokens_seen": 1322254336
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004843490471414243,
+ "loss": 1.2679,
+ "theoretical_loss": 3.5843275599661153,
+ "tokens_seen": 1322319872
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048433299899699095,
+ "loss": 1.4062,
+ "theoretical_loss": 3.5843116438119145,
+ "tokens_seen": 1322385408
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004843169508525577,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5842957286673287,
+ "tokens_seen": 1322450944
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048430090270812435,
+ "loss": 1.2989,
+ "theoretical_loss": 3.584279814532245,
+ "tokens_seen": 1322516480
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004842848545636911,
+ "loss": 1.3335,
+ "theoretical_loss": 3.5842639014065494,
+ "tokens_seen": 1322582016
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048426880641925775,
+ "loss": 1.2631,
+ "theoretical_loss": 3.5842479892901276,
+ "tokens_seen": 1322647552
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004842527582748245,
+ "loss": 1.2766,
+ "theoretical_loss": 3.5842320781828656,
+ "tokens_seen": 1322713088
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048423671013039115,
+ "loss": 1.3562,
+ "theoretical_loss": 3.5842161680846494,
+ "tokens_seen": 1322778624
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004842206619859579,
+ "loss": 1.3548,
+ "theoretical_loss": 3.5842002589953648,
+ "tokens_seen": 1322844160
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048420461384152455,
+ "loss": 1.2538,
+ "theoretical_loss": 3.5841843509148985,
+ "tokens_seen": 1322909696
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004841885656970913,
+ "loss": 1.2859,
+ "theoretical_loss": 3.5841684438431356,
+ "tokens_seen": 1322975232
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048417251755265796,
+ "loss": 1.3239,
+ "theoretical_loss": 3.5841525377799632,
+ "tokens_seen": 1323040768
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004841564694082247,
+ "loss": 1.3852,
+ "theoretical_loss": 3.584136632725267,
+ "tokens_seen": 1323106304
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048414042126379136,
+ "loss": 1.2987,
+ "theoretical_loss": 3.5841207286789327,
+ "tokens_seen": 1323171840
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004841243731193581,
+ "loss": 1.2242,
+ "theoretical_loss": 3.5841048256408463,
+ "tokens_seen": 1323237376
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048410832497492476,
+ "loss": 1.225,
+ "theoretical_loss": 3.584088923610895,
+ "tokens_seen": 1323302912
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004840922768304915,
+ "loss": 1.3383,
+ "theoretical_loss": 3.5840730225889645,
+ "tokens_seen": 1323368448
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048407622868605827,
+ "loss": 1.3257,
+ "theoretical_loss": 3.58405712257494,
+ "tokens_seen": 1323433984
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004840601805416249,
+ "loss": 1.3174,
+ "theoretical_loss": 3.5840412235687094,
+ "tokens_seen": 1323499520
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004840441323971916,
+ "loss": 1.398,
+ "theoretical_loss": 3.584025325570158,
+ "tokens_seen": 1323565056
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004840280842527583,
+ "loss": 1.2417,
+ "theoretical_loss": 3.5840094285791717,
+ "tokens_seen": 1323630592
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000484012036108325,
+ "loss": 1.4068,
+ "theoretical_loss": 3.5839935325956374,
+ "tokens_seen": 1323696128
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839959879638917,
+ "loss": 1.4216,
+ "theoretical_loss": 3.5839776376194408,
+ "tokens_seen": 1323761664
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 444455,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3063066005706787,
+ "objective/train/theoretical_loss": 3.583961743650469,
+ "objective/train/tokens_used": 1344287200,
+ "theoretical_loss": 3.583961743650469,
+ "tokens_seen": 1323827200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839799398194584,
+ "loss": 1.3326,
+ "theoretical_loss": 3.583961743650469,
+ "tokens_seen": 1323827200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839638916750251,
+ "loss": 1.3388,
+ "theoretical_loss": 3.583945850688608,
+ "tokens_seen": 1323892736
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839478435305918,
+ "loss": 1.3409,
+ "theoretical_loss": 3.5839299587337434,
+ "tokens_seen": 1323958272
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839317953861585,
+ "loss": 1.2974,
+ "theoretical_loss": 3.583914067785763,
+ "tokens_seen": 1324023808
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004839157472417252,
+ "loss": 1.2286,
+ "theoretical_loss": 3.583898177844552,
+ "tokens_seen": 1324089344
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048389969909729187,
+ "loss": 1.2968,
+ "theoretical_loss": 3.5838822889099973,
+ "tokens_seen": 1324154880
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004838836509528586,
+ "loss": 1.344,
+ "theoretical_loss": 3.5838664009819845,
+ "tokens_seen": 1324220416
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048386760280842527,
+ "loss": 1.2972,
+ "theoretical_loss": 3.5838505140604013,
+ "tokens_seen": 1324285952
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000483851554663992,
+ "loss": 1.3461,
+ "theoretical_loss": 3.583834628145133,
+ "tokens_seen": 1324351488
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048383550651955867,
+ "loss": 1.2023,
+ "theoretical_loss": 3.5838187432360673,
+ "tokens_seen": 1324417024
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004838194583751254,
+ "loss": 1.4498,
+ "theoretical_loss": 3.5838028593330895,
+ "tokens_seen": 1324482560
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048380341023069207,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5837869764360866,
+ "tokens_seen": 1324548096
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004837873620862588,
+ "loss": 1.3089,
+ "theoretical_loss": 3.583771094544945,
+ "tokens_seen": 1324613632
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048377131394182547,
+ "loss": 1.3479,
+ "theoretical_loss": 3.5837552136595514,
+ "tokens_seen": 1324679168
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004837552657973922,
+ "loss": 1.3109,
+ "theoretical_loss": 3.583739333779792,
+ "tokens_seen": 1324744704
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048373921765295887,
+ "loss": 1.2479,
+ "theoretical_loss": 3.583723454905554,
+ "tokens_seen": 1324810240
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048372316950852563,
+ "loss": 1.2528,
+ "theoretical_loss": 3.5837075770367237,
+ "tokens_seen": 1324875776
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004837071213640923,
+ "loss": 1.2651,
+ "theoretical_loss": 3.5836917001731874,
+ "tokens_seen": 1324941312
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048369107321965903,
+ "loss": 1.3618,
+ "theoretical_loss": 3.583675824314832,
+ "tokens_seen": 1325006848
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004836750250752257,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5836599494615444,
+ "tokens_seen": 1325072384
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048365897693079243,
+ "loss": 1.3018,
+ "theoretical_loss": 3.583644075613211,
+ "tokens_seen": 1325137920
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004836429287863591,
+ "loss": 1.2062,
+ "theoretical_loss": 3.5836282027697184,
+ "tokens_seen": 1325203456
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048362688064192583,
+ "loss": 1.4019,
+ "theoretical_loss": 3.5836123309309533,
+ "tokens_seen": 1325268992
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004836108324974925,
+ "loss": 1.3232,
+ "theoretical_loss": 3.5835964600968024,
+ "tokens_seen": 1325334528
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048359478435305923,
+ "loss": 1.3022,
+ "theoretical_loss": 3.5835805902671525,
+ "tokens_seen": 1325400064
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004835787362086259,
+ "loss": 1.368,
+ "theoretical_loss": 3.583564721441891,
+ "tokens_seen": 1325465600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048356268806419263,
+ "loss": 1.3103,
+ "theoretical_loss": 3.5835488536209033,
+ "tokens_seen": 1325531136
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004835466399197593,
+ "loss": 1.3369,
+ "theoretical_loss": 3.5835329868040775,
+ "tokens_seen": 1325596672
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048353059177532604,
+ "loss": 1.3383,
+ "theoretical_loss": 3.5835171209913,
+ "tokens_seen": 1325662208
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004835145436308927,
+ "loss": 1.3852,
+ "theoretical_loss": 3.5835012561824575,
+ "tokens_seen": 1325727744
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048349849548645944,
+ "loss": 1.2991,
+ "theoretical_loss": 3.5834853923774364,
+ "tokens_seen": 1325793280
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004834824473420261,
+ "loss": 1.3479,
+ "theoretical_loss": 3.5834695295761243,
+ "tokens_seen": 1325858816
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048346639919759284,
+ "loss": 1.3692,
+ "theoretical_loss": 3.583453667778408,
+ "tokens_seen": 1325924352
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004834503510531595,
+ "loss": 1.3161,
+ "theoretical_loss": 3.583437806984174,
+ "tokens_seen": 1325989888
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048343430290872624,
+ "loss": 1.3371,
+ "theoretical_loss": 3.58342194719331,
+ "tokens_seen": 1326055424
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004834182547642929,
+ "loss": 1.3452,
+ "theoretical_loss": 3.5834060884057015,
+ "tokens_seen": 1326120960
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048340220661985964,
+ "loss": 1.4371,
+ "theoretical_loss": 3.5833902306212373,
+ "tokens_seen": 1326186496
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004833861584754263,
+ "loss": 1.4161,
+ "theoretical_loss": 3.5833743738398027,
+ "tokens_seen": 1326252032
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048337011033099304,
+ "loss": 1.2408,
+ "theoretical_loss": 3.583358518061286,
+ "tokens_seen": 1326317568
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004833540621865597,
+ "loss": 1.322,
+ "theoretical_loss": 3.5833426632855736,
+ "tokens_seen": 1326383104
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048333801404212644,
+ "loss": 1.2247,
+ "theoretical_loss": 3.5833268095125526,
+ "tokens_seen": 1326448640
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004833219658976931,
+ "loss": 1.3168,
+ "theoretical_loss": 3.5833109567421095,
+ "tokens_seen": 1326514176
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048330591775325984,
+ "loss": 1.3679,
+ "theoretical_loss": 3.583295104974132,
+ "tokens_seen": 1326579712
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004832898696088265,
+ "loss": 1.3412,
+ "theoretical_loss": 3.583279254208508,
+ "tokens_seen": 1326645248
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004832738214643932,
+ "loss": 1.3053,
+ "theoretical_loss": 3.5832634044451233,
+ "tokens_seen": 1326710784
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004832577733199599,
+ "loss": 1.4155,
+ "theoretical_loss": 3.583247555683865,
+ "tokens_seen": 1326776320
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004832417251755266,
+ "loss": 1.2586,
+ "theoretical_loss": 3.583231707924621,
+ "tokens_seen": 1326841856
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004832256770310933,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5832158611672784,
+ "tokens_seen": 1326907392
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048320962888666,
+ "loss": 1.2988,
+ "theoretical_loss": 3.583200015411724,
+ "tokens_seen": 1326972928
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004831935807422267,
+ "loss": 1.3468,
+ "theoretical_loss": 3.5831841706578444,
+ "tokens_seen": 1327038464
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 445368,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4701875448226929,
+ "objective/train/theoretical_loss": 3.5831683269055286,
+ "objective/train/tokens_used": 1347564000,
+ "theoretical_loss": 3.5831683269055286,
+ "tokens_seen": 1327104000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004831775325977934,
+ "loss": 1.4034,
+ "theoretical_loss": 3.5831683269055286,
+ "tokens_seen": 1327104000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004831614844533601,
+ "loss": 1.2843,
+ "theoretical_loss": 3.5831524841546623,
+ "tokens_seen": 1327169536
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004831454363089268,
+ "loss": 1.2842,
+ "theoretical_loss": 3.583136642405133,
+ "tokens_seen": 1327235072
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048312938816449344,
+ "loss": 1.299,
+ "theoretical_loss": 3.5831208016568286,
+ "tokens_seen": 1327300608
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004831133400200602,
+ "loss": 1.3265,
+ "theoretical_loss": 3.583104961909636,
+ "tokens_seen": 1327366144
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048309729187562684,
+ "loss": 1.2967,
+ "theoretical_loss": 3.5830891231634423,
+ "tokens_seen": 1327431680
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004830812437311936,
+ "loss": 1.3434,
+ "theoretical_loss": 3.583073285418135,
+ "tokens_seen": 1327497216
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048306519558676025,
+ "loss": 1.2553,
+ "theoretical_loss": 3.5830574486736015,
+ "tokens_seen": 1327562752
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000483049147442327,
+ "loss": 1.2941,
+ "theoretical_loss": 3.583041612929729,
+ "tokens_seen": 1327628288
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048303309929789365,
+ "loss": 1.3652,
+ "theoretical_loss": 3.583025778186405,
+ "tokens_seen": 1327693824
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004830170511534604,
+ "loss": 1.4218,
+ "theoretical_loss": 3.5830099444435173,
+ "tokens_seen": 1327759360
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048300100300902705,
+ "loss": 1.4198,
+ "theoretical_loss": 3.5829941117009527,
+ "tokens_seen": 1327824896
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004829849548645938,
+ "loss": 1.4086,
+ "theoretical_loss": 3.582978279958599,
+ "tokens_seen": 1327890432
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048296890672016045,
+ "loss": 1.2971,
+ "theoretical_loss": 3.5829624492163434,
+ "tokens_seen": 1327955968
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004829528585757272,
+ "loss": 1.1543,
+ "theoretical_loss": 3.5829466194740736,
+ "tokens_seen": 1328021504
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048293681043129385,
+ "loss": 1.4362,
+ "theoretical_loss": 3.5829307907316768,
+ "tokens_seen": 1328087040
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004829207622868606,
+ "loss": 1.3065,
+ "theoretical_loss": 3.582914962989041,
+ "tokens_seen": 1328152576
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048290471414242725,
+ "loss": 1.3042,
+ "theoretical_loss": 3.582899136246053,
+ "tokens_seen": 1328218112
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000482888665997994,
+ "loss": 1.4558,
+ "theoretical_loss": 3.5828833105026012,
+ "tokens_seen": 1328283648
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048287261785356076,
+ "loss": 1.2149,
+ "theoretical_loss": 3.5828674857585723,
+ "tokens_seen": 1328349184
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004828565697091274,
+ "loss": 1.3571,
+ "theoretical_loss": 3.582851662013854,
+ "tokens_seen": 1328414720
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048284052156469416,
+ "loss": 1.2751,
+ "theoretical_loss": 3.582835839268335,
+ "tokens_seen": 1328480256
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004828244734202608,
+ "loss": 1.3507,
+ "theoretical_loss": 3.582820017521902,
+ "tokens_seen": 1328545792
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048280842527582756,
+ "loss": 1.3013,
+ "theoretical_loss": 3.5828041967744424,
+ "tokens_seen": 1328611328
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004827923771313942,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5827883770258446,
+ "tokens_seen": 1328676864
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048277632898696097,
+ "loss": 1.3641,
+ "theoretical_loss": 3.5827725582759955,
+ "tokens_seen": 1328742400
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004827602808425276,
+ "loss": 1.2678,
+ "theoretical_loss": 3.5827567405247835,
+ "tokens_seen": 1328807936
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048274423269809437,
+ "loss": 1.3406,
+ "theoretical_loss": 3.582740923772096,
+ "tokens_seen": 1328873472
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000482728184553661,
+ "loss": 1.3013,
+ "theoretical_loss": 3.5827251080178204,
+ "tokens_seen": 1328939008
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048271213640922777,
+ "loss": 1.2817,
+ "theoretical_loss": 3.582709293261845,
+ "tokens_seen": 1329004544
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004826960882647944,
+ "loss": 1.4249,
+ "theoretical_loss": 3.5826934795040573,
+ "tokens_seen": 1329070080
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048268004012036117,
+ "loss": 1.3481,
+ "theoretical_loss": 3.5826776667443454,
+ "tokens_seen": 1329135616
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004826639919759278,
+ "loss": 1.4349,
+ "theoretical_loss": 3.582661854982596,
+ "tokens_seen": 1329201152
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004826479438314945,
+ "loss": 1.3055,
+ "theoretical_loss": 3.5826460442186985,
+ "tokens_seen": 1329266688
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004826318956870612,
+ "loss": 1.3214,
+ "theoretical_loss": 3.582630234452539,
+ "tokens_seen": 1329332224
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004826158475426279,
+ "loss": 1.4142,
+ "theoretical_loss": 3.5826144256840076,
+ "tokens_seen": 1329397760
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004825997993981946,
+ "loss": 1.428,
+ "theoretical_loss": 3.58259861791299,
+ "tokens_seen": 1329463296
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004825837512537613,
+ "loss": 1.2922,
+ "theoretical_loss": 3.582582811139375,
+ "tokens_seen": 1329528832
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000482567703109328,
+ "loss": 1.3145,
+ "theoretical_loss": 3.582567005363051,
+ "tokens_seen": 1329594368
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004825516549648947,
+ "loss": 1.3879,
+ "theoretical_loss": 3.582551200583905,
+ "tokens_seen": 1329659904
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004825356068204614,
+ "loss": 1.4077,
+ "theoretical_loss": 3.582535396801825,
+ "tokens_seen": 1329725440
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004825195586760281,
+ "loss": 1.2434,
+ "theoretical_loss": 3.5825195940167003,
+ "tokens_seen": 1329790976
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048250351053159477,
+ "loss": 1.3787,
+ "theoretical_loss": 3.582503792228417,
+ "tokens_seen": 1329856512
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004824874623871615,
+ "loss": 1.2914,
+ "theoretical_loss": 3.5824879914368646,
+ "tokens_seen": 1329922048
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048247141424272817,
+ "loss": 1.3221,
+ "theoretical_loss": 3.58247219164193,
+ "tokens_seen": 1329987584
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004824553660982949,
+ "loss": 1.3212,
+ "theoretical_loss": 3.5824563928435023,
+ "tokens_seen": 1330053120
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048243931795386157,
+ "loss": 1.3086,
+ "theoretical_loss": 3.5824405950414686,
+ "tokens_seen": 1330118656
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004824232698094283,
+ "loss": 1.32,
+ "theoretical_loss": 3.582424798235717,
+ "tokens_seen": 1330184192
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048240722166499497,
+ "loss": 1.3382,
+ "theoretical_loss": 3.582409002426137,
+ "tokens_seen": 1330249728
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004823911735205617,
+ "loss": 1.3155,
+ "theoretical_loss": 3.582393207612615,
+ "tokens_seen": 1330315264
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 446542,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2807539701461792,
+ "objective/train/theoretical_loss": 3.58237741379504,
+ "objective/train/tokens_used": 1350840800,
+ "theoretical_loss": 3.58237741379504,
+ "tokens_seen": 1330380800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048237512537612837,
+ "loss": 1.3185,
+ "theoretical_loss": 3.58237741379504,
+ "tokens_seen": 1330380800
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048235907723169513,
+ "loss": 1.3071,
+ "theoretical_loss": 3.5823616209732996,
+ "tokens_seen": 1330446336
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004823430290872618,
+ "loss": 1.3834,
+ "theoretical_loss": 3.5823458291472825,
+ "tokens_seen": 1330511872
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048232698094282853,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5823300383168766,
+ "tokens_seen": 1330577408
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004823109327983952,
+ "loss": 1.358,
+ "theoretical_loss": 3.5823142484819703,
+ "tokens_seen": 1330642944
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048229488465396193,
+ "loss": 1.3405,
+ "theoretical_loss": 3.582298459642452,
+ "tokens_seen": 1330708480
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004822788365095286,
+ "loss": 1.2709,
+ "theoretical_loss": 3.5822826717982093,
+ "tokens_seen": 1330774016
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048226278836509533,
+ "loss": 1.2909,
+ "theoretical_loss": 3.5822668849491306,
+ "tokens_seen": 1330839552
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000482246740220662,
+ "loss": 1.3235,
+ "theoretical_loss": 3.5822510990951044,
+ "tokens_seen": 1330905088
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048223069207622873,
+ "loss": 1.3417,
+ "theoretical_loss": 3.5822353142360193,
+ "tokens_seen": 1330970624
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004822146439317954,
+ "loss": 1.3082,
+ "theoretical_loss": 3.5822195303717628,
+ "tokens_seen": 1331036160
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048219859578736213,
+ "loss": 1.2585,
+ "theoretical_loss": 3.5822037475022235,
+ "tokens_seen": 1331101696
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004821825476429288,
+ "loss": 1.336,
+ "theoretical_loss": 3.5821879656272904,
+ "tokens_seen": 1331167232
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048216649949849553,
+ "loss": 1.3647,
+ "theoretical_loss": 3.5821721847468515,
+ "tokens_seen": 1331232768
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004821504513540622,
+ "loss": 1.3162,
+ "theoretical_loss": 3.5821564048607946,
+ "tokens_seen": 1331298304
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048213440320962894,
+ "loss": 1.3587,
+ "theoretical_loss": 3.5821406259690085,
+ "tokens_seen": 1331363840
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004821183550651956,
+ "loss": 1.302,
+ "theoretical_loss": 3.5821248480713823,
+ "tokens_seen": 1331429376
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048210230692076234,
+ "loss": 1.288,
+ "theoretical_loss": 3.582109071167803,
+ "tokens_seen": 1331494912
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000482086258776329,
+ "loss": 1.3165,
+ "theoretical_loss": 3.58209329525816,
+ "tokens_seen": 1331560448
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048207021063189574,
+ "loss": 1.3853,
+ "theoretical_loss": 3.5820775203423416,
+ "tokens_seen": 1331625984
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004820541624874624,
+ "loss": 1.3913,
+ "theoretical_loss": 3.5820617464202367,
+ "tokens_seen": 1331691520
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048203811434302914,
+ "loss": 1.3406,
+ "theoretical_loss": 3.5820459734917334,
+ "tokens_seen": 1331757056
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004820220661985958,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5820302015567194,
+ "tokens_seen": 1331822592
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048200601805416254,
+ "loss": 1.2499,
+ "theoretical_loss": 3.582014430615085,
+ "tokens_seen": 1331888128
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004819899699097292,
+ "loss": 1.4842,
+ "theoretical_loss": 3.5819986606667173,
+ "tokens_seen": 1331953664
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048197392176529594,
+ "loss": 1.3569,
+ "theoretical_loss": 3.5819828917115055,
+ "tokens_seen": 1332019200
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004819578736208626,
+ "loss": 1.3203,
+ "theoretical_loss": 3.581967123749338,
+ "tokens_seen": 1332084736
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048194182547642934,
+ "loss": 1.2886,
+ "theoretical_loss": 3.581951356780104,
+ "tokens_seen": 1332150272
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000481925777331996,
+ "loss": 1.2725,
+ "theoretical_loss": 3.581935590803691,
+ "tokens_seen": 1332215808
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048190972918756275,
+ "loss": 1.3201,
+ "theoretical_loss": 3.5819198258199885,
+ "tokens_seen": 1332281344
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818936810431294,
+ "loss": 1.4101,
+ "theoretical_loss": 3.581904061828885,
+ "tokens_seen": 1332346880
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818776328986961,
+ "loss": 1.2548,
+ "theoretical_loss": 3.581888298830269,
+ "tokens_seen": 1332412416
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818615847542628,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5818725368240294,
+ "tokens_seen": 1332477952
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818455366098295,
+ "loss": 1.4159,
+ "theoretical_loss": 3.5818567758100546,
+ "tokens_seen": 1332543488
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818294884653962,
+ "loss": 1.422,
+ "theoretical_loss": 3.581841015788234,
+ "tokens_seen": 1332609024
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004818134403209629,
+ "loss": 1.2996,
+ "theoretical_loss": 3.5818252567584556,
+ "tokens_seen": 1332674560
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004817973921765296,
+ "loss": 1.1327,
+ "theoretical_loss": 3.5818094987206086,
+ "tokens_seen": 1332740096
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004817813440320963,
+ "loss": 1.3226,
+ "theoretical_loss": 3.581793741674582,
+ "tokens_seen": 1332805632
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000481765295887663,
+ "loss": 1.3293,
+ "theoretical_loss": 3.5817779856202643,
+ "tokens_seen": 1332871168
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004817492477432297,
+ "loss": 1.3704,
+ "theoretical_loss": 3.581762230557544,
+ "tokens_seen": 1332936704
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048173319959879634,
+ "loss": 1.1409,
+ "theoretical_loss": 3.5817464764863107,
+ "tokens_seen": 1333002240
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004817171514543631,
+ "loss": 1.2467,
+ "theoretical_loss": 3.5817307234064524,
+ "tokens_seen": 1333067776
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048170110330992974,
+ "loss": 1.4078,
+ "theoretical_loss": 3.5817149713178584,
+ "tokens_seen": 1333133312
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004816850551654965,
+ "loss": 1.4079,
+ "theoretical_loss": 3.5816992202204183,
+ "tokens_seen": 1333198848
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048166900702106325,
+ "loss": 1.389,
+ "theoretical_loss": 3.5816834701140197,
+ "tokens_seen": 1333264384
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004816529588766299,
+ "loss": 1.2974,
+ "theoretical_loss": 3.5816677209985524,
+ "tokens_seen": 1333329920
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048163691073219666,
+ "loss": 1.3524,
+ "theoretical_loss": 3.581651972873905,
+ "tokens_seen": 1333395456
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004816208625877633,
+ "loss": 1.3615,
+ "theoretical_loss": 3.5816362257399668,
+ "tokens_seen": 1333460992
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048160481444333006,
+ "loss": 1.2211,
+ "theoretical_loss": 3.5816204795966264,
+ "tokens_seen": 1333526528
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004815887662988967,
+ "loss": 1.3693,
+ "theoretical_loss": 3.5816047344437734,
+ "tokens_seen": 1333592064
+ },
+ {
+ "epoch": 0.4,
+ "objective/train/docs_used": 447424,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4249781370162964,
+ "objective/train/theoretical_loss": 3.581588990281296,
+ "objective/train/tokens_used": 1354117600,
+ "theoretical_loss": 3.581588990281296,
+ "tokens_seen": 1333657600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048157271815446346,
+ "loss": 1.3495,
+ "theoretical_loss": 3.581588990281296,
+ "tokens_seen": 1333657600
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004815566700100301,
+ "loss": 1.4024,
+ "theoretical_loss": 3.581573247109084,
+ "tokens_seen": 1333723136
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048154062186559686,
+ "loss": 1.2595,
+ "theoretical_loss": 3.581557504927026,
+ "tokens_seen": 1333788672
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004815245737211635,
+ "loss": 1.3404,
+ "theoretical_loss": 3.581541763735011,
+ "tokens_seen": 1333854208
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048150852557673026,
+ "loss": 1.4137,
+ "theoretical_loss": 3.5815260235329287,
+ "tokens_seen": 1333919744
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004814924774322969,
+ "loss": 1.3797,
+ "theoretical_loss": 3.5815102843206676,
+ "tokens_seen": 1333985280
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048147642928786366,
+ "loss": 1.1652,
+ "theoretical_loss": 3.581494546098117,
+ "tokens_seen": 1334050816
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004814603811434303,
+ "loss": 1.3512,
+ "theoretical_loss": 3.581478808865166,
+ "tokens_seen": 1334116352
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048144433299899706,
+ "loss": 1.3896,
+ "theoretical_loss": 3.5814630726217045,
+ "tokens_seen": 1334181888
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004814282848545637,
+ "loss": 1.4512,
+ "theoretical_loss": 3.5814473373676203,
+ "tokens_seen": 1334247424
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048141223671013046,
+ "loss": 1.362,
+ "theoretical_loss": 3.5814316031028035,
+ "tokens_seen": 1334312960
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004813961885656971,
+ "loss": 1.3385,
+ "theoretical_loss": 3.5814158698271434,
+ "tokens_seen": 1334378496
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048138014042126387,
+ "loss": 1.3252,
+ "theoretical_loss": 3.581400137540529,
+ "tokens_seen": 1334444032
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004813640922768305,
+ "loss": 1.3871,
+ "theoretical_loss": 3.58138440624285,
+ "tokens_seen": 1334509568
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048134804413239727,
+ "loss": 1.2668,
+ "theoretical_loss": 3.5813686759339944,
+ "tokens_seen": 1334575104
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004813319959879639,
+ "loss": 1.322,
+ "theoretical_loss": 3.581352946613853,
+ "tokens_seen": 1334640640
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048131594784353067,
+ "loss": 1.2272,
+ "theoretical_loss": 3.5813372182823136,
+ "tokens_seen": 1334706176
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812998996990973,
+ "loss": 1.1931,
+ "theoretical_loss": 3.581321490939267,
+ "tokens_seen": 1334771712
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048128385155466407,
+ "loss": 1.3972,
+ "theoretical_loss": 3.5813057645846023,
+ "tokens_seen": 1334837248
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812678034102307,
+ "loss": 1.2984,
+ "theoretical_loss": 3.581290039218208,
+ "tokens_seen": 1334902784
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812517552657974,
+ "loss": 1.3355,
+ "theoretical_loss": 3.581274314839974,
+ "tokens_seen": 1334968320
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812357071213641,
+ "loss": 1.4606,
+ "theoretical_loss": 3.5812585914497896,
+ "tokens_seen": 1335033856
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812196589769308,
+ "loss": 1.4568,
+ "theoretical_loss": 3.5812428690475446,
+ "tokens_seen": 1335099392
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004812036108324975,
+ "loss": 1.2637,
+ "theoretical_loss": 3.5812271476331277,
+ "tokens_seen": 1335164928
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004811875626880642,
+ "loss": 1.2876,
+ "theoretical_loss": 3.581211427206429,
+ "tokens_seen": 1335230464
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004811715145436309,
+ "loss": 1.3924,
+ "theoretical_loss": 3.5811957077673373,
+ "tokens_seen": 1335296000
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004811554663991976,
+ "loss": 1.3643,
+ "theoretical_loss": 3.5811799893157437,
+ "tokens_seen": 1335361536
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004811394182547643,
+ "loss": 1.3821,
+ "theoretical_loss": 3.5811642718515353,
+ "tokens_seen": 1335427072
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.000481123370110331,
+ "loss": 1.2616,
+ "theoretical_loss": 3.5811485553746034,
+ "tokens_seen": 1335492608
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048110732196589767,
+ "loss": 1.2583,
+ "theoretical_loss": 3.5811328398848374,
+ "tokens_seen": 1335558144
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004810912738214644,
+ "loss": 1.3647,
+ "theoretical_loss": 3.581117125382126,
+ "tokens_seen": 1335623680
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048107522567703107,
+ "loss": 1.2947,
+ "theoretical_loss": 3.581101411866359,
+ "tokens_seen": 1335689216
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004810591775325978,
+ "loss": 1.2276,
+ "theoretical_loss": 3.581085699337427,
+ "tokens_seen": 1335754752
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048104312938816447,
+ "loss": 1.4068,
+ "theoretical_loss": 3.5810699877952183,
+ "tokens_seen": 1335820288
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004810270812437312,
+ "loss": 1.2359,
+ "theoretical_loss": 3.5810542772396228,
+ "tokens_seen": 1335885824
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048101103309929787,
+ "loss": 1.3078,
+ "theoretical_loss": 3.5810385676705305,
+ "tokens_seen": 1335951360
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004809949849548646,
+ "loss": 1.3726,
+ "theoretical_loss": 3.581022859087832,
+ "tokens_seen": 1336016896
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004809789368104313,
+ "loss": 1.2807,
+ "theoretical_loss": 3.581007151491415,
+ "tokens_seen": 1336082432
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048096288866599803,
+ "loss": 1.3501,
+ "theoretical_loss": 3.58099144488117,
+ "tokens_seen": 1336147968
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004809468405215647,
+ "loss": 1.4591,
+ "theoretical_loss": 3.5809757392569876,
+ "tokens_seen": 1336213504
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048093079237713143,
+ "loss": 1.2377,
+ "theoretical_loss": 3.580960034618757,
+ "tokens_seen": 1336279040
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004809147442326981,
+ "loss": 1.349,
+ "theoretical_loss": 3.580944330966367,
+ "tokens_seen": 1336344576
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.00048089869608826483,
+ "loss": 1.3822,
+ "theoretical_loss": 3.5809286282997084,
+ "tokens_seen": 1336410112
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 0.0004808826479438315,
+ "loss": 1.3179,
+ "theoretical_loss": 3.580912926618671,
+ "tokens_seen": 1336475648
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048086659979939823,
+ "loss": 1.3979,
+ "theoretical_loss": 3.5808972259231444,
+ "tokens_seen": 1336541184
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004808505516549649,
+ "loss": 1.3848,
+ "theoretical_loss": 3.5808815262130183,
+ "tokens_seen": 1336606720
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048083450351053163,
+ "loss": 1.4087,
+ "theoretical_loss": 3.5808658274881826,
+ "tokens_seen": 1336672256
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004808184553660983,
+ "loss": 1.3429,
+ "theoretical_loss": 3.5808501297485273,
+ "tokens_seen": 1336737792
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048080240722166503,
+ "loss": 1.4033,
+ "theoretical_loss": 3.5808344329939423,
+ "tokens_seen": 1336803328
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004807863590772317,
+ "loss": 1.3355,
+ "theoretical_loss": 3.580818737224317,
+ "tokens_seen": 1336868864
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 448669,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3390278816223145,
+ "objective/train/theoretical_loss": 3.5808030424395416,
+ "objective/train/tokens_used": 1357394400,
+ "theoretical_loss": 3.5808030424395416,
+ "tokens_seen": 1336934400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048077031093279844,
+ "loss": 1.3454,
+ "theoretical_loss": 3.5808030424395416,
+ "tokens_seen": 1336934400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004807542627883651,
+ "loss": 1.361,
+ "theoretical_loss": 3.5807873486395065,
+ "tokens_seen": 1336999936
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048073821464393184,
+ "loss": 1.3739,
+ "theoretical_loss": 3.580771655824101,
+ "tokens_seen": 1337065472
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004807221664994985,
+ "loss": 1.4102,
+ "theoretical_loss": 3.5807559639932154,
+ "tokens_seen": 1337131008
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048070611835506524,
+ "loss": 1.37,
+ "theoretical_loss": 3.5807402731467395,
+ "tokens_seen": 1337196544
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004806900702106319,
+ "loss": 1.338,
+ "theoretical_loss": 3.5807245832845638,
+ "tokens_seen": 1337262080
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048067402206619864,
+ "loss": 1.3267,
+ "theoretical_loss": 3.5807088944065777,
+ "tokens_seen": 1337327616
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004806579739217653,
+ "loss": 1.2232,
+ "theoretical_loss": 3.580693206512671,
+ "tokens_seen": 1337393152
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048064192577733204,
+ "loss": 1.3858,
+ "theoretical_loss": 3.580677519602735,
+ "tokens_seen": 1337458688
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004806258776328987,
+ "loss": 1.2465,
+ "theoretical_loss": 3.580661833676659,
+ "tokens_seen": 1337524224
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048060982948846544,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5806461487343326,
+ "tokens_seen": 1337589760
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004805937813440321,
+ "loss": 1.2402,
+ "theoretical_loss": 3.5806304647756466,
+ "tokens_seen": 1337655296
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048057773319959884,
+ "loss": 1.3859,
+ "theoretical_loss": 3.5806147818004908,
+ "tokens_seen": 1337720832
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004805616850551655,
+ "loss": 1.4221,
+ "theoretical_loss": 3.5805990998087562,
+ "tokens_seen": 1337786368
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048054563691073224,
+ "loss": 1.2923,
+ "theoretical_loss": 3.5805834188003316,
+ "tokens_seen": 1337851904
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004805295887662989,
+ "loss": 1.3699,
+ "theoretical_loss": 3.580567738775108,
+ "tokens_seen": 1337917440
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048051354062186565,
+ "loss": 1.3643,
+ "theoretical_loss": 3.580552059732975,
+ "tokens_seen": 1337982976
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004804974924774323,
+ "loss": 1.4674,
+ "theoretical_loss": 3.5805363816738236,
+ "tokens_seen": 1338048512
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000480481444332999,
+ "loss": 1.2965,
+ "theoretical_loss": 3.580520704597544,
+ "tokens_seen": 1338114048
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048046539618856575,
+ "loss": 1.3388,
+ "theoretical_loss": 3.5805050285040254,
+ "tokens_seen": 1338179584
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004804493480441324,
+ "loss": 1.3883,
+ "theoretical_loss": 3.580489353393159,
+ "tokens_seen": 1338245120
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048043329989969915,
+ "loss": 1.2544,
+ "theoretical_loss": 3.580473679264835,
+ "tokens_seen": 1338310656
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004804172517552658,
+ "loss": 1.2456,
+ "theoretical_loss": 3.5804580061189437,
+ "tokens_seen": 1338376192
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048040120361083255,
+ "loss": 1.268,
+ "theoretical_loss": 3.580442333955375,
+ "tokens_seen": 1338441728
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004803851554663992,
+ "loss": 1.4147,
+ "theoretical_loss": 3.5804266627740198,
+ "tokens_seen": 1338507264
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048036910732196595,
+ "loss": 1.2712,
+ "theoretical_loss": 3.580410992574768,
+ "tokens_seen": 1338572800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004803530591775326,
+ "loss": 1.2593,
+ "theoretical_loss": 3.5803953233575103,
+ "tokens_seen": 1338638336
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048033701103309935,
+ "loss": 1.2848,
+ "theoretical_loss": 3.5803796551221367,
+ "tokens_seen": 1338703872
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000480320962888666,
+ "loss": 1.3717,
+ "theoretical_loss": 3.5803639878685383,
+ "tokens_seen": 1338769408
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048030491474423275,
+ "loss": 1.304,
+ "theoretical_loss": 3.5803483215966048,
+ "tokens_seen": 1338834944
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004802888665997994,
+ "loss": 1.3585,
+ "theoretical_loss": 3.580332656306227,
+ "tokens_seen": 1338900480
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048027281845536616,
+ "loss": 1.3024,
+ "theoretical_loss": 3.580316991997295,
+ "tokens_seen": 1338966016
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004802567703109328,
+ "loss": 1.3721,
+ "theoretical_loss": 3.5803013286697,
+ "tokens_seen": 1339031552
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048024072216649956,
+ "loss": 1.3501,
+ "theoretical_loss": 3.5802856663233316,
+ "tokens_seen": 1339097088
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004802246740220662,
+ "loss": 1.339,
+ "theoretical_loss": 3.580270004958081,
+ "tokens_seen": 1339162624
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048020862587763296,
+ "loss": 1.2659,
+ "theoretical_loss": 3.580254344573839,
+ "tokens_seen": 1339228160
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004801925777331996,
+ "loss": 1.3614,
+ "theoretical_loss": 3.580238685170495,
+ "tokens_seen": 1339293696
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048017652958876636,
+ "loss": 1.3372,
+ "theoretical_loss": 3.58022302674794,
+ "tokens_seen": 1339359232
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000480160481444333,
+ "loss": 1.3331,
+ "theoretical_loss": 3.5802073693060654,
+ "tokens_seen": 1339424768
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048014443329989976,
+ "loss": 1.3586,
+ "theoretical_loss": 3.580191712844761,
+ "tokens_seen": 1339490304
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004801283851554664,
+ "loss": 1.3113,
+ "theoretical_loss": 3.580176057363918,
+ "tokens_seen": 1339555840
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048011233701103316,
+ "loss": 1.307,
+ "theoretical_loss": 3.5801604028634264,
+ "tokens_seen": 1339621376
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004800962888665998,
+ "loss": 1.3671,
+ "theoretical_loss": 3.5801447493431766,
+ "tokens_seen": 1339686912
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048008024072216656,
+ "loss": 1.3435,
+ "theoretical_loss": 3.5801290968030606,
+ "tokens_seen": 1339752448
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004800641925777332,
+ "loss": 1.3388,
+ "theoretical_loss": 3.5801134452429677,
+ "tokens_seen": 1339817984
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048004814443329996,
+ "loss": 1.2955,
+ "theoretical_loss": 3.580097794662789,
+ "tokens_seen": 1339883520
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004800320962888666,
+ "loss": 1.3901,
+ "theoretical_loss": 3.5800821450624163,
+ "tokens_seen": 1339949056
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048001604814443337,
+ "loss": 1.3408,
+ "theoretical_loss": 3.580066496441739,
+ "tokens_seen": 1340014592
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00048,
+ "loss": 1.4272,
+ "theoretical_loss": 3.5800508488006484,
+ "tokens_seen": 1340080128
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047998395185556677,
+ "loss": 1.3211,
+ "theoretical_loss": 3.580035202139035,
+ "tokens_seen": 1340145664
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 450060,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.155937910079956,
+ "objective/train/theoretical_loss": 3.58001955645679,
+ "objective/train/tokens_used": 1360671200,
+ "theoretical_loss": 3.58001955645679,
+ "tokens_seen": 1340211200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004799679037111334,
+ "loss": 1.2487,
+ "theoretical_loss": 3.58001955645679,
+ "tokens_seen": 1340211200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047995185556670017,
+ "loss": 1.2357,
+ "theoretical_loss": 3.5800039117538036,
+ "tokens_seen": 1340276736
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004799358074222668,
+ "loss": 1.2733,
+ "theoretical_loss": 3.5799882680299673,
+ "tokens_seen": 1340342272
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047991975927783357,
+ "loss": 1.2845,
+ "theoretical_loss": 3.579972625285172,
+ "tokens_seen": 1340407808
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004799037111334002,
+ "loss": 1.3828,
+ "theoretical_loss": 3.579956983519308,
+ "tokens_seen": 1340473344
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047988766298896697,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5799413427322664,
+ "tokens_seen": 1340538880
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004798716148445336,
+ "loss": 1.2779,
+ "theoretical_loss": 3.579925702923938,
+ "tokens_seen": 1340604416
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047985556670010037,
+ "loss": 1.3103,
+ "theoretical_loss": 3.5799100640942134,
+ "tokens_seen": 1340669952
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000479839518555667,
+ "loss": 1.2648,
+ "theoretical_loss": 3.5798944262429853,
+ "tokens_seen": 1340735488
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004798234704112337,
+ "loss": 1.2802,
+ "theoretical_loss": 3.579878789370142,
+ "tokens_seen": 1340801024
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004798074222668004,
+ "loss": 1.4822,
+ "theoretical_loss": 3.5798631534755767,
+ "tokens_seen": 1340866560
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004797913741223671,
+ "loss": 1.4289,
+ "theoretical_loss": 3.5798475185591787,
+ "tokens_seen": 1340932096
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004797753259779338,
+ "loss": 1.3829,
+ "theoretical_loss": 3.57983188462084,
+ "tokens_seen": 1340997632
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004797592778335005,
+ "loss": 1.4056,
+ "theoretical_loss": 3.5798162516604517,
+ "tokens_seen": 1341063168
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004797432296890672,
+ "loss": 1.3528,
+ "theoretical_loss": 3.579800619677905,
+ "tokens_seen": 1341128704
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004797271815446339,
+ "loss": 1.4509,
+ "theoretical_loss": 3.5797849886730897,
+ "tokens_seen": 1341194240
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047971113340020057,
+ "loss": 1.3058,
+ "theoretical_loss": 3.579769358645898,
+ "tokens_seen": 1341259776
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004796950852557673,
+ "loss": 1.4055,
+ "theoretical_loss": 3.5797537295962205,
+ "tokens_seen": 1341325312
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047967903711133397,
+ "loss": 1.2921,
+ "theoretical_loss": 3.579738101523949,
+ "tokens_seen": 1341390848
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004796629889669007,
+ "loss": 1.4404,
+ "theoretical_loss": 3.5797224744289737,
+ "tokens_seen": 1341456384
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047964694082246737,
+ "loss": 1.3229,
+ "theoretical_loss": 3.579706848311186,
+ "tokens_seen": 1341521920
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004796308926780341,
+ "loss": 1.308,
+ "theoretical_loss": 3.579691223170477,
+ "tokens_seen": 1341587456
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004796148445336008,
+ "loss": 1.2353,
+ "theoretical_loss": 3.579675599006739,
+ "tokens_seen": 1341652992
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047959879638916753,
+ "loss": 1.2569,
+ "theoretical_loss": 3.5796599758198617,
+ "tokens_seen": 1341718528
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004795827482447342,
+ "loss": 1.4666,
+ "theoretical_loss": 3.579644353609737,
+ "tokens_seen": 1341784064
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047956670010030093,
+ "loss": 1.2893,
+ "theoretical_loss": 3.579628732376256,
+ "tokens_seen": 1341849600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004795506519558676,
+ "loss": 1.3149,
+ "theoretical_loss": 3.57961311211931,
+ "tokens_seen": 1341915136
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047953460381143433,
+ "loss": 1.396,
+ "theoretical_loss": 3.5795974928387904,
+ "tokens_seen": 1341980672
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000479518555667001,
+ "loss": 1.3927,
+ "theoretical_loss": 3.579581874534588,
+ "tokens_seen": 1342046208
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047950250752256773,
+ "loss": 1.4129,
+ "theoretical_loss": 3.5795662572065945,
+ "tokens_seen": 1342111744
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004794864593781344,
+ "loss": 1.2728,
+ "theoretical_loss": 3.5795506408547015,
+ "tokens_seen": 1342177280
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047947041123370113,
+ "loss": 1.4039,
+ "theoretical_loss": 3.5795350254788,
+ "tokens_seen": 1342242816
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004794543630892678,
+ "loss": 1.4029,
+ "theoretical_loss": 3.579519411078781,
+ "tokens_seen": 1342308352
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047943831494483453,
+ "loss": 1.4798,
+ "theoretical_loss": 3.5795037976545365,
+ "tokens_seen": 1342373888
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004794222668004012,
+ "loss": 1.3412,
+ "theoretical_loss": 3.5794881852059577,
+ "tokens_seen": 1342439424
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047940621865596794,
+ "loss": 1.431,
+ "theoretical_loss": 3.5794725737329354,
+ "tokens_seen": 1342504960
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004793901705115346,
+ "loss": 1.2976,
+ "theoretical_loss": 3.579456963235362,
+ "tokens_seen": 1342570496
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047937412236710134,
+ "loss": 1.314,
+ "theoretical_loss": 3.579441353713128,
+ "tokens_seen": 1342636032
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000479358074222668,
+ "loss": 1.3727,
+ "theoretical_loss": 3.5794257451661258,
+ "tokens_seen": 1342701568
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047934202607823474,
+ "loss": 1.4277,
+ "theoretical_loss": 3.5794101375942464,
+ "tokens_seen": 1342767104
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004793259779338014,
+ "loss": 1.2167,
+ "theoretical_loss": 3.579394530997381,
+ "tokens_seen": 1342832640
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047930992978936814,
+ "loss": 1.2519,
+ "theoretical_loss": 3.579378925375422,
+ "tokens_seen": 1342898176
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004792938816449348,
+ "loss": 1.3776,
+ "theoretical_loss": 3.5793633207282594,
+ "tokens_seen": 1342963712
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047927783350050154,
+ "loss": 1.3619,
+ "theoretical_loss": 3.579347717055786,
+ "tokens_seen": 1343029248
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004792617853560683,
+ "loss": 1.3105,
+ "theoretical_loss": 3.5793321143578933,
+ "tokens_seen": 1343094784
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047924573721163494,
+ "loss": 1.3094,
+ "theoretical_loss": 3.5793165126344726,
+ "tokens_seen": 1343160320
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004792296890672017,
+ "loss": 1.2959,
+ "theoretical_loss": 3.579300911885415,
+ "tokens_seen": 1343225856
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047921364092276834,
+ "loss": 1.3763,
+ "theoretical_loss": 3.5792853121106134,
+ "tokens_seen": 1343291392
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047919759277833504,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5792697133099587,
+ "tokens_seen": 1343356928
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047918154463390174,
+ "loss": 1.3474,
+ "theoretical_loss": 3.5792541154833417,
+ "tokens_seen": 1343422464
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 450997,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3282132148742676,
+ "objective/train/theoretical_loss": 3.579238518630656,
+ "objective/train/tokens_used": 1363948000,
+ "theoretical_loss": 3.579238518630656,
+ "tokens_seen": 1343488000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047916549648946845,
+ "loss": 1.2727,
+ "theoretical_loss": 3.579238518630656,
+ "tokens_seen": 1343488000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047914944834503515,
+ "loss": 1.4423,
+ "theoretical_loss": 3.579222922751791,
+ "tokens_seen": 1343553536
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047913340020060185,
+ "loss": 1.3274,
+ "theoretical_loss": 3.5792073278466403,
+ "tokens_seen": 1343619072
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047911735205616855,
+ "loss": 1.2289,
+ "theoretical_loss": 3.5791917339150947,
+ "tokens_seen": 1343684608
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047910130391173525,
+ "loss": 1.3859,
+ "theoretical_loss": 3.579176140957046,
+ "tokens_seen": 1343750144
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047908525576730195,
+ "loss": 1.3617,
+ "theoretical_loss": 3.579160548972386,
+ "tokens_seen": 1343815680
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047906920762286865,
+ "loss": 1.292,
+ "theoretical_loss": 3.579144957961007,
+ "tokens_seen": 1343881216
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004790531594784353,
+ "loss": 1.3962,
+ "theoretical_loss": 3.5791293679228,
+ "tokens_seen": 1343946752
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047903711133400205,
+ "loss": 1.2153,
+ "theoretical_loss": 3.579113778857657,
+ "tokens_seen": 1344012288
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004790210631895687,
+ "loss": 1.347,
+ "theoretical_loss": 3.57909819076547,
+ "tokens_seen": 1344077824
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047900501504513545,
+ "loss": 1.3906,
+ "theoretical_loss": 3.579082603646131,
+ "tokens_seen": 1344143360
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004789889669007021,
+ "loss": 1.2948,
+ "theoretical_loss": 3.579067017499532,
+ "tokens_seen": 1344208896
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047897291875626885,
+ "loss": 1.475,
+ "theoretical_loss": 3.5790514323255636,
+ "tokens_seen": 1344274432
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004789568706118355,
+ "loss": 1.258,
+ "theoretical_loss": 3.579035848124119,
+ "tokens_seen": 1344339968
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047894082246740225,
+ "loss": 1.3343,
+ "theoretical_loss": 3.5790202648950897,
+ "tokens_seen": 1344405504
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004789247743229689,
+ "loss": 1.4002,
+ "theoretical_loss": 3.579004682638368,
+ "tokens_seen": 1344471040
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047890872617853566,
+ "loss": 1.295,
+ "theoretical_loss": 3.5789891013538453,
+ "tokens_seen": 1344536576
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004788926780341023,
+ "loss": 1.3492,
+ "theoretical_loss": 3.5789735210414135,
+ "tokens_seen": 1344602112
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047887662988966906,
+ "loss": 1.371,
+ "theoretical_loss": 3.578957941700965,
+ "tokens_seen": 1344667648
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004788605817452357,
+ "loss": 1.274,
+ "theoretical_loss": 3.578942363332392,
+ "tokens_seen": 1344733184
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047884453360080246,
+ "loss": 1.3448,
+ "theoretical_loss": 3.5789267859355856,
+ "tokens_seen": 1344798720
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004788284854563691,
+ "loss": 1.3471,
+ "theoretical_loss": 3.5789112095104385,
+ "tokens_seen": 1344864256
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047881243731193586,
+ "loss": 1.3998,
+ "theoretical_loss": 3.5788956340568427,
+ "tokens_seen": 1344929792
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004787963891675025,
+ "loss": 1.3067,
+ "theoretical_loss": 3.57888005957469,
+ "tokens_seen": 1344995328
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047878034102306926,
+ "loss": 1.3002,
+ "theoretical_loss": 3.578864486063873,
+ "tokens_seen": 1345060864
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004787642928786359,
+ "loss": 1.3365,
+ "theoretical_loss": 3.578848913524283,
+ "tokens_seen": 1345126400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047874824473420266,
+ "loss": 1.3591,
+ "theoretical_loss": 3.578833341955813,
+ "tokens_seen": 1345191936
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004787321965897693,
+ "loss": 1.3407,
+ "theoretical_loss": 3.5788177713583544,
+ "tokens_seen": 1345257472
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047871614844533606,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5788022017317997,
+ "tokens_seen": 1345323008
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004787001003009027,
+ "loss": 1.4138,
+ "theoretical_loss": 3.5787866330760414,
+ "tokens_seen": 1345388544
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047868405215646946,
+ "loss": 1.3204,
+ "theoretical_loss": 3.5787710653909706,
+ "tokens_seen": 1345454080
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004786680040120361,
+ "loss": 1.3798,
+ "theoretical_loss": 3.5787554986764807,
+ "tokens_seen": 1345519616
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047865195586760287,
+ "loss": 1.3444,
+ "theoretical_loss": 3.5787399329324634,
+ "tokens_seen": 1345585152
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004786359077231695,
+ "loss": 1.3323,
+ "theoretical_loss": 3.5787243681588112,
+ "tokens_seen": 1345650688
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047861985957873627,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5787088043554154,
+ "tokens_seen": 1345716224
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004786038114343029,
+ "loss": 1.325,
+ "theoretical_loss": 3.5786932415221697,
+ "tokens_seen": 1345781760
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047858776328986967,
+ "loss": 1.3466,
+ "theoretical_loss": 3.578677679658965,
+ "tokens_seen": 1345847296
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004785717151454363,
+ "loss": 1.3353,
+ "theoretical_loss": 3.5786621187656946,
+ "tokens_seen": 1345912832
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047855566700100307,
+ "loss": 1.3241,
+ "theoretical_loss": 3.5786465588422507,
+ "tokens_seen": 1345978368
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004785396188565697,
+ "loss": 1.299,
+ "theoretical_loss": 3.5786309998885253,
+ "tokens_seen": 1346043904
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047852357071213647,
+ "loss": 1.3289,
+ "theoretical_loss": 3.578615441904411,
+ "tokens_seen": 1346109440
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004785075225677031,
+ "loss": 1.2738,
+ "theoretical_loss": 3.5785998848897997,
+ "tokens_seen": 1346174976
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047849147442326987,
+ "loss": 1.3434,
+ "theoretical_loss": 3.5785843288445847,
+ "tokens_seen": 1346240512
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784754262788365,
+ "loss": 1.3577,
+ "theoretical_loss": 3.5785687737686573,
+ "tokens_seen": 1346306048
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784593781344033,
+ "loss": 1.389,
+ "theoretical_loss": 3.5785532196619108,
+ "tokens_seen": 1346371584
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784593781344033,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5785376665242374,
+ "tokens_seen": 1346437120
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784433299899699,
+ "loss": 1.3693,
+ "theoretical_loss": 3.578522114355529,
+ "tokens_seen": 1346502656
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784272818455366,
+ "loss": 1.2745,
+ "theoretical_loss": 3.5785065631556794,
+ "tokens_seen": 1346568192
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004784112337011033,
+ "loss": 1.2882,
+ "theoretical_loss": 3.5784910129245793,
+ "tokens_seen": 1346633728
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047839518555667,
+ "loss": 1.3367,
+ "theoretical_loss": 3.578475463662123,
+ "tokens_seen": 1346699264
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 452300,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2804325819015503,
+ "objective/train/theoretical_loss": 3.5784599153682017,
+ "objective/train/tokens_used": 1367224800,
+ "theoretical_loss": 3.5784599153682017,
+ "tokens_seen": 1346764800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004783791374122367,
+ "loss": 1.2656,
+ "theoretical_loss": 3.5784599153682017,
+ "tokens_seen": 1346764800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004783630892678034,
+ "loss": 1.2759,
+ "theoretical_loss": 3.578444368042709,
+ "tokens_seen": 1346830336
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004783470411233701,
+ "loss": 1.4242,
+ "theoretical_loss": 3.578428821685536,
+ "tokens_seen": 1346895872
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004783309929789368,
+ "loss": 1.2695,
+ "theoretical_loss": 3.578413276296577,
+ "tokens_seen": 1346961408
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004783149448345035,
+ "loss": 1.3269,
+ "theoretical_loss": 3.5783977318757234,
+ "tokens_seen": 1347026944
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004782988966900702,
+ "loss": 1.1833,
+ "theoretical_loss": 3.578382188422869,
+ "tokens_seen": 1347092480
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047828284854563687,
+ "loss": 1.3813,
+ "theoretical_loss": 3.578366645937905,
+ "tokens_seen": 1347158016
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004782668004012036,
+ "loss": 1.4579,
+ "theoretical_loss": 3.578351104420724,
+ "tokens_seen": 1347223552
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004782507522567703,
+ "loss": 1.2303,
+ "theoretical_loss": 3.57833556387122,
+ "tokens_seen": 1347289088
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047823470411233703,
+ "loss": 1.3805,
+ "theoretical_loss": 3.5783200242892854,
+ "tokens_seen": 1347354624
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004782186559679037,
+ "loss": 1.3677,
+ "theoretical_loss": 3.578304485674812,
+ "tokens_seen": 1347420160
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047820260782347043,
+ "loss": 1.313,
+ "theoretical_loss": 3.5782889480276934,
+ "tokens_seen": 1347485696
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004781865596790371,
+ "loss": 1.3977,
+ "theoretical_loss": 3.578273411347822,
+ "tokens_seen": 1347551232
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047817051153460383,
+ "loss": 1.3989,
+ "theoretical_loss": 3.57825787563509,
+ "tokens_seen": 1347616768
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004781544633901705,
+ "loss": 1.383,
+ "theoretical_loss": 3.578242340889391,
+ "tokens_seen": 1347682304
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047813841524573723,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5782268071106174,
+ "tokens_seen": 1347747840
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004781223671013039,
+ "loss": 1.2073,
+ "theoretical_loss": 3.5782112742986625,
+ "tokens_seen": 1347813376
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047810631895687063,
+ "loss": 1.328,
+ "theoretical_loss": 3.5781957424534183,
+ "tokens_seen": 1347878912
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004780902708124373,
+ "loss": 1.3598,
+ "theoretical_loss": 3.578180211574778,
+ "tokens_seen": 1347944448
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047807422266800403,
+ "loss": 1.3714,
+ "theoretical_loss": 3.578164681662635,
+ "tokens_seen": 1348009984
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004780581745235708,
+ "loss": 1.3467,
+ "theoretical_loss": 3.578149152716881,
+ "tokens_seen": 1348075520
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047804212637913744,
+ "loss": 1.4247,
+ "theoretical_loss": 3.57813362473741,
+ "tokens_seen": 1348141056
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004780260782347042,
+ "loss": 1.3423,
+ "theoretical_loss": 3.578118097724114,
+ "tokens_seen": 1348206592
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047801003009027084,
+ "loss": 1.3935,
+ "theoretical_loss": 3.578102571676887,
+ "tokens_seen": 1348272128
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004779939819458376,
+ "loss": 1.3446,
+ "theoretical_loss": 3.578087046595621,
+ "tokens_seen": 1348337664
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047797793380140424,
+ "loss": 1.3589,
+ "theoretical_loss": 3.5780715224802093,
+ "tokens_seen": 1348403200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000477961885656971,
+ "loss": 1.2795,
+ "theoretical_loss": 3.5780559993305445,
+ "tokens_seen": 1348468736
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047794583751253764,
+ "loss": 1.2751,
+ "theoretical_loss": 3.5780404771465206,
+ "tokens_seen": 1348534272
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004779297893681044,
+ "loss": 1.3591,
+ "theoretical_loss": 3.5780249559280297,
+ "tokens_seen": 1348599808
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047791374122367104,
+ "loss": 1.2809,
+ "theoretical_loss": 3.578009435674965,
+ "tokens_seen": 1348665344
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004778976930792378,
+ "loss": 1.3244,
+ "theoretical_loss": 3.5779939163872196,
+ "tokens_seen": 1348730880
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047788164493480444,
+ "loss": 1.3271,
+ "theoretical_loss": 3.577978398064687,
+ "tokens_seen": 1348796416
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004778655967903712,
+ "loss": 1.3408,
+ "theoretical_loss": 3.577962880707259,
+ "tokens_seen": 1348861952
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047784954864593784,
+ "loss": 1.3476,
+ "theoretical_loss": 3.57794736431483,
+ "tokens_seen": 1348927488
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004778335005015046,
+ "loss": 1.3463,
+ "theoretical_loss": 3.5779318488872924,
+ "tokens_seen": 1348993024
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047781745235707124,
+ "loss": 1.2752,
+ "theoretical_loss": 3.57791633442454,
+ "tokens_seen": 1349058560
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047780140421263794,
+ "loss": 1.2638,
+ "theoretical_loss": 3.577900820926465,
+ "tokens_seen": 1349124096
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047778535606820465,
+ "loss": 1.3608,
+ "theoretical_loss": 3.577885308392962,
+ "tokens_seen": 1349189632
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047776930792377135,
+ "loss": 1.243,
+ "theoretical_loss": 3.577869796823922,
+ "tokens_seen": 1349255168
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047775325977933805,
+ "loss": 1.3345,
+ "theoretical_loss": 3.5778542862192406,
+ "tokens_seen": 1349320704
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047773721163490475,
+ "loss": 1.3975,
+ "theoretical_loss": 3.577838776578809,
+ "tokens_seen": 1349386240
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047772116349047145,
+ "loss": 1.373,
+ "theoretical_loss": 3.577823267902522,
+ "tokens_seen": 1349451776
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047770511534603815,
+ "loss": 1.2175,
+ "theoretical_loss": 3.5778077601902716,
+ "tokens_seen": 1349517312
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047768906720160485,
+ "loss": 1.2608,
+ "theoretical_loss": 3.577792253441952,
+ "tokens_seen": 1349582848
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047767301905717155,
+ "loss": 1.2634,
+ "theoretical_loss": 3.577776747657456,
+ "tokens_seen": 1349648384
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004776569709127382,
+ "loss": 1.3247,
+ "theoretical_loss": 3.5777612428366767,
+ "tokens_seen": 1349713920
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047764092276830495,
+ "loss": 1.2539,
+ "theoretical_loss": 3.577745738979508,
+ "tokens_seen": 1349779456
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004776248746238716,
+ "loss": 1.2019,
+ "theoretical_loss": 3.5777302360858423,
+ "tokens_seen": 1349844992
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047760882647943835,
+ "loss": 1.2937,
+ "theoretical_loss": 3.577714734155574,
+ "tokens_seen": 1349910528
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000477592778335005,
+ "loss": 1.3451,
+ "theoretical_loss": 3.5776992331885964,
+ "tokens_seen": 1349976064
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 453237,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.378117322921753,
+ "objective/train/theoretical_loss": 3.577683733184802,
+ "objective/train/tokens_used": 1370501600,
+ "theoretical_loss": 3.577683733184802,
+ "tokens_seen": 1350041600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047757673019057175,
+ "loss": 1.343,
+ "theoretical_loss": 3.577683733184802,
+ "tokens_seen": 1350041600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004775606820461384,
+ "loss": 1.2933,
+ "theoretical_loss": 3.577668234144085,
+ "tokens_seen": 1350107136
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047754463390170516,
+ "loss": 1.3646,
+ "theoretical_loss": 3.577652736066338,
+ "tokens_seen": 1350172672
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004775285857572718,
+ "loss": 1.2913,
+ "theoretical_loss": 3.5776372389514557,
+ "tokens_seen": 1350238208
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047751253761283856,
+ "loss": 1.2864,
+ "theoretical_loss": 3.57762174279933,
+ "tokens_seen": 1350303744
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004774964894684052,
+ "loss": 1.2866,
+ "theoretical_loss": 3.5776062476098565,
+ "tokens_seen": 1350369280
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047748044132397196,
+ "loss": 1.3274,
+ "theoretical_loss": 3.577590753382926,
+ "tokens_seen": 1350434816
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004774643931795386,
+ "loss": 1.3718,
+ "theoretical_loss": 3.577575260118434,
+ "tokens_seen": 1350500352
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047744834503510536,
+ "loss": 1.1962,
+ "theoretical_loss": 3.577559767816273,
+ "tokens_seen": 1350565888
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000477432296890672,
+ "loss": 1.3703,
+ "theoretical_loss": 3.5775442764763374,
+ "tokens_seen": 1350631424
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047741624874623876,
+ "loss": 1.377,
+ "theoretical_loss": 3.5775287860985197,
+ "tokens_seen": 1350696960
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004774002006018054,
+ "loss": 1.2782,
+ "theoretical_loss": 3.5775132966827146,
+ "tokens_seen": 1350762496
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047738415245737216,
+ "loss": 1.3029,
+ "theoretical_loss": 3.5774978082288147,
+ "tokens_seen": 1350828032
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004773681043129388,
+ "loss": 1.4288,
+ "theoretical_loss": 3.5774823207367143,
+ "tokens_seen": 1350893568
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047735205616850556,
+ "loss": 1.247,
+ "theoretical_loss": 3.5774668342063065,
+ "tokens_seen": 1350959104
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004773360080240722,
+ "loss": 1.2179,
+ "theoretical_loss": 3.5774513486374855,
+ "tokens_seen": 1351024640
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047731995987963896,
+ "loss": 1.3451,
+ "theoretical_loss": 3.5774358640301447,
+ "tokens_seen": 1351090176
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004773039117352056,
+ "loss": 1.3644,
+ "theoretical_loss": 3.577420380384177,
+ "tokens_seen": 1351155712
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047728786359077237,
+ "loss": 1.3545,
+ "theoretical_loss": 3.577404897699477,
+ "tokens_seen": 1351221248
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000477271815446339,
+ "loss": 1.3961,
+ "theoretical_loss": 3.5773894159759383,
+ "tokens_seen": 1351286784
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047725576730190577,
+ "loss": 1.3106,
+ "theoretical_loss": 3.577373935213455,
+ "tokens_seen": 1351352320
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004772397191574724,
+ "loss": 1.3005,
+ "theoretical_loss": 3.57735845541192,
+ "tokens_seen": 1351417856
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047722367101303917,
+ "loss": 1.3862,
+ "theoretical_loss": 3.5773429765712272,
+ "tokens_seen": 1351483392
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004772076228686058,
+ "loss": 1.3219,
+ "theoretical_loss": 3.5773274986912704,
+ "tokens_seen": 1351548928
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047719157472417257,
+ "loss": 1.3756,
+ "theoretical_loss": 3.577312021771944,
+ "tokens_seen": 1351614464
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004771755265797392,
+ "loss": 1.3361,
+ "theoretical_loss": 3.577296545813141,
+ "tokens_seen": 1351680000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047715947843530597,
+ "loss": 1.2426,
+ "theoretical_loss": 3.577281070814756,
+ "tokens_seen": 1351745536
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004771434302908726,
+ "loss": 1.349,
+ "theoretical_loss": 3.577265596776682,
+ "tokens_seen": 1351811072
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047712738214643937,
+ "loss": 1.3574,
+ "theoretical_loss": 3.5772501236988132,
+ "tokens_seen": 1351876608
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000477111334002006,
+ "loss": 1.3271,
+ "theoretical_loss": 3.577234651581044,
+ "tokens_seen": 1351942144
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047709528585757277,
+ "loss": 1.3236,
+ "theoretical_loss": 3.577219180423268,
+ "tokens_seen": 1352007680
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004770792377131394,
+ "loss": 1.2752,
+ "theoretical_loss": 3.577203710225378,
+ "tokens_seen": 1352073216
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004770631895687062,
+ "loss": 1.2766,
+ "theoretical_loss": 3.5771882409872697,
+ "tokens_seen": 1352138752
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004770471414242728,
+ "loss": 1.3151,
+ "theoretical_loss": 3.5771727727088356,
+ "tokens_seen": 1352204288
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004770310932798395,
+ "loss": 1.3158,
+ "theoretical_loss": 3.577157305389971,
+ "tokens_seen": 1352269824
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004770150451354062,
+ "loss": 1.3837,
+ "theoretical_loss": 3.5771418390305687,
+ "tokens_seen": 1352335360
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769989969909729,
+ "loss": 1.3076,
+ "theoretical_loss": 3.577126373630523,
+ "tokens_seen": 1352400896
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769829488465396,
+ "loss": 1.3721,
+ "theoretical_loss": 3.577110909189728,
+ "tokens_seen": 1352466432
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769669007021063,
+ "loss": 1.3973,
+ "theoretical_loss": 3.5770954457080784,
+ "tokens_seen": 1352531968
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000476950852557673,
+ "loss": 1.3649,
+ "theoretical_loss": 3.577079983185467,
+ "tokens_seen": 1352597504
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769348044132397,
+ "loss": 1.3097,
+ "theoretical_loss": 3.577064521621789,
+ "tokens_seen": 1352663040
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769187562688064,
+ "loss": 1.3028,
+ "theoretical_loss": 3.577049061016937,
+ "tokens_seen": 1352728576
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004769027081243731,
+ "loss": 1.3148,
+ "theoretical_loss": 3.5770336013708066,
+ "tokens_seen": 1352794112
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047688665997993977,
+ "loss": 1.337,
+ "theoretical_loss": 3.5770181426832917,
+ "tokens_seen": 1352859648
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047687061183550653,
+ "loss": 1.2915,
+ "theoretical_loss": 3.5770026849542855,
+ "tokens_seen": 1352925184
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004768545636910733,
+ "loss": 1.2745,
+ "theoretical_loss": 3.576987228183683,
+ "tokens_seen": 1352990720
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047683851554663993,
+ "loss": 1.263,
+ "theoretical_loss": 3.576971772371378,
+ "tokens_seen": 1353056256
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004768224674022067,
+ "loss": 1.3746,
+ "theoretical_loss": 3.576956317517264,
+ "tokens_seen": 1353121792
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047680641925777333,
+ "loss": 1.3158,
+ "theoretical_loss": 3.576940863621237,
+ "tokens_seen": 1353187328
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004767903711133401,
+ "loss": 1.2643,
+ "theoretical_loss": 3.57692541068319,
+ "tokens_seen": 1353252864
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 454374,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4121934175491333,
+ "objective/train/theoretical_loss": 3.5769099587030166,
+ "objective/train/tokens_used": 1373778400,
+ "theoretical_loss": 3.5769099587030166,
+ "tokens_seen": 1353318400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047677432296890673,
+ "loss": 1.3257,
+ "theoretical_loss": 3.5769099587030166,
+ "tokens_seen": 1353318400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004767582748244735,
+ "loss": 1.2999,
+ "theoretical_loss": 3.5768945076806125,
+ "tokens_seen": 1353383936
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047674222668004013,
+ "loss": 1.317,
+ "theoretical_loss": 3.576879057615871,
+ "tokens_seen": 1353449472
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004767261785356069,
+ "loss": 1.3362,
+ "theoretical_loss": 3.5768636085086865,
+ "tokens_seen": 1353515008
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047671013039117353,
+ "loss": 1.3674,
+ "theoretical_loss": 3.5768481603589537,
+ "tokens_seen": 1353580544
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004766940822467403,
+ "loss": 1.3929,
+ "theoretical_loss": 3.5768327131665667,
+ "tokens_seen": 1353646080
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047667803410230694,
+ "loss": 1.3458,
+ "theoretical_loss": 3.5768172669314193,
+ "tokens_seen": 1353711616
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004766619859578737,
+ "loss": 1.3494,
+ "theoretical_loss": 3.5768018216534068,
+ "tokens_seen": 1353777152
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047664593781344034,
+ "loss": 1.3724,
+ "theoretical_loss": 3.5767863773324233,
+ "tokens_seen": 1353842688
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004766298896690071,
+ "loss": 1.3222,
+ "theoretical_loss": 3.576770933968362,
+ "tokens_seen": 1353908224
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047661384152457374,
+ "loss": 1.4499,
+ "theoretical_loss": 3.5767554915611193,
+ "tokens_seen": 1353973760
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004765977933801405,
+ "loss": 1.3342,
+ "theoretical_loss": 3.5767400501105877,
+ "tokens_seen": 1354039296
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047658174523570714,
+ "loss": 1.3485,
+ "theoretical_loss": 3.5767246096166634,
+ "tokens_seen": 1354104832
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004765656970912739,
+ "loss": 1.2786,
+ "theoretical_loss": 3.576709170079239,
+ "tokens_seen": 1354170368
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047654964894684054,
+ "loss": 1.3495,
+ "theoretical_loss": 3.57669373149821,
+ "tokens_seen": 1354235904
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004765336008024073,
+ "loss": 1.3129,
+ "theoretical_loss": 3.576678293873471,
+ "tokens_seen": 1354301440
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047651755265797394,
+ "loss": 1.3542,
+ "theoretical_loss": 3.5766628572049166,
+ "tokens_seen": 1354366976
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004765015045135407,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5766474214924404,
+ "tokens_seen": 1354432512
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047648545636910734,
+ "loss": 1.3236,
+ "theoretical_loss": 3.5766319867359377,
+ "tokens_seen": 1354498048
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004764694082246741,
+ "loss": 1.3899,
+ "theoretical_loss": 3.5766165529353033,
+ "tokens_seen": 1354563584
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047645336008024074,
+ "loss": 1.308,
+ "theoretical_loss": 3.5766011200904306,
+ "tokens_seen": 1354629120
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004764373119358075,
+ "loss": 1.41,
+ "theoretical_loss": 3.5765856882012153,
+ "tokens_seen": 1354694656
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047642126379137415,
+ "loss": 1.3213,
+ "theoretical_loss": 3.5765702572675515,
+ "tokens_seen": 1354760192
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047640521564694085,
+ "loss": 1.3043,
+ "theoretical_loss": 3.5765548272893337,
+ "tokens_seen": 1354825728
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047638916750250755,
+ "loss": 1.3058,
+ "theoretical_loss": 3.5765393982664566,
+ "tokens_seen": 1354891264
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047637311935807425,
+ "loss": 1.2932,
+ "theoretical_loss": 3.5765239701988154,
+ "tokens_seen": 1354956800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047635707121364095,
+ "loss": 1.3814,
+ "theoretical_loss": 3.5765085430863044,
+ "tokens_seen": 1355022336
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047634102306920765,
+ "loss": 1.3512,
+ "theoretical_loss": 3.576493116928818,
+ "tokens_seen": 1355087872
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047632497492477435,
+ "loss": 1.3128,
+ "theoretical_loss": 3.576477691726251,
+ "tokens_seen": 1355153408
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047630892678034105,
+ "loss": 1.3136,
+ "theoretical_loss": 3.5764622674784983,
+ "tokens_seen": 1355218944
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047629287863590775,
+ "loss": 1.4381,
+ "theoretical_loss": 3.576446844185454,
+ "tokens_seen": 1355284480
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047627683049147445,
+ "loss": 1.2857,
+ "theoretical_loss": 3.576431421847014,
+ "tokens_seen": 1355350016
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004762607823470411,
+ "loss": 1.3775,
+ "theoretical_loss": 3.5764160004630723,
+ "tokens_seen": 1355415552
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047624473420260785,
+ "loss": 1.2717,
+ "theoretical_loss": 3.576400580033524,
+ "tokens_seen": 1355481088
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004762286860581745,
+ "loss": 1.2164,
+ "theoretical_loss": 3.5763851605582637,
+ "tokens_seen": 1355546624
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047621263791374125,
+ "loss": 1.3729,
+ "theoretical_loss": 3.576369742037186,
+ "tokens_seen": 1355612160
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004761965897693079,
+ "loss": 1.3306,
+ "theoretical_loss": 3.576354324470186,
+ "tokens_seen": 1355677696
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047618054162487465,
+ "loss": 1.2721,
+ "theoretical_loss": 3.5763389078571586,
+ "tokens_seen": 1355743232
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004761644934804413,
+ "loss": 1.3621,
+ "theoretical_loss": 3.5763234921979983,
+ "tokens_seen": 1355808768
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047614844533600806,
+ "loss": 1.3879,
+ "theoretical_loss": 3.576308077492601,
+ "tokens_seen": 1355874304
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004761323971915747,
+ "loss": 1.3844,
+ "theoretical_loss": 3.57629266374086,
+ "tokens_seen": 1355939840
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047611634904714146,
+ "loss": 1.2722,
+ "theoretical_loss": 3.5762772509426712,
+ "tokens_seen": 1356005376
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004761003009027081,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5762618390979295,
+ "tokens_seen": 1356070912
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047608425275827486,
+ "loss": 1.3334,
+ "theoretical_loss": 3.57624642820653,
+ "tokens_seen": 1356136448
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004760682046138415,
+ "loss": 1.3057,
+ "theoretical_loss": 3.576231018268367,
+ "tokens_seen": 1356201984
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047605215646940826,
+ "loss": 1.3257,
+ "theoretical_loss": 3.576215609283336,
+ "tokens_seen": 1356267520
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004760361083249749,
+ "loss": 1.366,
+ "theoretical_loss": 3.576200201251332,
+ "tokens_seen": 1356333056
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047602006018054166,
+ "loss": 1.3436,
+ "theoretical_loss": 3.5761847941722493,
+ "tokens_seen": 1356398592
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004760040120361083,
+ "loss": 1.3706,
+ "theoretical_loss": 3.576169388045984,
+ "tokens_seen": 1356464128
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047598796389167506,
+ "loss": 1.3702,
+ "theoretical_loss": 3.576153982872431,
+ "tokens_seen": 1356529664
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 455288,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3363687992095947,
+ "objective/train/theoretical_loss": 3.5761385786514843,
+ "objective/train/tokens_used": 1377055200,
+ "theoretical_loss": 3.5761385786514843,
+ "tokens_seen": 1356595200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004759719157472417,
+ "loss": 1.3978,
+ "theoretical_loss": 3.5761385786514843,
+ "tokens_seen": 1356595200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047595586760280846,
+ "loss": 1.323,
+ "theoretical_loss": 3.5761231753830405,
+ "tokens_seen": 1356660736
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004759398194583751,
+ "loss": 1.32,
+ "theoretical_loss": 3.576107773066993,
+ "tokens_seen": 1356726272
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047592377131394186,
+ "loss": 1.3252,
+ "theoretical_loss": 3.576092371703238,
+ "tokens_seen": 1356791808
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004759077231695085,
+ "loss": 1.2315,
+ "theoretical_loss": 3.5760769712916707,
+ "tokens_seen": 1356857344
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047589167502507527,
+ "loss": 1.2437,
+ "theoretical_loss": 3.576061571832186,
+ "tokens_seen": 1356922880
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004758756268806419,
+ "loss": 1.3991,
+ "theoretical_loss": 3.576046173324679,
+ "tokens_seen": 1356988416
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047585957873620867,
+ "loss": 1.3663,
+ "theoretical_loss": 3.576030775769045,
+ "tokens_seen": 1357053952
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004758435305917753,
+ "loss": 1.4817,
+ "theoretical_loss": 3.5760153791651783,
+ "tokens_seen": 1357119488
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047582748244734207,
+ "loss": 1.2868,
+ "theoretical_loss": 3.5759999835129754,
+ "tokens_seen": 1357185024
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004758114343029087,
+ "loss": 1.2257,
+ "theoretical_loss": 3.5759845888123314,
+ "tokens_seen": 1357250560
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047579538615847547,
+ "loss": 1.2162,
+ "theoretical_loss": 3.575969195063141,
+ "tokens_seen": 1357316096
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004757793380140421,
+ "loss": 1.3297,
+ "theoretical_loss": 3.5759538022652997,
+ "tokens_seen": 1357381632
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047576328986960887,
+ "loss": 1.4411,
+ "theoretical_loss": 3.5759384104187024,
+ "tokens_seen": 1357447168
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004757472417251755,
+ "loss": 1.2751,
+ "theoretical_loss": 3.575923019523245,
+ "tokens_seen": 1357512704
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047573119358074227,
+ "loss": 1.2694,
+ "theoretical_loss": 3.5759076295788224,
+ "tokens_seen": 1357578240
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004757151454363089,
+ "loss": 1.3627,
+ "theoretical_loss": 3.5758922405853304,
+ "tokens_seen": 1357643776
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756990972918757,
+ "loss": 1.3891,
+ "theoretical_loss": 3.5758768525426636,
+ "tokens_seen": 1357709312
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756830491474423,
+ "loss": 1.3531,
+ "theoretical_loss": 3.5758614654507177,
+ "tokens_seen": 1357774848
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756670010030091,
+ "loss": 1.3185,
+ "theoretical_loss": 3.5758460793093887,
+ "tokens_seen": 1357840384
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756509528585758,
+ "loss": 1.2617,
+ "theoretical_loss": 3.575830694118571,
+ "tokens_seen": 1357905920
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756349047141424,
+ "loss": 1.3435,
+ "theoretical_loss": 3.5758153098781604,
+ "tokens_seen": 1357971456
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756188565697092,
+ "loss": 1.3266,
+ "theoretical_loss": 3.5757999265880525,
+ "tokens_seen": 1358036992
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004756028084252758,
+ "loss": 1.4084,
+ "theoretical_loss": 3.5757845442481426,
+ "tokens_seen": 1358102528
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004755867602808426,
+ "loss": 1.2733,
+ "theoretical_loss": 3.575769162858326,
+ "tokens_seen": 1358168064
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004755707121364092,
+ "loss": 1.2879,
+ "theoretical_loss": 3.5757537824184986,
+ "tokens_seen": 1358233600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000475554663991976,
+ "loss": 1.2708,
+ "theoretical_loss": 3.5757384029285557,
+ "tokens_seen": 1358299136
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004755386158475426,
+ "loss": 1.3172,
+ "theoretical_loss": 3.5757230243883926,
+ "tokens_seen": 1358364672
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004755225677031094,
+ "loss": 1.3364,
+ "theoretical_loss": 3.5757076467979045,
+ "tokens_seen": 1358430208
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047550651955867603,
+ "loss": 1.358,
+ "theoretical_loss": 3.575692270156988,
+ "tokens_seen": 1358495744
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004754904714142428,
+ "loss": 1.3155,
+ "theoretical_loss": 3.5756768944655377,
+ "tokens_seen": 1358561280
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047547442326980943,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5756615197234503,
+ "tokens_seen": 1358626816
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004754583751253762,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5756461459306204,
+ "tokens_seen": 1358692352
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047544232698094283,
+ "loss": 1.4255,
+ "theoretical_loss": 3.575630773086943,
+ "tokens_seen": 1358757888
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004754262788365096,
+ "loss": 1.3004,
+ "theoretical_loss": 3.575615401192316,
+ "tokens_seen": 1358823424
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047541023069207623,
+ "loss": 1.3632,
+ "theoretical_loss": 3.5756000302466324,
+ "tokens_seen": 1358888960
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000475394182547643,
+ "loss": 1.4778,
+ "theoretical_loss": 3.575584660249789,
+ "tokens_seen": 1358954496
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047537813440320963,
+ "loss": 1.4276,
+ "theoretical_loss": 3.5755692912016825,
+ "tokens_seen": 1359020032
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004753620862587764,
+ "loss": 1.3964,
+ "theoretical_loss": 3.5755539231022073,
+ "tokens_seen": 1359085568
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047534603811434303,
+ "loss": 1.4651,
+ "theoretical_loss": 3.575538555951259,
+ "tokens_seen": 1359151104
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004753299899699098,
+ "loss": 1.4241,
+ "theoretical_loss": 3.5755231897487345,
+ "tokens_seen": 1359216640
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047531394182547643,
+ "loss": 1.2928,
+ "theoretical_loss": 3.5755078244945286,
+ "tokens_seen": 1359282176
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004752978936810432,
+ "loss": 1.3191,
+ "theoretical_loss": 3.5754924601885367,
+ "tokens_seen": 1359347712
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047528184553660984,
+ "loss": 1.2639,
+ "theoretical_loss": 3.5754770968306557,
+ "tokens_seen": 1359413248
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004752657973921766,
+ "loss": 1.3114,
+ "theoretical_loss": 3.575461734420781,
+ "tokens_seen": 1359478784
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047524974924774324,
+ "loss": 1.3596,
+ "theoretical_loss": 3.575446372958808,
+ "tokens_seen": 1359544320
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047523370110331,
+ "loss": 1.3917,
+ "theoretical_loss": 3.5754310124446325,
+ "tokens_seen": 1359609856
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047521765295887664,
+ "loss": 1.3005,
+ "theoretical_loss": 3.575415652878151,
+ "tokens_seen": 1359675392
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004752016048144434,
+ "loss": 1.3469,
+ "theoretical_loss": 3.575400294259259,
+ "tokens_seen": 1359740928
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047518555667001004,
+ "loss": 1.3826,
+ "theoretical_loss": 3.5753849365878523,
+ "tokens_seen": 1359806464
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 456448,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3537814617156982,
+ "objective/train/theoretical_loss": 3.5753695798638265,
+ "objective/train/tokens_used": 1380332000,
+ "theoretical_loss": 3.5753695798638265,
+ "tokens_seen": 1359872000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004751695085255768,
+ "loss": 1.3345,
+ "theoretical_loss": 3.5753695798638265,
+ "tokens_seen": 1359872000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047515346038114344,
+ "loss": 1.3273,
+ "theoretical_loss": 3.5753542240870777,
+ "tokens_seen": 1359937536
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004751374122367102,
+ "loss": 1.2333,
+ "theoretical_loss": 3.5753388692575023,
+ "tokens_seen": 1360003072
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047512136409227684,
+ "loss": 1.351,
+ "theoretical_loss": 3.575323515374996,
+ "tokens_seen": 1360068608
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004751053159478436,
+ "loss": 1.3505,
+ "theoretical_loss": 3.5753081624394545,
+ "tokens_seen": 1360134144
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047508926780341024,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5752928104507737,
+ "tokens_seen": 1360199680
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000475073219658977,
+ "loss": 1.2254,
+ "theoretical_loss": 3.5752774594088503,
+ "tokens_seen": 1360265216
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047505717151454364,
+ "loss": 1.2612,
+ "theoretical_loss": 3.57526210931358,
+ "tokens_seen": 1360330752
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004750411233701104,
+ "loss": 1.311,
+ "theoretical_loss": 3.5752467601648577,
+ "tokens_seen": 1360396288
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047502507522567705,
+ "loss": 1.3675,
+ "theoretical_loss": 3.575231411962581,
+ "tokens_seen": 1360461824
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047500902708124375,
+ "loss": 1.3748,
+ "theoretical_loss": 3.575216064706645,
+ "tokens_seen": 1360527360
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047499297893681045,
+ "loss": 1.2639,
+ "theoretical_loss": 3.575200718396946,
+ "tokens_seen": 1360592896
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047497693079237715,
+ "loss": 1.4052,
+ "theoretical_loss": 3.5751853730333805,
+ "tokens_seen": 1360658432
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047496088264794385,
+ "loss": 1.3142,
+ "theoretical_loss": 3.5751700286158443,
+ "tokens_seen": 1360723968
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047494483450351055,
+ "loss": 1.3593,
+ "theoretical_loss": 3.5751546851442333,
+ "tokens_seen": 1360789504
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047492878635907725,
+ "loss": 1.2908,
+ "theoretical_loss": 3.575139342618444,
+ "tokens_seen": 1360855040
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047491273821464395,
+ "loss": 1.3743,
+ "theoretical_loss": 3.5751240010383727,
+ "tokens_seen": 1360920576
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047489669007021065,
+ "loss": 1.3119,
+ "theoretical_loss": 3.575108660403915,
+ "tokens_seen": 1360986112
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047488064192577735,
+ "loss": 1.2732,
+ "theoretical_loss": 3.575093320714967,
+ "tokens_seen": 1361051648
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000474864593781344,
+ "loss": 1.328,
+ "theoretical_loss": 3.5750779819714253,
+ "tokens_seen": 1361117184
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047484854563691075,
+ "loss": 1.3256,
+ "theoretical_loss": 3.5750626441731868,
+ "tokens_seen": 1361182720
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004748324974924774,
+ "loss": 1.3487,
+ "theoretical_loss": 3.575047307320146,
+ "tokens_seen": 1361248256
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047481644934804415,
+ "loss": 1.3667,
+ "theoretical_loss": 3.5750319714122005,
+ "tokens_seen": 1361313792
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004748004012036108,
+ "loss": 1.3611,
+ "theoretical_loss": 3.5750166364492464,
+ "tokens_seen": 1361379328
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047478435305917756,
+ "loss": 1.4097,
+ "theoretical_loss": 3.5750013024311795,
+ "tokens_seen": 1361444864
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004747683049147442,
+ "loss": 1.3568,
+ "theoretical_loss": 3.574985969357897,
+ "tokens_seen": 1361510400
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047475225677031096,
+ "loss": 1.2948,
+ "theoretical_loss": 3.5749706372292938,
+ "tokens_seen": 1361575936
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004747362086258776,
+ "loss": 1.3441,
+ "theoretical_loss": 3.5749553060452675,
+ "tokens_seen": 1361641472
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047472016048144436,
+ "loss": 1.3073,
+ "theoretical_loss": 3.574939975805714,
+ "tokens_seen": 1361707008
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000474704112337011,
+ "loss": 1.2755,
+ "theoretical_loss": 3.5749246465105298,
+ "tokens_seen": 1361772544
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047468806419257776,
+ "loss": 1.4098,
+ "theoretical_loss": 3.5749093181596105,
+ "tokens_seen": 1361838080
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004746720160481444,
+ "loss": 1.3682,
+ "theoretical_loss": 3.5748939907528534,
+ "tokens_seen": 1361903616
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047465596790371116,
+ "loss": 1.4077,
+ "theoretical_loss": 3.574878664290155,
+ "tokens_seen": 1361969152
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004746399197592778,
+ "loss": 1.29,
+ "theoretical_loss": 3.574863338771411,
+ "tokens_seen": 1362034688
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047462387161484456,
+ "loss": 1.3673,
+ "theoretical_loss": 3.5748480141965184,
+ "tokens_seen": 1362100224
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004746078234704112,
+ "loss": 1.3854,
+ "theoretical_loss": 3.5748326905653736,
+ "tokens_seen": 1362165760
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047459177532597796,
+ "loss": 1.3443,
+ "theoretical_loss": 3.5748173678778725,
+ "tokens_seen": 1362231296
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004745757271815446,
+ "loss": 1.3209,
+ "theoretical_loss": 3.5748020461339123,
+ "tokens_seen": 1362296832
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047455967903711136,
+ "loss": 1.2935,
+ "theoretical_loss": 3.5747867253333894,
+ "tokens_seen": 1362362368
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000474543630892678,
+ "loss": 1.4221,
+ "theoretical_loss": 3.5747714054762003,
+ "tokens_seen": 1362427904
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047452758274824477,
+ "loss": 1.2886,
+ "theoretical_loss": 3.5747560865622408,
+ "tokens_seen": 1362493440
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004745115346038114,
+ "loss": 1.3608,
+ "theoretical_loss": 3.574740768591409,
+ "tokens_seen": 1362558976
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047449548645937817,
+ "loss": 1.3151,
+ "theoretical_loss": 3.5747254515635998,
+ "tokens_seen": 1362624512
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004744794383149448,
+ "loss": 1.3692,
+ "theoretical_loss": 3.5747101354787114,
+ "tokens_seen": 1362690048
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047446339017051157,
+ "loss": 1.315,
+ "theoretical_loss": 3.5746948203366387,
+ "tokens_seen": 1362755584
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004744473420260783,
+ "loss": 1.432,
+ "theoretical_loss": 3.57467950613728,
+ "tokens_seen": 1362821120
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047443129388164497,
+ "loss": 1.4033,
+ "theoretical_loss": 3.5746641928805305,
+ "tokens_seen": 1362886656
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004744152457372117,
+ "loss": 1.4039,
+ "theoretical_loss": 3.574648880566288,
+ "tokens_seen": 1362952192
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047439919759277837,
+ "loss": 1.2897,
+ "theoretical_loss": 3.5746335691944484,
+ "tokens_seen": 1363017728
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004743831494483451,
+ "loss": 1.3326,
+ "theoretical_loss": 3.5746182587649087,
+ "tokens_seen": 1363083264
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 457783,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.313234806060791,
+ "objective/train/theoretical_loss": 3.5746029492775655,
+ "objective/train/tokens_used": 1383608800,
+ "theoretical_loss": 3.5746029492775655,
+ "tokens_seen": 1363148800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047436710130391177,
+ "loss": 1.3109,
+ "theoretical_loss": 3.5746029492775655,
+ "tokens_seen": 1363148800
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004743510531594785,
+ "loss": 1.3396,
+ "theoretical_loss": 3.5745876407323163,
+ "tokens_seen": 1363214336
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004743350050150452,
+ "loss": 1.3384,
+ "theoretical_loss": 3.574572333129056,
+ "tokens_seen": 1363279872
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004743189568706119,
+ "loss": 1.3015,
+ "theoretical_loss": 3.5745570264676836,
+ "tokens_seen": 1363345408
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004743029087261786,
+ "loss": 1.3519,
+ "theoretical_loss": 3.574541720748094,
+ "tokens_seen": 1363410944
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004742868605817453,
+ "loss": 1.3502,
+ "theoretical_loss": 3.574526415970185,
+ "tokens_seen": 1363476480
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000474270812437312,
+ "loss": 1.3159,
+ "theoretical_loss": 3.5745111121338535,
+ "tokens_seen": 1363542016
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004742547642928787,
+ "loss": 1.2457,
+ "theoretical_loss": 3.5744958092389956,
+ "tokens_seen": 1363607552
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004742387161484453,
+ "loss": 1.3811,
+ "theoretical_loss": 3.5744805072855086,
+ "tokens_seen": 1363673088
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004742226680040121,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5744652062732896,
+ "tokens_seen": 1363738624
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004742066198595787,
+ "loss": 1.3209,
+ "theoretical_loss": 3.5744499062022346,
+ "tokens_seen": 1363804160
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004741905717151455,
+ "loss": 1.3802,
+ "theoretical_loss": 3.5744346070722415,
+ "tokens_seen": 1363869696
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004741745235707121,
+ "loss": 1.2311,
+ "theoretical_loss": 3.574419308883207,
+ "tokens_seen": 1363935232
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004741584754262789,
+ "loss": 1.3884,
+ "theoretical_loss": 3.5744040116350266,
+ "tokens_seen": 1364000768
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004741424272818455,
+ "loss": 1.2998,
+ "theoretical_loss": 3.5743887153275997,
+ "tokens_seen": 1364066304
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004741263791374123,
+ "loss": 1.2812,
+ "theoretical_loss": 3.5743734199608213,
+ "tokens_seen": 1364131840
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047411033099297893,
+ "loss": 1.3264,
+ "theoretical_loss": 3.574358125534589,
+ "tokens_seen": 1364197376
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004740942828485457,
+ "loss": 1.3705,
+ "theoretical_loss": 3.5743428320488,
+ "tokens_seen": 1364262912
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047407823470411233,
+ "loss": 1.2873,
+ "theoretical_loss": 3.574327539503351,
+ "tokens_seen": 1364328448
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004740621865596791,
+ "loss": 1.254,
+ "theoretical_loss": 3.5743122478981393,
+ "tokens_seen": 1364393984
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047404613841524573,
+ "loss": 1.302,
+ "theoretical_loss": 3.5742969572330616,
+ "tokens_seen": 1364459520
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004740300902708125,
+ "loss": 1.2448,
+ "theoretical_loss": 3.574281667508015,
+ "tokens_seen": 1364525056
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047401404212637913,
+ "loss": 1.3864,
+ "theoretical_loss": 3.574266378722897,
+ "tokens_seen": 1364590592
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004739979939819459,
+ "loss": 1.3481,
+ "theoretical_loss": 3.5742510908776044,
+ "tokens_seen": 1364656128
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047398194583751253,
+ "loss": 1.3396,
+ "theoretical_loss": 3.574235803972034,
+ "tokens_seen": 1364721664
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004739658976930793,
+ "loss": 1.3188,
+ "theoretical_loss": 3.574220518006083,
+ "tokens_seen": 1364787200
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047394984954864593,
+ "loss": 1.3528,
+ "theoretical_loss": 3.574205232979649,
+ "tokens_seen": 1364852736
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004739338014042127,
+ "loss": 1.322,
+ "theoretical_loss": 3.574189948892629,
+ "tokens_seen": 1364918272
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047391775325977934,
+ "loss": 1.3503,
+ "theoretical_loss": 3.57417466574492,
+ "tokens_seen": 1364983808
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004739017051153461,
+ "loss": 1.3435,
+ "theoretical_loss": 3.5741593835364185,
+ "tokens_seen": 1365049344
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047388565697091274,
+ "loss": 1.3456,
+ "theoretical_loss": 3.574144102267023,
+ "tokens_seen": 1365114880
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004738696088264795,
+ "loss": 1.2507,
+ "theoretical_loss": 3.5741288219366294,
+ "tokens_seen": 1365180416
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047385356068204614,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5741135425451365,
+ "tokens_seen": 1365245952
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004738375125376129,
+ "loss": 1.3467,
+ "theoretical_loss": 3.5740982640924397,
+ "tokens_seen": 1365311488
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047382146439317954,
+ "loss": 1.3542,
+ "theoretical_loss": 3.5740829865784383,
+ "tokens_seen": 1365377024
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004738054162487463,
+ "loss": 1.4454,
+ "theoretical_loss": 3.5740677100030274,
+ "tokens_seen": 1365442560
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047378936810431294,
+ "loss": 1.3942,
+ "theoretical_loss": 3.574052434366106,
+ "tokens_seen": 1365508096
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004737733199598797,
+ "loss": 1.3959,
+ "theoretical_loss": 3.5740371596675713,
+ "tokens_seen": 1365573632
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047375727181544634,
+ "loss": 1.3032,
+ "theoretical_loss": 3.574021885907319,
+ "tokens_seen": 1365639168
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004737412236710131,
+ "loss": 1.3207,
+ "theoretical_loss": 3.574006613085248,
+ "tokens_seen": 1365704704
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047372517552657974,
+ "loss": 1.2565,
+ "theoretical_loss": 3.5739913412012556,
+ "tokens_seen": 1365770240
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004737091273821465,
+ "loss": 1.2285,
+ "theoretical_loss": 3.573976070255238,
+ "tokens_seen": 1365835776
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047369307923771314,
+ "loss": 1.3508,
+ "theoretical_loss": 3.573960800247094,
+ "tokens_seen": 1365901312
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004736770310932799,
+ "loss": 1.3147,
+ "theoretical_loss": 3.5739455311767205,
+ "tokens_seen": 1365966848
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047366098294884655,
+ "loss": 1.3043,
+ "theoretical_loss": 3.573930263044015,
+ "tokens_seen": 1366032384
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004736449348044133,
+ "loss": 1.4096,
+ "theoretical_loss": 3.573914995848874,
+ "tokens_seen": 1366097920
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047362888665997995,
+ "loss": 1.2915,
+ "theoretical_loss": 3.573899729591196,
+ "tokens_seen": 1366163456
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004736128385155467,
+ "loss": 1.2662,
+ "theoretical_loss": 3.5738844642708782,
+ "tokens_seen": 1366228992
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047359679037111335,
+ "loss": 1.4209,
+ "theoretical_loss": 3.5738691998878176,
+ "tokens_seen": 1366294528
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047358074222668005,
+ "loss": 1.2719,
+ "theoretical_loss": 3.5738539364419126,
+ "tokens_seen": 1366360064
+ },
+ {
+ "epoch": 0.41,
+ "objective/train/docs_used": 458648,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3905396461486816,
+ "objective/train/theoretical_loss": 3.5738386739330603,
+ "objective/train/tokens_used": 1386885600,
+ "theoretical_loss": 3.5738386739330603,
+ "tokens_seen": 1366425600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047356469408224675,
+ "loss": 1.3947,
+ "theoretical_loss": 3.5738386739330603,
+ "tokens_seen": 1366425600
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047354864593781345,
+ "loss": 1.3053,
+ "theoretical_loss": 3.573823412361158,
+ "tokens_seen": 1366491136
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047353259779338015,
+ "loss": 1.318,
+ "theoretical_loss": 3.5738081517261033,
+ "tokens_seen": 1366556672
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047351654964894685,
+ "loss": 1.3456,
+ "theoretical_loss": 3.573792892027795,
+ "tokens_seen": 1366622208
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047350050150451355,
+ "loss": 1.3967,
+ "theoretical_loss": 3.573777633266128,
+ "tokens_seen": 1366687744
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047348445336008025,
+ "loss": 1.3089,
+ "theoretical_loss": 3.5737623754410026,
+ "tokens_seen": 1366753280
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004734684052156469,
+ "loss": 1.3499,
+ "theoretical_loss": 3.573747118552315,
+ "tokens_seen": 1366818816
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047345235707121365,
+ "loss": 1.341,
+ "theoretical_loss": 3.573731862599963,
+ "tokens_seen": 1366884352
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004734363089267803,
+ "loss": 1.4185,
+ "theoretical_loss": 3.5737166075838447,
+ "tokens_seen": 1366949888
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047342026078234706,
+ "loss": 1.3863,
+ "theoretical_loss": 3.573701353503857,
+ "tokens_seen": 1367015424
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004734042126379137,
+ "loss": 1.4295,
+ "theoretical_loss": 3.5736861003598985,
+ "tokens_seen": 1367080960
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047338816449348046,
+ "loss": 1.3948,
+ "theoretical_loss": 3.573670848151866,
+ "tokens_seen": 1367146496
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004733721163490471,
+ "loss": 1.25,
+ "theoretical_loss": 3.573655596879658,
+ "tokens_seen": 1367212032
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047335606820461386,
+ "loss": 1.3141,
+ "theoretical_loss": 3.573640346543172,
+ "tokens_seen": 1367277568
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004733400200601805,
+ "loss": 1.3778,
+ "theoretical_loss": 3.5736250971423056,
+ "tokens_seen": 1367343104
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047332397191574726,
+ "loss": 1.2543,
+ "theoretical_loss": 3.5736098486769556,
+ "tokens_seen": 1367408640
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004733079237713139,
+ "loss": 1.3102,
+ "theoretical_loss": 3.573594601147022,
+ "tokens_seen": 1367474176
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047329187562688066,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5735793545524004,
+ "tokens_seen": 1367539712
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004732758274824473,
+ "loss": 1.2981,
+ "theoretical_loss": 3.57356410889299,
+ "tokens_seen": 1367605248
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047325977933801406,
+ "loss": 1.282,
+ "theoretical_loss": 3.5735488641686883,
+ "tokens_seen": 1367670784
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004732437311935808,
+ "loss": 1.4454,
+ "theoretical_loss": 3.5735336203793926,
+ "tokens_seen": 1367736320
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047322768304914746,
+ "loss": 1.4393,
+ "theoretical_loss": 3.5735183775250015,
+ "tokens_seen": 1367801856
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004732116349047142,
+ "loss": 1.3012,
+ "theoretical_loss": 3.5735031356054123,
+ "tokens_seen": 1367867392
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047319558676028086,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5734878946205235,
+ "tokens_seen": 1367932928
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004731795386158476,
+ "loss": 1.348,
+ "theoretical_loss": 3.5734726545702324,
+ "tokens_seen": 1367998464
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047316349047141427,
+ "loss": 1.3849,
+ "theoretical_loss": 3.573457415454437,
+ "tokens_seen": 1368064000
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000473147442326981,
+ "loss": 1.2892,
+ "theoretical_loss": 3.5734421772730354,
+ "tokens_seen": 1368129536
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047313139418254767,
+ "loss": 1.3821,
+ "theoretical_loss": 3.5734269400259255,
+ "tokens_seen": 1368195072
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004731153460381144,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5734117037130053,
+ "tokens_seen": 1368260608
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047309929789368107,
+ "loss": 1.4114,
+ "theoretical_loss": 3.573396468334173,
+ "tokens_seen": 1368326144
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004730832497492478,
+ "loss": 1.2412,
+ "theoretical_loss": 3.5733812338893265,
+ "tokens_seen": 1368391680
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047306720160481447,
+ "loss": 1.3724,
+ "theoretical_loss": 3.5733660003783636,
+ "tokens_seen": 1368457216
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004730511534603812,
+ "loss": 1.3053,
+ "theoretical_loss": 3.573350767801182,
+ "tokens_seen": 1368522752
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047303510531594787,
+ "loss": 1.3152,
+ "theoretical_loss": 3.5733355361576806,
+ "tokens_seen": 1368588288
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004730190571715146,
+ "loss": 1.3617,
+ "theoretical_loss": 3.573320305447757,
+ "tokens_seen": 1368653824
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.00047300300902708127,
+ "loss": 1.3034,
+ "theoretical_loss": 3.5733050756713096,
+ "tokens_seen": 1368719360
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000472986960882648,
+ "loss": 1.3214,
+ "theoretical_loss": 3.573289846828236,
+ "tokens_seen": 1368784896
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004729709127382147,
+ "loss": 1.224,
+ "theoretical_loss": 3.5732746189184343,
+ "tokens_seen": 1368850432
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004729548645937814,
+ "loss": 1.24,
+ "theoretical_loss": 3.5732593919418028,
+ "tokens_seen": 1368915968
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004729388164493481,
+ "loss": 1.2527,
+ "theoretical_loss": 3.57324416589824,
+ "tokens_seen": 1368981504
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004729227683049148,
+ "loss": 1.3667,
+ "theoretical_loss": 3.5732289407876436,
+ "tokens_seen": 1369047040
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004729067201604815,
+ "loss": 1.4329,
+ "theoretical_loss": 3.5732137166099123,
+ "tokens_seen": 1369112576
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004728906720160482,
+ "loss": 1.3794,
+ "theoretical_loss": 3.5731984933649437,
+ "tokens_seen": 1369178112
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004728746238716149,
+ "loss": 1.3097,
+ "theoretical_loss": 3.573183271052636,
+ "tokens_seen": 1369243648
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004728585757271816,
+ "loss": 1.3117,
+ "theoretical_loss": 3.573168049672888,
+ "tokens_seen": 1369309184
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.0004728425275827483,
+ "loss": 1.2598,
+ "theoretical_loss": 3.573152829225598,
+ "tokens_seen": 1369374720
+ },
+ {
+ "epoch": 0.41,
+ "learning_rate": 0.000472826479438315,
+ "loss": 1.2374,
+ "theoretical_loss": 3.573137609710663,
+ "tokens_seen": 1369440256
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004728104312938816,
+ "loss": 1.3218,
+ "theoretical_loss": 3.573122391127982,
+ "tokens_seen": 1369505792
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004727943831494484,
+ "loss": 1.3227,
+ "theoretical_loss": 3.5731071734774544,
+ "tokens_seen": 1369571328
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000472778335005015,
+ "loss": 1.2823,
+ "theoretical_loss": 3.5730919567589767,
+ "tokens_seen": 1369636864
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 459889,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1413302421569824,
+ "objective/train/theoretical_loss": 3.573076740972449,
+ "objective/train/tokens_used": 1390162400,
+ "theoretical_loss": 3.573076740972449,
+ "tokens_seen": 1369702400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004727622868605818,
+ "loss": 1.2678,
+ "theoretical_loss": 3.573076740972449,
+ "tokens_seen": 1369702400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047274623871614843,
+ "loss": 1.2219,
+ "theoretical_loss": 3.573061526117768,
+ "tokens_seen": 1369767936
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004727301905717152,
+ "loss": 1.4363,
+ "theoretical_loss": 3.573046312194833,
+ "tokens_seen": 1369833472
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047271414242728183,
+ "loss": 1.2873,
+ "theoretical_loss": 3.5730310992035417,
+ "tokens_seen": 1369899008
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004726980942828486,
+ "loss": 1.2869,
+ "theoretical_loss": 3.573015887143793,
+ "tokens_seen": 1369964544
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047268204613841523,
+ "loss": 1.2866,
+ "theoretical_loss": 3.5730006760154853,
+ "tokens_seen": 1370030080
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000472665997993982,
+ "loss": 1.4166,
+ "theoretical_loss": 3.572985465818517,
+ "tokens_seen": 1370095616
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047264994984954863,
+ "loss": 1.3116,
+ "theoretical_loss": 3.5729702565527868,
+ "tokens_seen": 1370161152
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004726339017051154,
+ "loss": 1.3564,
+ "theoretical_loss": 3.572955048218192,
+ "tokens_seen": 1370226688
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047261785356068203,
+ "loss": 1.3333,
+ "theoretical_loss": 3.5729398408146325,
+ "tokens_seen": 1370292224
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004726018054162488,
+ "loss": 1.2981,
+ "theoretical_loss": 3.5729246343420056,
+ "tokens_seen": 1370357760
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047258575727181543,
+ "loss": 1.4071,
+ "theoretical_loss": 3.5729094288002106,
+ "tokens_seen": 1370423296
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004725697091273822,
+ "loss": 1.2527,
+ "theoretical_loss": 3.5728942241891457,
+ "tokens_seen": 1370488832
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047255366098294884,
+ "loss": 1.3149,
+ "theoretical_loss": 3.5728790205087093,
+ "tokens_seen": 1370554368
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004725376128385156,
+ "loss": 1.3044,
+ "theoretical_loss": 3.5728638177588,
+ "tokens_seen": 1370619904
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047252156469408224,
+ "loss": 1.3668,
+ "theoretical_loss": 3.572848615939317,
+ "tokens_seen": 1370685440
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000472505516549649,
+ "loss": 1.2671,
+ "theoretical_loss": 3.572833415050158,
+ "tokens_seen": 1370750976
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047248946840521564,
+ "loss": 1.4222,
+ "theoretical_loss": 3.572818215091222,
+ "tokens_seen": 1370816512
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004724734202607824,
+ "loss": 1.3043,
+ "theoretical_loss": 3.572803016062408,
+ "tokens_seen": 1370882048
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047245737211634904,
+ "loss": 1.2599,
+ "theoretical_loss": 3.572787817963613,
+ "tokens_seen": 1370947584
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004724413239719158,
+ "loss": 1.4172,
+ "theoretical_loss": 3.572772620794738,
+ "tokens_seen": 1371013120
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047242527582748244,
+ "loss": 1.4319,
+ "theoretical_loss": 3.5727574245556797,
+ "tokens_seen": 1371078656
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004724092276830492,
+ "loss": 1.2669,
+ "theoretical_loss": 3.572742229246338,
+ "tokens_seen": 1371144192
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047239317953861584,
+ "loss": 1.4297,
+ "theoretical_loss": 3.572727034866611,
+ "tokens_seen": 1371209728
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004723771313941826,
+ "loss": 1.4387,
+ "theoretical_loss": 3.572711841416397,
+ "tokens_seen": 1371275264
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047236108324974924,
+ "loss": 1.255,
+ "theoretical_loss": 3.5726966488955956,
+ "tokens_seen": 1371340800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000472345035105316,
+ "loss": 1.3279,
+ "theoretical_loss": 3.572681457304105,
+ "tokens_seen": 1371406336
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047232898696088264,
+ "loss": 1.4369,
+ "theoretical_loss": 3.5726662666418245,
+ "tokens_seen": 1371471872
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004723129388164494,
+ "loss": 1.3403,
+ "theoretical_loss": 3.5726510769086524,
+ "tokens_seen": 1371537408
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047229689067201605,
+ "loss": 1.3083,
+ "theoretical_loss": 3.5726358881044873,
+ "tokens_seen": 1371602944
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004722808425275828,
+ "loss": 1.3177,
+ "theoretical_loss": 3.572620700229228,
+ "tokens_seen": 1371668480
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047226479438314945,
+ "loss": 1.3407,
+ "theoretical_loss": 3.5726055132827748,
+ "tokens_seen": 1371734016
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004722487462387162,
+ "loss": 1.4713,
+ "theoretical_loss": 3.572590327265024,
+ "tokens_seen": 1371799552
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047223269809428285,
+ "loss": 1.4398,
+ "theoretical_loss": 3.572575142175876,
+ "tokens_seen": 1371865088
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004722166499498496,
+ "loss": 1.3684,
+ "theoretical_loss": 3.57255995801523,
+ "tokens_seen": 1371930624
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047220060180541625,
+ "loss": 1.3469,
+ "theoretical_loss": 3.5725447747829833,
+ "tokens_seen": 1371996160
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047218455366098295,
+ "loss": 1.3924,
+ "theoretical_loss": 3.572529592479037,
+ "tokens_seen": 1372061696
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047216850551654965,
+ "loss": 1.338,
+ "theoretical_loss": 3.572514411103288,
+ "tokens_seen": 1372127232
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047215245737211635,
+ "loss": 1.2579,
+ "theoretical_loss": 3.572499230655636,
+ "tokens_seen": 1372192768
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047213640922768305,
+ "loss": 1.3848,
+ "theoretical_loss": 3.57248405113598,
+ "tokens_seen": 1372258304
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047212036108324975,
+ "loss": 1.4693,
+ "theoretical_loss": 3.572468872544219,
+ "tokens_seen": 1372323840
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047210431293881645,
+ "loss": 1.367,
+ "theoretical_loss": 3.572453694880252,
+ "tokens_seen": 1372389376
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047208826479438315,
+ "loss": 1.3872,
+ "theoretical_loss": 3.5724385181439775,
+ "tokens_seen": 1372454912
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047207221664994985,
+ "loss": 1.366,
+ "theoretical_loss": 3.572423342335295,
+ "tokens_seen": 1372520448
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047205616850551656,
+ "loss": 1.4044,
+ "theoretical_loss": 3.5724081674541033,
+ "tokens_seen": 1372585984
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004720401203610833,
+ "loss": 1.3479,
+ "theoretical_loss": 3.572392993500302,
+ "tokens_seen": 1372651520
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047202407221664996,
+ "loss": 1.3709,
+ "theoretical_loss": 3.572377820473789,
+ "tokens_seen": 1372717056
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004720080240722167,
+ "loss": 1.3712,
+ "theoretical_loss": 3.5723626483744644,
+ "tokens_seen": 1372782592
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047199197592778336,
+ "loss": 1.3318,
+ "theoretical_loss": 3.572347477202227,
+ "tokens_seen": 1372848128
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004719759277833501,
+ "loss": 1.3188,
+ "theoretical_loss": 3.5723323069569757,
+ "tokens_seen": 1372913664
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 460768,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2740132808685303,
+ "objective/train/theoretical_loss": 3.57231713763861,
+ "objective/train/tokens_used": 1393439200,
+ "theoretical_loss": 3.57231713763861,
+ "tokens_seen": 1372979200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047195987963891676,
+ "loss": 1.2705,
+ "theoretical_loss": 3.57231713763861,
+ "tokens_seen": 1372979200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004719438314944835,
+ "loss": 1.3173,
+ "theoretical_loss": 3.572301969247028,
+ "tokens_seen": 1373044736
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047192778335005016,
+ "loss": 1.3587,
+ "theoretical_loss": 3.57228680178213,
+ "tokens_seen": 1373110272
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004719117352056169,
+ "loss": 1.297,
+ "theoretical_loss": 3.572271635243815,
+ "tokens_seen": 1373175808
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047189568706118356,
+ "loss": 1.31,
+ "theoretical_loss": 3.5722564696319816,
+ "tokens_seen": 1373241344
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004718796389167503,
+ "loss": 1.3391,
+ "theoretical_loss": 3.57224130494653,
+ "tokens_seen": 1373306880
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047186359077231696,
+ "loss": 1.288,
+ "theoretical_loss": 3.572226141187358,
+ "tokens_seen": 1373372416
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004718475426278837,
+ "loss": 1.294,
+ "theoretical_loss": 3.5722109783543656,
+ "tokens_seen": 1373437952
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047183149448345036,
+ "loss": 1.3175,
+ "theoretical_loss": 3.5721958164474525,
+ "tokens_seen": 1373503488
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004718154463390171,
+ "loss": 1.3887,
+ "theoretical_loss": 3.572180655466517,
+ "tokens_seen": 1373569024
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047179939819458377,
+ "loss": 1.4171,
+ "theoretical_loss": 3.572165495411459,
+ "tokens_seen": 1373634560
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004717833500501505,
+ "loss": 1.3422,
+ "theoretical_loss": 3.572150336282178,
+ "tokens_seen": 1373700096
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047176730190571717,
+ "loss": 1.3605,
+ "theoretical_loss": 3.572135178078572,
+ "tokens_seen": 1373765632
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004717512537612839,
+ "loss": 1.2494,
+ "theoretical_loss": 3.572120020800542,
+ "tokens_seen": 1373831168
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047173520561685057,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5721048644479865,
+ "tokens_seen": 1373896704
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004717191574724173,
+ "loss": 1.3181,
+ "theoretical_loss": 3.572089709020805,
+ "tokens_seen": 1373962240
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047170310932798397,
+ "loss": 1.3014,
+ "theoretical_loss": 3.5720745545188963,
+ "tokens_seen": 1374027776
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004716870611835507,
+ "loss": 1.4695,
+ "theoretical_loss": 3.5720594009421607,
+ "tokens_seen": 1374093312
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047167101303911737,
+ "loss": 1.2828,
+ "theoretical_loss": 3.572044248290497,
+ "tokens_seen": 1374158848
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004716549648946841,
+ "loss": 1.4203,
+ "theoretical_loss": 3.5720290965638046,
+ "tokens_seen": 1374224384
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047163891675025077,
+ "loss": 1.2572,
+ "theoretical_loss": 3.5720139457619835,
+ "tokens_seen": 1374289920
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004716228686058175,
+ "loss": 1.3911,
+ "theoretical_loss": 3.571998795884933,
+ "tokens_seen": 1374355456
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004716068204613842,
+ "loss": 1.1725,
+ "theoretical_loss": 3.5719836469325514,
+ "tokens_seen": 1374420992
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047159077231695093,
+ "loss": 1.3756,
+ "theoretical_loss": 3.5719684989047398,
+ "tokens_seen": 1374486528
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004715747241725176,
+ "loss": 1.3474,
+ "theoretical_loss": 3.5719533518013966,
+ "tokens_seen": 1374552064
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004715586760280843,
+ "loss": 1.3653,
+ "theoretical_loss": 3.5719382056224225,
+ "tokens_seen": 1374617600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000471542627883651,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5719230603677152,
+ "tokens_seen": 1374683136
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004715265797392177,
+ "loss": 1.4509,
+ "theoretical_loss": 3.5719079160371763,
+ "tokens_seen": 1374748672
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004715105315947844,
+ "loss": 1.3045,
+ "theoretical_loss": 3.5718927726307035,
+ "tokens_seen": 1374814208
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714944834503511,
+ "loss": 1.2463,
+ "theoretical_loss": 3.5718776301481974,
+ "tokens_seen": 1374879744
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714784353059178,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5718624885895576,
+ "tokens_seen": 1374945280
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714623871614845,
+ "loss": 1.315,
+ "theoretical_loss": 3.571847347954683,
+ "tokens_seen": 1375010816
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714463390170512,
+ "loss": 1.3062,
+ "theoretical_loss": 3.5718322082434737,
+ "tokens_seen": 1375076352
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714302908726179,
+ "loss": 1.2935,
+ "theoretical_loss": 3.5718170694558298,
+ "tokens_seen": 1375141888
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004714142427281845,
+ "loss": 1.383,
+ "theoretical_loss": 3.5718019315916503,
+ "tokens_seen": 1375207424
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004713981945837513,
+ "loss": 1.3552,
+ "theoretical_loss": 3.5717867946508353,
+ "tokens_seen": 1375272960
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047138214643931793,
+ "loss": 1.2173,
+ "theoretical_loss": 3.5717716586332835,
+ "tokens_seen": 1375338496
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004713660982948847,
+ "loss": 1.3346,
+ "theoretical_loss": 3.571756523538896,
+ "tokens_seen": 1375404032
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047135005015045133,
+ "loss": 1.3706,
+ "theoretical_loss": 3.5717413893675714,
+ "tokens_seen": 1375469568
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004713340020060181,
+ "loss": 1.3294,
+ "theoretical_loss": 3.5717262561192102,
+ "tokens_seen": 1375535104
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047131795386158473,
+ "loss": 1.3575,
+ "theoretical_loss": 3.5717111237937114,
+ "tokens_seen": 1375600640
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004713019057171515,
+ "loss": 1.3182,
+ "theoretical_loss": 3.5716959923909757,
+ "tokens_seen": 1375666176
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047128585757271813,
+ "loss": 1.3512,
+ "theoretical_loss": 3.5716808619109015,
+ "tokens_seen": 1375731712
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004712698094282849,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5716657323533902,
+ "tokens_seen": 1375797248
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047125376128385153,
+ "loss": 1.3223,
+ "theoretical_loss": 3.57165060371834,
+ "tokens_seen": 1375862784
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004712377131394183,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5716354760056523,
+ "tokens_seen": 1375928320
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047122166499498493,
+ "loss": 1.3762,
+ "theoretical_loss": 3.5716203492152263,
+ "tokens_seen": 1375993856
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004712056168505517,
+ "loss": 1.342,
+ "theoretical_loss": 3.5716052233469613,
+ "tokens_seen": 1376059392
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047118956870611834,
+ "loss": 1.3211,
+ "theoretical_loss": 3.571590098400758,
+ "tokens_seen": 1376124928
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004711735205616851,
+ "loss": 1.3122,
+ "theoretical_loss": 3.5715749743765155,
+ "tokens_seen": 1376190464
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 462043,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3740935325622559,
+ "objective/train/theoretical_loss": 3.5715598512741344,
+ "objective/train/tokens_used": 1396716000,
+ "theoretical_loss": 3.5715598512741344,
+ "tokens_seen": 1376256000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047115747241725174,
+ "loss": 1.3715,
+ "theoretical_loss": 3.5715598512741344,
+ "tokens_seen": 1376256000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004711414242728185,
+ "loss": 1.4589,
+ "theoretical_loss": 3.571544729093514,
+ "tokens_seen": 1376321536
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047112537612838514,
+ "loss": 1.3145,
+ "theoretical_loss": 3.5715296078345546,
+ "tokens_seen": 1376387072
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004711093279839519,
+ "loss": 1.3687,
+ "theoretical_loss": 3.571514487497156,
+ "tokens_seen": 1376452608
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047109327983951854,
+ "loss": 1.2702,
+ "theoretical_loss": 3.5714993680812186,
+ "tokens_seen": 1376518144
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004710772316950853,
+ "loss": 1.3214,
+ "theoretical_loss": 3.571484249586642,
+ "tokens_seen": 1376583680
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047106118355065194,
+ "loss": 1.3622,
+ "theoretical_loss": 3.571469132013326,
+ "tokens_seen": 1376649216
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004710451354062187,
+ "loss": 1.2574,
+ "theoretical_loss": 3.5714540153611702,
+ "tokens_seen": 1376714752
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047102908726178534,
+ "loss": 1.2868,
+ "theoretical_loss": 3.5714388996300763,
+ "tokens_seen": 1376780288
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004710130391173521,
+ "loss": 1.3985,
+ "theoretical_loss": 3.5714237848199426,
+ "tokens_seen": 1376845824
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047099699097291874,
+ "loss": 1.2811,
+ "theoretical_loss": 3.5714086709306705,
+ "tokens_seen": 1376911360
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004709809428284855,
+ "loss": 1.3024,
+ "theoretical_loss": 3.571393557962159,
+ "tokens_seen": 1376976896
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047096489468405214,
+ "loss": 1.3559,
+ "theoretical_loss": 3.5713784459143088,
+ "tokens_seen": 1377042432
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004709488465396189,
+ "loss": 1.2328,
+ "theoretical_loss": 3.5713633347870193,
+ "tokens_seen": 1377107968
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047093279839518555,
+ "loss": 1.3789,
+ "theoretical_loss": 3.5713482245801917,
+ "tokens_seen": 1377173504
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004709167502507523,
+ "loss": 1.3142,
+ "theoretical_loss": 3.5713331152937253,
+ "tokens_seen": 1377239040
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047090070210631895,
+ "loss": 1.2707,
+ "theoretical_loss": 3.5713180069275205,
+ "tokens_seen": 1377304576
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708846539618857,
+ "loss": 1.3661,
+ "theoretical_loss": 3.571302899481478,
+ "tokens_seen": 1377370112
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708686058174524,
+ "loss": 1.2541,
+ "theoretical_loss": 3.5712877929554967,
+ "tokens_seen": 1377435648
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708525576730191,
+ "loss": 1.2824,
+ "theoretical_loss": 3.571272687349478,
+ "tokens_seen": 1377501184
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708365095285858,
+ "loss": 1.43,
+ "theoretical_loss": 3.5712575826633213,
+ "tokens_seen": 1377566720
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708204613841525,
+ "loss": 1.2732,
+ "theoretical_loss": 3.5712424788969273,
+ "tokens_seen": 1377632256
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004708044132397192,
+ "loss": 1.303,
+ "theoretical_loss": 3.5712273760501967,
+ "tokens_seen": 1377697792
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047078836509528585,
+ "loss": 1.3798,
+ "theoretical_loss": 3.5712122741230283,
+ "tokens_seen": 1377763328
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004707723169508526,
+ "loss": 1.344,
+ "theoretical_loss": 3.5711971731153236,
+ "tokens_seen": 1377828864
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047075626880641925,
+ "loss": 1.2814,
+ "theoretical_loss": 3.571182073026983,
+ "tokens_seen": 1377894400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000470740220661986,
+ "loss": 1.321,
+ "theoretical_loss": 3.571166973857906,
+ "tokens_seen": 1377959936
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047072417251755265,
+ "loss": 1.2907,
+ "theoretical_loss": 3.571151875607993,
+ "tokens_seen": 1378025472
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004707081243731194,
+ "loss": 1.3214,
+ "theoretical_loss": 3.571136778277145,
+ "tokens_seen": 1378091008
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047069207622868605,
+ "loss": 1.2896,
+ "theoretical_loss": 3.571121681865262,
+ "tokens_seen": 1378156544
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004706760280842528,
+ "loss": 1.3386,
+ "theoretical_loss": 3.5711065863722444,
+ "tokens_seen": 1378222080
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047065997993981946,
+ "loss": 1.3329,
+ "theoretical_loss": 3.5710914917979917,
+ "tokens_seen": 1378287616
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004706439317953862,
+ "loss": 1.3,
+ "theoretical_loss": 3.5710763981424063,
+ "tokens_seen": 1378353152
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047062788365095286,
+ "loss": 1.3914,
+ "theoretical_loss": 3.5710613054053866,
+ "tokens_seen": 1378418688
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004706118355065196,
+ "loss": 1.226,
+ "theoretical_loss": 3.5710462135868344,
+ "tokens_seen": 1378484224
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047059578736208626,
+ "loss": 1.3071,
+ "theoretical_loss": 3.571031122686649,
+ "tokens_seen": 1378549760
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000470579739217653,
+ "loss": 1.3541,
+ "theoretical_loss": 3.571016032704732,
+ "tokens_seen": 1378615296
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047056369107321966,
+ "loss": 1.2992,
+ "theoretical_loss": 3.571000943640983,
+ "tokens_seen": 1378680832
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004705476429287864,
+ "loss": 1.4267,
+ "theoretical_loss": 3.5709858554953033,
+ "tokens_seen": 1378746368
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047053159478435306,
+ "loss": 1.2772,
+ "theoretical_loss": 3.5709707682675926,
+ "tokens_seen": 1378811904
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004705155466399198,
+ "loss": 1.4052,
+ "theoretical_loss": 3.5709556819577513,
+ "tokens_seen": 1378877440
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047049949849548646,
+ "loss": 1.354,
+ "theoretical_loss": 3.570940596565681,
+ "tokens_seen": 1378942976
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004704834503510532,
+ "loss": 1.3964,
+ "theoretical_loss": 3.5709255120912817,
+ "tokens_seen": 1379008512
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047046740220661986,
+ "loss": 1.1837,
+ "theoretical_loss": 3.5709104285344537,
+ "tokens_seen": 1379074048
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004704513540621866,
+ "loss": 1.3102,
+ "theoretical_loss": 3.5708953458950976,
+ "tokens_seen": 1379139584
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047043530591775327,
+ "loss": 1.3306,
+ "theoretical_loss": 3.5708802641731148,
+ "tokens_seen": 1379205120
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047041925777332,
+ "loss": 1.3896,
+ "theoretical_loss": 3.5708651833684053,
+ "tokens_seen": 1379270656
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047040320962888667,
+ "loss": 1.3955,
+ "theoretical_loss": 3.5708501034808693,
+ "tokens_seen": 1379336192
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004703871614844534,
+ "loss": 1.339,
+ "theoretical_loss": 3.5708350245104077,
+ "tokens_seen": 1379401728
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047037111334002007,
+ "loss": 1.3313,
+ "theoretical_loss": 3.570819946456922,
+ "tokens_seen": 1379467264
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 463003,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3521519899368286,
+ "objective/train/theoretical_loss": 3.5708048693203116,
+ "objective/train/tokens_used": 1399992800,
+ "theoretical_loss": 3.5708048693203116,
+ "tokens_seen": 1379532800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004703550651955868,
+ "loss": 1.3013,
+ "theoretical_loss": 3.5708048693203116,
+ "tokens_seen": 1379532800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047033901705115347,
+ "loss": 1.3763,
+ "theoretical_loss": 3.570789793100478,
+ "tokens_seen": 1379598336
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004703229689067202,
+ "loss": 1.2739,
+ "theoretical_loss": 3.570774717797322,
+ "tokens_seen": 1379663872
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047030692076228687,
+ "loss": 1.3336,
+ "theoretical_loss": 3.5707596434107445,
+ "tokens_seen": 1379729408
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004702908726178536,
+ "loss": 1.3077,
+ "theoretical_loss": 3.5707445699406453,
+ "tokens_seen": 1379794944
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047027482447342027,
+ "loss": 1.2372,
+ "theoretical_loss": 3.5707294973869255,
+ "tokens_seen": 1379860480
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000470258776328987,
+ "loss": 1.3269,
+ "theoretical_loss": 3.570714425749486,
+ "tokens_seen": 1379926016
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047024272818455367,
+ "loss": 1.3211,
+ "theoretical_loss": 3.570699355028228,
+ "tokens_seen": 1379991552
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047022668004012043,
+ "loss": 1.3326,
+ "theoretical_loss": 3.570684285223052,
+ "tokens_seen": 1380057088
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004702106318956871,
+ "loss": 1.3618,
+ "theoretical_loss": 3.570669216333858,
+ "tokens_seen": 1380122624
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047019458375125383,
+ "loss": 1.2397,
+ "theoretical_loss": 3.570654148360548,
+ "tokens_seen": 1380188160
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004701785356068205,
+ "loss": 1.2833,
+ "theoretical_loss": 3.570639081303023,
+ "tokens_seen": 1380253696
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004701624874623872,
+ "loss": 1.299,
+ "theoretical_loss": 3.570624015161182,
+ "tokens_seen": 1380319232
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004701464393179539,
+ "loss": 1.2476,
+ "theoretical_loss": 3.570608949934928,
+ "tokens_seen": 1380384768
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004701303911735206,
+ "loss": 1.3759,
+ "theoretical_loss": 3.5705938856241612,
+ "tokens_seen": 1380450304
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004701143430290873,
+ "loss": 1.2601,
+ "theoretical_loss": 3.570578822228782,
+ "tokens_seen": 1380515840
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000470098294884654,
+ "loss": 1.3817,
+ "theoretical_loss": 3.570563759748692,
+ "tokens_seen": 1380581376
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004700822467402207,
+ "loss": 1.3232,
+ "theoretical_loss": 3.5705486981837913,
+ "tokens_seen": 1380646912
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004700661985957874,
+ "loss": 1.2852,
+ "theoretical_loss": 3.5705336375339813,
+ "tokens_seen": 1380712448
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004700501504513541,
+ "loss": 1.4252,
+ "theoretical_loss": 3.570518577799164,
+ "tokens_seen": 1380777984
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004700341023069208,
+ "loss": 1.2892,
+ "theoretical_loss": 3.5705035189792387,
+ "tokens_seen": 1380843520
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00047001805416248743,
+ "loss": 1.3714,
+ "theoretical_loss": 3.570488461074107,
+ "tokens_seen": 1380909056
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004700020060180542,
+ "loss": 1.38,
+ "theoretical_loss": 3.5704734040836703,
+ "tokens_seen": 1380974592
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046998595787362083,
+ "loss": 1.2231,
+ "theoretical_loss": 3.5704583480078296,
+ "tokens_seen": 1381040128
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004699699097291876,
+ "loss": 1.3391,
+ "theoretical_loss": 3.5704432928464858,
+ "tokens_seen": 1381105664
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046995386158475423,
+ "loss": 1.4116,
+ "theoretical_loss": 3.57042823859954,
+ "tokens_seen": 1381171200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000469937813440321,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5704131852668923,
+ "tokens_seen": 1381236736
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046992176529588763,
+ "loss": 1.2999,
+ "theoretical_loss": 3.570398132848445,
+ "tokens_seen": 1381302272
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004699057171514544,
+ "loss": 1.3806,
+ "theoretical_loss": 3.570383081344099,
+ "tokens_seen": 1381367808
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046988966900702103,
+ "loss": 1.32,
+ "theoretical_loss": 3.5703680307537553,
+ "tokens_seen": 1381433344
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004698736208625878,
+ "loss": 1.3567,
+ "theoretical_loss": 3.570352981077315,
+ "tokens_seen": 1381498880
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046985757271815443,
+ "loss": 1.3425,
+ "theoretical_loss": 3.57033793231468,
+ "tokens_seen": 1381564416
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004698415245737212,
+ "loss": 1.3485,
+ "theoretical_loss": 3.57032288446575,
+ "tokens_seen": 1381629952
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046982547642928784,
+ "loss": 1.2665,
+ "theoretical_loss": 3.5703078375304274,
+ "tokens_seen": 1381695488
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004698094282848546,
+ "loss": 1.3152,
+ "theoretical_loss": 3.5702927915086127,
+ "tokens_seen": 1381761024
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046979338014042124,
+ "loss": 1.427,
+ "theoretical_loss": 3.5702777464002073,
+ "tokens_seen": 1381826560
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000469777331995988,
+ "loss": 1.407,
+ "theoretical_loss": 3.5702627022051128,
+ "tokens_seen": 1381892096
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046976128385155464,
+ "loss": 1.3846,
+ "theoretical_loss": 3.57024765892323,
+ "tokens_seen": 1381957632
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004697452357071214,
+ "loss": 1.3046,
+ "theoretical_loss": 3.57023261655446,
+ "tokens_seen": 1382023168
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046972918756268804,
+ "loss": 1.3372,
+ "theoretical_loss": 3.570217575098705,
+ "tokens_seen": 1382088704
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004697131394182548,
+ "loss": 1.3188,
+ "theoretical_loss": 3.5702025345558654,
+ "tokens_seen": 1382154240
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046969709127382144,
+ "loss": 1.3798,
+ "theoretical_loss": 3.5701874949258423,
+ "tokens_seen": 1382219776
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004696810431293882,
+ "loss": 1.2812,
+ "theoretical_loss": 3.5701724562085384,
+ "tokens_seen": 1382285312
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046966499498495495,
+ "loss": 1.3829,
+ "theoretical_loss": 3.570157418403854,
+ "tokens_seen": 1382350848
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004696489468405216,
+ "loss": 1.4301,
+ "theoretical_loss": 3.5701423815116904,
+ "tokens_seen": 1382416384
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046963289869608835,
+ "loss": 1.3868,
+ "theoretical_loss": 3.570127345531949,
+ "tokens_seen": 1382481920
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000469616850551655,
+ "loss": 1.4893,
+ "theoretical_loss": 3.5701123104645314,
+ "tokens_seen": 1382547456
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046960080240722175,
+ "loss": 1.4028,
+ "theoretical_loss": 3.5700972763093395,
+ "tokens_seen": 1382612992
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004695847542627884,
+ "loss": 1.3122,
+ "theoretical_loss": 3.570082243066274,
+ "tokens_seen": 1382678528
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046956870611835515,
+ "loss": 1.432,
+ "theoretical_loss": 3.5700672107352363,
+ "tokens_seen": 1382744064
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 464319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3647305965423584,
+ "objective/train/theoretical_loss": 3.570052179316128,
+ "objective/train/tokens_used": 1403269600,
+ "theoretical_loss": 3.570052179316128,
+ "tokens_seen": 1382809600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004695526579739218,
+ "loss": 1.3253,
+ "theoretical_loss": 3.570052179316128,
+ "tokens_seen": 1382809600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004695366098294885,
+ "loss": 1.3306,
+ "theoretical_loss": 3.570037148808851,
+ "tokens_seen": 1382875136
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004695205616850552,
+ "loss": 1.3643,
+ "theoretical_loss": 3.570022119213306,
+ "tokens_seen": 1382940672
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004695045135406219,
+ "loss": 1.304,
+ "theoretical_loss": 3.5700070905293955,
+ "tokens_seen": 1383006208
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004694884653961886,
+ "loss": 1.3723,
+ "theoretical_loss": 3.56999206275702,
+ "tokens_seen": 1383071744
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004694724172517553,
+ "loss": 1.3308,
+ "theoretical_loss": 3.5699770358960814,
+ "tokens_seen": 1383137280
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000469456369107322,
+ "loss": 1.3572,
+ "theoretical_loss": 3.5699620099464813,
+ "tokens_seen": 1383202816
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004694403209628887,
+ "loss": 1.2558,
+ "theoretical_loss": 3.569946984908122,
+ "tokens_seen": 1383268352
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004694242728184554,
+ "loss": 1.3347,
+ "theoretical_loss": 3.5699319607809032,
+ "tokens_seen": 1383333888
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004694082246740221,
+ "loss": 1.3705,
+ "theoretical_loss": 3.569916937564728,
+ "tokens_seen": 1383399424
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046939217652958875,
+ "loss": 1.3819,
+ "theoretical_loss": 3.569901915259498,
+ "tokens_seen": 1383464960
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004693761283851555,
+ "loss": 1.3042,
+ "theoretical_loss": 3.569886893865114,
+ "tokens_seen": 1383530496
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046936008024072215,
+ "loss": 1.2829,
+ "theoretical_loss": 3.5698718733814783,
+ "tokens_seen": 1383596032
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004693440320962889,
+ "loss": 1.3505,
+ "theoretical_loss": 3.569856853808492,
+ "tokens_seen": 1383661568
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046932798395185555,
+ "loss": 1.375,
+ "theoretical_loss": 3.5698418351460575,
+ "tokens_seen": 1383727104
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004693119358074223,
+ "loss": 1.2966,
+ "theoretical_loss": 3.5698268173940755,
+ "tokens_seen": 1383792640
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046929588766298896,
+ "loss": 1.2349,
+ "theoretical_loss": 3.5698118005524484,
+ "tokens_seen": 1383858176
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004692798395185557,
+ "loss": 1.3181,
+ "theoretical_loss": 3.5697967846210776,
+ "tokens_seen": 1383923712
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046926379137412236,
+ "loss": 1.2792,
+ "theoretical_loss": 3.5697817695998655,
+ "tokens_seen": 1383989248
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004692477432296891,
+ "loss": 1.2963,
+ "theoretical_loss": 3.569766755488713,
+ "tokens_seen": 1384054784
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046923169508525576,
+ "loss": 1.3114,
+ "theoretical_loss": 3.569751742287522,
+ "tokens_seen": 1384120320
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004692156469408225,
+ "loss": 1.2276,
+ "theoretical_loss": 3.569736729996194,
+ "tokens_seen": 1384185856
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046919959879638916,
+ "loss": 1.3369,
+ "theoretical_loss": 3.5697217186146317,
+ "tokens_seen": 1384251392
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004691835506519559,
+ "loss": 1.4601,
+ "theoretical_loss": 3.5697067081427365,
+ "tokens_seen": 1384316928
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046916750250752256,
+ "loss": 1.3295,
+ "theoretical_loss": 3.56969169858041,
+ "tokens_seen": 1384382464
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004691514543630893,
+ "loss": 1.3331,
+ "theoretical_loss": 3.569676689927554,
+ "tokens_seen": 1384448000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046913540621865596,
+ "loss": 1.4236,
+ "theoretical_loss": 3.5696616821840705,
+ "tokens_seen": 1384513536
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004691193580742227,
+ "loss": 1.4035,
+ "theoretical_loss": 3.5696466753498615,
+ "tokens_seen": 1384579072
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046910330992978936,
+ "loss": 1.3353,
+ "theoretical_loss": 3.5696316694248287,
+ "tokens_seen": 1384644608
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004690872617853561,
+ "loss": 1.3365,
+ "theoretical_loss": 3.5696166644088736,
+ "tokens_seen": 1384710144
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046907121364092276,
+ "loss": 1.3888,
+ "theoretical_loss": 3.569601660301899,
+ "tokens_seen": 1384775680
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004690551654964895,
+ "loss": 1.233,
+ "theoretical_loss": 3.569586657103806,
+ "tokens_seen": 1384841216
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046903911735205617,
+ "loss": 1.2804,
+ "theoretical_loss": 3.569571654814497,
+ "tokens_seen": 1384906752
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004690230692076229,
+ "loss": 1.349,
+ "theoretical_loss": 3.5695566534338736,
+ "tokens_seen": 1384972288
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046900702106318957,
+ "loss": 1.2633,
+ "theoretical_loss": 3.5695416529618385,
+ "tokens_seen": 1385037824
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004689909729187563,
+ "loss": 1.2326,
+ "theoretical_loss": 3.569526653398293,
+ "tokens_seen": 1385103360
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046897492477432297,
+ "loss": 1.3941,
+ "theoretical_loss": 3.569511654743139,
+ "tokens_seen": 1385168896
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004689588766298897,
+ "loss": 1.4688,
+ "theoretical_loss": 3.569496656996279,
+ "tokens_seen": 1385234432
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046894282848545637,
+ "loss": 1.3033,
+ "theoretical_loss": 3.569481660157615,
+ "tokens_seen": 1385299968
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004689267803410231,
+ "loss": 1.359,
+ "theoretical_loss": 3.5694666642270487,
+ "tokens_seen": 1385365504
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046891073219658977,
+ "loss": 1.3083,
+ "theoretical_loss": 3.569451669204482,
+ "tokens_seen": 1385431040
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004688946840521565,
+ "loss": 1.3679,
+ "theoretical_loss": 3.5694366750898174,
+ "tokens_seen": 1385496576
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046887863590772317,
+ "loss": 1.3626,
+ "theoretical_loss": 3.569421681882957,
+ "tokens_seen": 1385562112
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046886258776328993,
+ "loss": 1.3638,
+ "theoretical_loss": 3.569406689583803,
+ "tokens_seen": 1385627648
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004688465396188566,
+ "loss": 1.3713,
+ "theoretical_loss": 3.5693916981922564,
+ "tokens_seen": 1385693184
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046883049147442333,
+ "loss": 1.3344,
+ "theoretical_loss": 3.569376707708221,
+ "tokens_seen": 1385758720
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046881444332999,
+ "loss": 1.2535,
+ "theoretical_loss": 3.5693617181315984,
+ "tokens_seen": 1385824256
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046879839518555673,
+ "loss": 1.3254,
+ "theoretical_loss": 3.56934672946229,
+ "tokens_seen": 1385889792
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687823470411234,
+ "loss": 1.3007,
+ "theoretical_loss": 3.5693317417001986,
+ "tokens_seen": 1385955328
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687662988966901,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5693167548452265,
+ "tokens_seen": 1386020864
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 465326,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.428898572921753,
+ "objective/train/theoretical_loss": 3.5693017688972755,
+ "objective/train/tokens_used": 1406546400,
+ "theoretical_loss": 3.5693017688972755,
+ "tokens_seen": 1386086400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687502507522568,
+ "loss": 1.375,
+ "theoretical_loss": 3.5693017688972755,
+ "tokens_seen": 1386086400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687342026078235,
+ "loss": 1.2942,
+ "theoretical_loss": 3.5692867838562483,
+ "tokens_seen": 1386151936
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687181544633902,
+ "loss": 1.2628,
+ "theoretical_loss": 3.5692717997220464,
+ "tokens_seen": 1386217472
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004687021063189569,
+ "loss": 1.3732,
+ "theoretical_loss": 3.569256816494573,
+ "tokens_seen": 1386283008
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004686860581745236,
+ "loss": 1.2398,
+ "theoretical_loss": 3.56924183417373,
+ "tokens_seen": 1386348544
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004686700100300903,
+ "loss": 1.3177,
+ "theoretical_loss": 3.569226852759419,
+ "tokens_seen": 1386414080
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000468653961885657,
+ "loss": 1.2729,
+ "theoretical_loss": 3.569211872251543,
+ "tokens_seen": 1386479616
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004686379137412237,
+ "loss": 1.3639,
+ "theoretical_loss": 3.5691968926500053,
+ "tokens_seen": 1386545152
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046862186559679033,
+ "loss": 1.3352,
+ "theoretical_loss": 3.569181913954706,
+ "tokens_seen": 1386610688
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004686058174523571,
+ "loss": 1.3415,
+ "theoretical_loss": 3.5691669361655487,
+ "tokens_seen": 1386676224
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046858976930792373,
+ "loss": 1.3406,
+ "theoretical_loss": 3.5691519592824363,
+ "tokens_seen": 1386741760
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004685737211634905,
+ "loss": 1.2054,
+ "theoretical_loss": 3.5691369833052695,
+ "tokens_seen": 1386807296
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046855767301905713,
+ "loss": 1.348,
+ "theoretical_loss": 3.5691220082339523,
+ "tokens_seen": 1386872832
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004685416248746239,
+ "loss": 1.3887,
+ "theoretical_loss": 3.5691070340683866,
+ "tokens_seen": 1386938368
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046852557673019053,
+ "loss": 1.2267,
+ "theoretical_loss": 3.569092060808475,
+ "tokens_seen": 1387003904
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004685095285857573,
+ "loss": 1.3383,
+ "theoretical_loss": 3.569077088454119,
+ "tokens_seen": 1387069440
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046849348044132393,
+ "loss": 1.3754,
+ "theoretical_loss": 3.569062117005222,
+ "tokens_seen": 1387134976
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004684774322968907,
+ "loss": 1.4019,
+ "theoretical_loss": 3.5690471464616866,
+ "tokens_seen": 1387200512
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046846138415245744,
+ "loss": 1.3249,
+ "theoretical_loss": 3.5690321768234146,
+ "tokens_seen": 1387266048
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004684453360080241,
+ "loss": 1.3314,
+ "theoretical_loss": 3.5690172080903086,
+ "tokens_seen": 1387331584
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046842928786359084,
+ "loss": 1.3259,
+ "theoretical_loss": 3.5690022402622716,
+ "tokens_seen": 1387397120
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004684132397191575,
+ "loss": 1.3604,
+ "theoretical_loss": 3.5689872733392054,
+ "tokens_seen": 1387462656
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046839719157472425,
+ "loss": 1.2822,
+ "theoretical_loss": 3.568972307321013,
+ "tokens_seen": 1387528192
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004683811434302909,
+ "loss": 1.3639,
+ "theoretical_loss": 3.5689573422075975,
+ "tokens_seen": 1387593728
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046836509528585765,
+ "loss": 1.2465,
+ "theoretical_loss": 3.5689423779988605,
+ "tokens_seen": 1387659264
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004683490471414243,
+ "loss": 1.2601,
+ "theoretical_loss": 3.5689274146947048,
+ "tokens_seen": 1387724800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046833299899699105,
+ "loss": 1.3717,
+ "theoretical_loss": 3.5689124522950335,
+ "tokens_seen": 1387790336
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004683169508525577,
+ "loss": 1.2633,
+ "theoretical_loss": 3.568897490799749,
+ "tokens_seen": 1387855872
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046830090270812445,
+ "loss": 1.3051,
+ "theoretical_loss": 3.5688825302087537,
+ "tokens_seen": 1387921408
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004682848545636911,
+ "loss": 1.3714,
+ "theoretical_loss": 3.5688675705219497,
+ "tokens_seen": 1387986944
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046826880641925785,
+ "loss": 1.2908,
+ "theoretical_loss": 3.568852611739241,
+ "tokens_seen": 1388052480
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004682527582748245,
+ "loss": 1.3092,
+ "theoretical_loss": 3.5688376538605295,
+ "tokens_seen": 1388118016
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046823671013039125,
+ "loss": 1.444,
+ "theoretical_loss": 3.5688226968857184,
+ "tokens_seen": 1388183552
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004682206619859579,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5688077408147096,
+ "tokens_seen": 1388249088
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046820461384152465,
+ "loss": 1.2782,
+ "theoretical_loss": 3.5687927856474064,
+ "tokens_seen": 1388314624
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004681885656970913,
+ "loss": 1.4044,
+ "theoretical_loss": 3.568777831383711,
+ "tokens_seen": 1388380160
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046817251755265805,
+ "loss": 1.3039,
+ "theoretical_loss": 3.568762878023527,
+ "tokens_seen": 1388445696
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004681564694082247,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5687479255667567,
+ "tokens_seen": 1388511232
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046814042126379146,
+ "loss": 1.2748,
+ "theoretical_loss": 3.5687329740133027,
+ "tokens_seen": 1388576768
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004681243731193581,
+ "loss": 1.4446,
+ "theoretical_loss": 3.5687180233630684,
+ "tokens_seen": 1388642304
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004681083249749248,
+ "loss": 1.4675,
+ "theoretical_loss": 3.5687030736159557,
+ "tokens_seen": 1388707840
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004680922768304915,
+ "loss": 1.3458,
+ "theoretical_loss": 3.5686881247718683,
+ "tokens_seen": 1388773376
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004680762286860582,
+ "loss": 1.4207,
+ "theoretical_loss": 3.5686731768307087,
+ "tokens_seen": 1388838912
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004680601805416249,
+ "loss": 1.3286,
+ "theoretical_loss": 3.568658229792379,
+ "tokens_seen": 1388904448
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004680441323971916,
+ "loss": 1.3601,
+ "theoretical_loss": 3.5686432836567836,
+ "tokens_seen": 1388969984
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004680280842527583,
+ "loss": 1.316,
+ "theoretical_loss": 3.5686283384238244,
+ "tokens_seen": 1389035520
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000468012036108325,
+ "loss": 1.2939,
+ "theoretical_loss": 3.5686133940934046,
+ "tokens_seen": 1389101056
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046799598796389165,
+ "loss": 1.3304,
+ "theoretical_loss": 3.5685984506654265,
+ "tokens_seen": 1389166592
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004679799398194584,
+ "loss": 1.414,
+ "theoretical_loss": 3.568583508139794,
+ "tokens_seen": 1389232128
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046796389167502505,
+ "loss": 1.1521,
+ "theoretical_loss": 3.5685685665164097,
+ "tokens_seen": 1389297664
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 466536,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.399806261062622,
+ "objective/train/theoretical_loss": 3.568553625795176,
+ "objective/train/tokens_used": 1409823200,
+ "theoretical_loss": 3.568553625795176,
+ "tokens_seen": 1389363200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004679478435305918,
+ "loss": 1.4144,
+ "theoretical_loss": 3.568553625795176,
+ "tokens_seen": 1389363200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046793179538615846,
+ "loss": 1.3563,
+ "theoretical_loss": 3.568538685975997,
+ "tokens_seen": 1389428736
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004679157472417252,
+ "loss": 1.2399,
+ "theoretical_loss": 3.5685237470587747,
+ "tokens_seen": 1389494272
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046789969909729186,
+ "loss": 1.2987,
+ "theoretical_loss": 3.5685088090434123,
+ "tokens_seen": 1389559808
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004678836509528586,
+ "loss": 1.2261,
+ "theoretical_loss": 3.5684938719298134,
+ "tokens_seen": 1389625344
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046786760280842526,
+ "loss": 1.301,
+ "theoretical_loss": 3.5684789357178803,
+ "tokens_seen": 1389690880
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000467851554663992,
+ "loss": 1.3076,
+ "theoretical_loss": 3.5684640004075163,
+ "tokens_seen": 1389756416
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046783550651955866,
+ "loss": 1.2074,
+ "theoretical_loss": 3.5684490659986245,
+ "tokens_seen": 1389821952
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004678194583751254,
+ "loss": 1.3984,
+ "theoretical_loss": 3.5684341324911086,
+ "tokens_seen": 1389887488
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046780341023069206,
+ "loss": 1.2291,
+ "theoretical_loss": 3.5684191998848704,
+ "tokens_seen": 1389953024
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004677873620862588,
+ "loss": 1.4563,
+ "theoretical_loss": 3.5684042681798145,
+ "tokens_seen": 1390018560
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046777131394182546,
+ "loss": 1.26,
+ "theoretical_loss": 3.5683893373758426,
+ "tokens_seen": 1390084096
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004677552657973922,
+ "loss": 1.289,
+ "theoretical_loss": 3.5683744074728585,
+ "tokens_seen": 1390149632
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046773921765295886,
+ "loss": 1.2825,
+ "theoretical_loss": 3.5683594784707657,
+ "tokens_seen": 1390215168
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004677231695085256,
+ "loss": 1.3856,
+ "theoretical_loss": 3.568344550369467,
+ "tokens_seen": 1390280704
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046770712136409226,
+ "loss": 1.448,
+ "theoretical_loss": 3.5683296231688653,
+ "tokens_seen": 1390346240
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000467691073219659,
+ "loss": 1.3966,
+ "theoretical_loss": 3.568314696868865,
+ "tokens_seen": 1390411776
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046767502507522567,
+ "loss": 1.4233,
+ "theoretical_loss": 3.5682997714693676,
+ "tokens_seen": 1390477312
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004676589769307924,
+ "loss": 1.3166,
+ "theoretical_loss": 3.5682848469702773,
+ "tokens_seen": 1390542848
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046764292878635907,
+ "loss": 1.2426,
+ "theoretical_loss": 3.5682699233714974,
+ "tokens_seen": 1390608384
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004676268806419258,
+ "loss": 1.2408,
+ "theoretical_loss": 3.568255000672931,
+ "tokens_seen": 1390673920
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046761083249749247,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5682400788744815,
+ "tokens_seen": 1390739456
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004675947843530592,
+ "loss": 1.3709,
+ "theoretical_loss": 3.5682251579760518,
+ "tokens_seen": 1390804992
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046757873620862587,
+ "loss": 1.397,
+ "theoretical_loss": 3.5682102379775458,
+ "tokens_seen": 1390870528
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004675626880641926,
+ "loss": 1.3926,
+ "theoretical_loss": 3.5681953188788667,
+ "tokens_seen": 1390936064
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046754663991975927,
+ "loss": 1.2739,
+ "theoretical_loss": 3.568180400679917,
+ "tokens_seen": 1391001600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000467530591775326,
+ "loss": 1.2746,
+ "theoretical_loss": 3.568165483380601,
+ "tokens_seen": 1391067136
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046751454363089267,
+ "loss": 1.3454,
+ "theoretical_loss": 3.568150566980822,
+ "tokens_seen": 1391132672
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004674984954864594,
+ "loss": 1.3187,
+ "theoretical_loss": 3.5681356514804827,
+ "tokens_seen": 1391198208
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004674824473420261,
+ "loss": 1.3364,
+ "theoretical_loss": 3.568120736879487,
+ "tokens_seen": 1391263744
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046746639919759283,
+ "loss": 1.3145,
+ "theoretical_loss": 3.5681058231777385,
+ "tokens_seen": 1391329280
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004674503510531595,
+ "loss": 1.504,
+ "theoretical_loss": 3.5680909103751404,
+ "tokens_seen": 1391394816
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046743430290872623,
+ "loss": 1.4819,
+ "theoretical_loss": 3.5680759984715964,
+ "tokens_seen": 1391460352
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004674182547642929,
+ "loss": 1.3309,
+ "theoretical_loss": 3.568061087467009,
+ "tokens_seen": 1391525888
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046740220661985963,
+ "loss": 1.3604,
+ "theoretical_loss": 3.568046177361283,
+ "tokens_seen": 1391591424
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004673861584754263,
+ "loss": 1.3768,
+ "theoretical_loss": 3.5680312681543205,
+ "tokens_seen": 1391656960
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046737011033099303,
+ "loss": 1.2238,
+ "theoretical_loss": 3.5680163598460264,
+ "tokens_seen": 1391722496
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004673540621865597,
+ "loss": 1.3065,
+ "theoretical_loss": 3.5680014524363033,
+ "tokens_seen": 1391788032
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004673380140421264,
+ "loss": 1.2669,
+ "theoretical_loss": 3.5679865459250553,
+ "tokens_seen": 1391853568
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004673219658976931,
+ "loss": 1.3582,
+ "theoretical_loss": 3.5679716403121855,
+ "tokens_seen": 1391919104
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004673059177532598,
+ "loss": 1.326,
+ "theoretical_loss": 3.5679567355975976,
+ "tokens_seen": 1391984640
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004672898696088265,
+ "loss": 1.3256,
+ "theoretical_loss": 3.567941831781195,
+ "tokens_seen": 1392050176
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004672738214643932,
+ "loss": 1.3533,
+ "theoretical_loss": 3.5679269288628817,
+ "tokens_seen": 1392115712
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046725777331995994,
+ "loss": 1.3419,
+ "theoretical_loss": 3.567912026842561,
+ "tokens_seen": 1392181248
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004672417251755266,
+ "loss": 1.3674,
+ "theoretical_loss": 3.567897125720137,
+ "tokens_seen": 1392246784
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046722567703109334,
+ "loss": 1.3898,
+ "theoretical_loss": 3.567882225495512,
+ "tokens_seen": 1392312320
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046720962888666,
+ "loss": 1.3017,
+ "theoretical_loss": 3.5678673261685914,
+ "tokens_seen": 1392377856
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046719358074222674,
+ "loss": 1.3046,
+ "theoretical_loss": 3.567852427739278,
+ "tokens_seen": 1392443392
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004671775325977934,
+ "loss": 1.3052,
+ "theoretical_loss": 3.567837530207475,
+ "tokens_seen": 1392508928
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046716148445336014,
+ "loss": 1.3724,
+ "theoretical_loss": 3.567822633573087,
+ "tokens_seen": 1392574464
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 467439,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3975262641906738,
+ "objective/train/theoretical_loss": 3.5678077378360173,
+ "objective/train/tokens_used": 1413100000,
+ "theoretical_loss": 3.5678077378360173,
+ "tokens_seen": 1392640000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004671454363089268,
+ "loss": 1.3818,
+ "theoretical_loss": 3.5678077378360173,
+ "tokens_seen": 1392640000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046712938816449354,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5677928429961705,
+ "tokens_seen": 1392705536
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004671133400200602,
+ "loss": 1.3975,
+ "theoretical_loss": 3.5677779490534487,
+ "tokens_seen": 1392771072
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046709729187562694,
+ "loss": 1.3079,
+ "theoretical_loss": 3.567763056007756,
+ "tokens_seen": 1392836608
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004670812437311936,
+ "loss": 1.264,
+ "theoretical_loss": 3.5677481638589974,
+ "tokens_seen": 1392902144
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046706519558676034,
+ "loss": 1.2806,
+ "theoretical_loss": 3.567733272607076,
+ "tokens_seen": 1392967680
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000467049147442327,
+ "loss": 1.4315,
+ "theoretical_loss": 3.567718382251895,
+ "tokens_seen": 1393033216
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046703309929789375,
+ "loss": 1.3923,
+ "theoretical_loss": 3.5677034927933593,
+ "tokens_seen": 1393098752
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004670170511534604,
+ "loss": 1.3241,
+ "theoretical_loss": 3.567688604231372,
+ "tokens_seen": 1393164288
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046700100300902715,
+ "loss": 1.3409,
+ "theoretical_loss": 3.567673716565837,
+ "tokens_seen": 1393229824
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004669849548645938,
+ "loss": 1.2756,
+ "theoretical_loss": 3.5676588297966583,
+ "tokens_seen": 1393295360
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046696890672016055,
+ "loss": 1.3889,
+ "theoretical_loss": 3.5676439439237395,
+ "tokens_seen": 1393360896
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004669528585757272,
+ "loss": 1.4035,
+ "theoretical_loss": 3.5676290589469852,
+ "tokens_seen": 1393426432
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046693681043129395,
+ "loss": 1.2813,
+ "theoretical_loss": 3.567614174866299,
+ "tokens_seen": 1393491968
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004669207622868606,
+ "loss": 1.2742,
+ "theoretical_loss": 3.567599291681584,
+ "tokens_seen": 1393557504
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046690471414242735,
+ "loss": 1.3633,
+ "theoretical_loss": 3.5675844093927456,
+ "tokens_seen": 1393623040
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000466888665997994,
+ "loss": 1.2263,
+ "theoretical_loss": 3.5675695279996864,
+ "tokens_seen": 1393688576
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046687261785356075,
+ "loss": 1.4491,
+ "theoretical_loss": 3.5675546475023108,
+ "tokens_seen": 1393754112
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004668565697091274,
+ "loss": 1.344,
+ "theoretical_loss": 3.567539767900523,
+ "tokens_seen": 1393819648
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046684052156469415,
+ "loss": 1.4258,
+ "theoretical_loss": 3.5675248891942273,
+ "tokens_seen": 1393885184
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004668244734202608,
+ "loss": 1.357,
+ "theoretical_loss": 3.5675100113833267,
+ "tokens_seen": 1393950720
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046680842527582755,
+ "loss": 1.3401,
+ "theoretical_loss": 3.567495134467726,
+ "tokens_seen": 1394016256
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004667923771313942,
+ "loss": 1.3624,
+ "theoretical_loss": 3.5674802584473295,
+ "tokens_seen": 1394081792
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046677632898696096,
+ "loss": 1.4048,
+ "theoretical_loss": 3.56746538332204,
+ "tokens_seen": 1394147328
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004667602808425276,
+ "loss": 1.36,
+ "theoretical_loss": 3.5674505090917634,
+ "tokens_seen": 1394212864
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046674423269809436,
+ "loss": 1.2604,
+ "theoretical_loss": 3.567435635756402,
+ "tokens_seen": 1394278400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000466728184553661,
+ "loss": 1.3066,
+ "theoretical_loss": 3.567420763315861,
+ "tokens_seen": 1394343936
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004667121364092277,
+ "loss": 1.2258,
+ "theoretical_loss": 3.567405891770044,
+ "tokens_seen": 1394409472
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666960882647944,
+ "loss": 1.2764,
+ "theoretical_loss": 3.5673910211188553,
+ "tokens_seen": 1394475008
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666800401203611,
+ "loss": 1.2881,
+ "theoretical_loss": 3.567376151362199,
+ "tokens_seen": 1394540544
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666639919759278,
+ "loss": 1.3034,
+ "theoretical_loss": 3.5673612824999794,
+ "tokens_seen": 1394606080
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666479438314945,
+ "loss": 1.3499,
+ "theoretical_loss": 3.5673464145321003,
+ "tokens_seen": 1394671616
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666318956870612,
+ "loss": 1.3134,
+ "theoretical_loss": 3.5673315474584664,
+ "tokens_seen": 1394737152
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004666158475426279,
+ "loss": 1.2493,
+ "theoretical_loss": 3.5673166812789816,
+ "tokens_seen": 1394802688
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004665997993981946,
+ "loss": 1.3878,
+ "theoretical_loss": 3.56730181599355,
+ "tokens_seen": 1394868224
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004665837512537613,
+ "loss": 1.346,
+ "theoretical_loss": 3.5672869516020764,
+ "tokens_seen": 1394933760
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046656770310932796,
+ "loss": 1.3826,
+ "theoretical_loss": 3.567272088104464,
+ "tokens_seen": 1394999296
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004665516549648947,
+ "loss": 1.3058,
+ "theoretical_loss": 3.567257225500618,
+ "tokens_seen": 1395064832
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046653560682046136,
+ "loss": 1.2699,
+ "theoretical_loss": 3.5672423637904425,
+ "tokens_seen": 1395130368
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004665195586760281,
+ "loss": 1.3409,
+ "theoretical_loss": 3.567227502973841,
+ "tokens_seen": 1395195904
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046650351053159476,
+ "loss": 1.4202,
+ "theoretical_loss": 3.5672126430507194,
+ "tokens_seen": 1395261440
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004664874623871615,
+ "loss": 1.3728,
+ "theoretical_loss": 3.5671977840209808,
+ "tokens_seen": 1395326976
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046647141424272816,
+ "loss": 1.321,
+ "theoretical_loss": 3.56718292588453,
+ "tokens_seen": 1395392512
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004664553660982949,
+ "loss": 1.402,
+ "theoretical_loss": 3.56716806864127,
+ "tokens_seen": 1395458048
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046643931795386156,
+ "loss": 1.2982,
+ "theoretical_loss": 3.5671532122911076,
+ "tokens_seen": 1395523584
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004664232698094283,
+ "loss": 1.3687,
+ "theoretical_loss": 3.5671383568339454,
+ "tokens_seen": 1395589120
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046640722166499496,
+ "loss": 1.3898,
+ "theoretical_loss": 3.567123502269688,
+ "tokens_seen": 1395654656
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004663911735205617,
+ "loss": 1.3348,
+ "theoretical_loss": 3.5671086485982406,
+ "tokens_seen": 1395720192
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046637512537612836,
+ "loss": 1.2986,
+ "theoretical_loss": 3.567093795819507,
+ "tokens_seen": 1395785728
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004663590772316951,
+ "loss": 1.2645,
+ "theoretical_loss": 3.5670789439333914,
+ "tokens_seen": 1395851264
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 468614,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.128334403038025,
+ "objective/train/theoretical_loss": 3.5670640929397988,
+ "objective/train/tokens_used": 1416376800,
+ "theoretical_loss": 3.5670640929397988,
+ "tokens_seen": 1395916800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046634302908726176,
+ "loss": 1.151,
+ "theoretical_loss": 3.5670640929397988,
+ "tokens_seen": 1395916800
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004663269809428285,
+ "loss": 1.3527,
+ "theoretical_loss": 3.5670492428386336,
+ "tokens_seen": 1395982336
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046631093279839517,
+ "loss": 1.2882,
+ "theoretical_loss": 3.5670343936298,
+ "tokens_seen": 1396047872
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004662948846539619,
+ "loss": 1.2289,
+ "theoretical_loss": 3.5670195453132028,
+ "tokens_seen": 1396113408
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046627883650952857,
+ "loss": 1.2635,
+ "theoretical_loss": 3.5670046978887457,
+ "tokens_seen": 1396178944
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004662627883650953,
+ "loss": 1.3362,
+ "theoretical_loss": 3.5669898513563347,
+ "tokens_seen": 1396244480
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046624674022066197,
+ "loss": 1.3272,
+ "theoretical_loss": 3.5669750057158733,
+ "tokens_seen": 1396310016
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004662306920762287,
+ "loss": 1.3136,
+ "theoretical_loss": 3.566960160967266,
+ "tokens_seen": 1396375552
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046621464393179537,
+ "loss": 1.2719,
+ "theoretical_loss": 3.5669453171104175,
+ "tokens_seen": 1396441088
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004661985957873621,
+ "loss": 1.3208,
+ "theoretical_loss": 3.566930474145233,
+ "tokens_seen": 1396506624
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046618254764292877,
+ "loss": 1.3715,
+ "theoretical_loss": 3.566915632071616,
+ "tokens_seen": 1396572160
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004661664994984955,
+ "loss": 1.3914,
+ "theoretical_loss": 3.5669007908894725,
+ "tokens_seen": 1396637696
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046615045135406217,
+ "loss": 1.2028,
+ "theoretical_loss": 3.5668859505987056,
+ "tokens_seen": 1396703232
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004661344032096289,
+ "loss": 1.313,
+ "theoretical_loss": 3.5668711111992213,
+ "tokens_seen": 1396768768
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004661183550651956,
+ "loss": 1.3542,
+ "theoretical_loss": 3.5668562726909236,
+ "tokens_seen": 1396834304
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046610230692076233,
+ "loss": 1.3529,
+ "theoretical_loss": 3.5668414350737168,
+ "tokens_seen": 1396899840
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000466086258776329,
+ "loss": 1.2057,
+ "theoretical_loss": 3.5668265983475065,
+ "tokens_seen": 1396965376
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046607021063189573,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5668117625121964,
+ "tokens_seen": 1397030912
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046605416248746243,
+ "loss": 1.3327,
+ "theoretical_loss": 3.566796927567692,
+ "tokens_seen": 1397096448
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046603811434302913,
+ "loss": 1.2781,
+ "theoretical_loss": 3.566782093513898,
+ "tokens_seen": 1397161984
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046602206619859583,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5667672603507183,
+ "tokens_seen": 1397227520
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046600601805416253,
+ "loss": 1.3553,
+ "theoretical_loss": 3.5667524280780585,
+ "tokens_seen": 1397293056
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046598996990972923,
+ "loss": 1.4168,
+ "theoretical_loss": 3.5667375966958232,
+ "tokens_seen": 1397358592
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046597392176529593,
+ "loss": 1.1694,
+ "theoretical_loss": 3.566722766203917,
+ "tokens_seen": 1397424128
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046595787362086263,
+ "loss": 1.2808,
+ "theoretical_loss": 3.5667079366022447,
+ "tokens_seen": 1397489664
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004659418254764293,
+ "loss": 1.4308,
+ "theoretical_loss": 3.5666931078907114,
+ "tokens_seen": 1397555200
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046592577733199603,
+ "loss": 1.42,
+ "theoretical_loss": 3.5666782800692216,
+ "tokens_seen": 1397620736
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004659097291875627,
+ "loss": 1.2872,
+ "theoretical_loss": 3.566663453137681,
+ "tokens_seen": 1397686272
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046589368104312944,
+ "loss": 1.4318,
+ "theoretical_loss": 3.566648627095993,
+ "tokens_seen": 1397751808
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004658776328986961,
+ "loss": 1.3049,
+ "theoretical_loss": 3.5666338019440635,
+ "tokens_seen": 1397817344
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046586158475426284,
+ "loss": 1.3829,
+ "theoretical_loss": 3.5666189776817974,
+ "tokens_seen": 1397882880
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004658455366098295,
+ "loss": 1.2541,
+ "theoretical_loss": 3.5666041543090987,
+ "tokens_seen": 1397948416
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046582948846539624,
+ "loss": 1.3511,
+ "theoretical_loss": 3.566589331825874,
+ "tokens_seen": 1398013952
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004658134403209629,
+ "loss": 1.3563,
+ "theoretical_loss": 3.566574510232026,
+ "tokens_seen": 1398079488
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046579739217652964,
+ "loss": 1.368,
+ "theoretical_loss": 3.5665596895274616,
+ "tokens_seen": 1398145024
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004657813440320963,
+ "loss": 1.3602,
+ "theoretical_loss": 3.566544869712085,
+ "tokens_seen": 1398210560
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046576529588766304,
+ "loss": 1.3254,
+ "theoretical_loss": 3.5665300507858007,
+ "tokens_seen": 1398276096
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004657492477432297,
+ "loss": 1.3215,
+ "theoretical_loss": 3.5665152327485146,
+ "tokens_seen": 1398341632
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046573319959879644,
+ "loss": 1.3215,
+ "theoretical_loss": 3.5665004156001308,
+ "tokens_seen": 1398407168
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004657171514543631,
+ "loss": 1.3927,
+ "theoretical_loss": 3.566485599340555,
+ "tokens_seen": 1398472704
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046570110330992984,
+ "loss": 1.3609,
+ "theoretical_loss": 3.5664707839696925,
+ "tokens_seen": 1398538240
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004656850551654965,
+ "loss": 1.3725,
+ "theoretical_loss": 3.566455969487448,
+ "tokens_seen": 1398603776
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046566900702106325,
+ "loss": 1.252,
+ "theoretical_loss": 3.5664411558937257,
+ "tokens_seen": 1398669312
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004656529588766299,
+ "loss": 1.3539,
+ "theoretical_loss": 3.566426343188432,
+ "tokens_seen": 1398734848
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046563691073219665,
+ "loss": 1.3519,
+ "theoretical_loss": 3.5664115313714713,
+ "tokens_seen": 1398800384
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004656208625877633,
+ "loss": 1.2448,
+ "theoretical_loss": 3.566396720442749,
+ "tokens_seen": 1398865920
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046560481444333005,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5663819104021695,
+ "tokens_seen": 1398931456
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004655887662988967,
+ "loss": 1.3861,
+ "theoretical_loss": 3.5663671012496394,
+ "tokens_seen": 1398996992
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046557271815446345,
+ "loss": 1.433,
+ "theoretical_loss": 3.566352292985062,
+ "tokens_seen": 1399062528
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004655566700100301,
+ "loss": 1.3061,
+ "theoretical_loss": 3.5663374856083445,
+ "tokens_seen": 1399128064
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 469650,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3298465013504028,
+ "objective/train/theoretical_loss": 3.5663226791193905,
+ "objective/train/tokens_used": 1419653600,
+ "theoretical_loss": 3.5663226791193905,
+ "tokens_seen": 1399193600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046554062186559685,
+ "loss": 1.2899,
+ "theoretical_loss": 3.5663226791193905,
+ "tokens_seen": 1399193600
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004655245737211635,
+ "loss": 1.3579,
+ "theoretical_loss": 3.566307873518105,
+ "tokens_seen": 1399259136
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046550852557673025,
+ "loss": 1.4427,
+ "theoretical_loss": 3.566293068804395,
+ "tokens_seen": 1399324672
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004654924774322969,
+ "loss": 1.3123,
+ "theoretical_loss": 3.5662782649781644,
+ "tokens_seen": 1399390208
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046547642928786365,
+ "loss": 1.2523,
+ "theoretical_loss": 3.566263462039318,
+ "tokens_seen": 1399455744
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004654603811434303,
+ "loss": 1.3049,
+ "theoretical_loss": 3.5662486599877625,
+ "tokens_seen": 1399521280
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046544433299899705,
+ "loss": 1.3442,
+ "theoretical_loss": 3.566233858823402,
+ "tokens_seen": 1399586816
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004654282848545637,
+ "loss": 1.2045,
+ "theoretical_loss": 3.566219058546143,
+ "tokens_seen": 1399652352
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046541223671013046,
+ "loss": 1.301,
+ "theoretical_loss": 3.566204259155889,
+ "tokens_seen": 1399717888
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004653961885656971,
+ "loss": 1.4409,
+ "theoretical_loss": 3.566189460652547,
+ "tokens_seen": 1399783424
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046538014042126386,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5661746630360214,
+ "tokens_seen": 1399848960
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004653640922768305,
+ "loss": 1.2533,
+ "theoretical_loss": 3.5661598663062177,
+ "tokens_seen": 1399914496
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046534804413239726,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5661450704630413,
+ "tokens_seen": 1399980032
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004653319959879639,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5661302755063975,
+ "tokens_seen": 1400045568
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004653159478435306,
+ "loss": 1.3057,
+ "theoretical_loss": 3.566115481436192,
+ "tokens_seen": 1400111104
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652998996990973,
+ "loss": 1.3049,
+ "theoretical_loss": 3.5661006882523303,
+ "tokens_seen": 1400176640
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000465283851554664,
+ "loss": 1.333,
+ "theoretical_loss": 3.5660858959547173,
+ "tokens_seen": 1400242176
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652678034102307,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5660711045432585,
+ "tokens_seen": 1400307712
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652517552657974,
+ "loss": 1.3437,
+ "theoretical_loss": 3.56605631401786,
+ "tokens_seen": 1400373248
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652357071213641,
+ "loss": 1.2676,
+ "theoretical_loss": 3.5660415243784263,
+ "tokens_seen": 1400438784
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652196589769308,
+ "loss": 1.3211,
+ "theoretical_loss": 3.566026735624863,
+ "tokens_seen": 1400504320
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004652036108324975,
+ "loss": 1.2517,
+ "theoretical_loss": 3.5660119477570764,
+ "tokens_seen": 1400569856
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004651875626880642,
+ "loss": 1.3758,
+ "theoretical_loss": 3.5659971607749714,
+ "tokens_seen": 1400635392
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046517151454363086,
+ "loss": 1.3239,
+ "theoretical_loss": 3.5659823746784536,
+ "tokens_seen": 1400700928
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004651554663991976,
+ "loss": 1.2681,
+ "theoretical_loss": 3.5659675894674288,
+ "tokens_seen": 1400766464
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046513941825476426,
+ "loss": 1.3821,
+ "theoretical_loss": 3.565952805141802,
+ "tokens_seen": 1400832000
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000465123370110331,
+ "loss": 1.4107,
+ "theoretical_loss": 3.565938021701479,
+ "tokens_seen": 1400897536
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046510732196589766,
+ "loss": 1.3933,
+ "theoretical_loss": 3.5659232391463656,
+ "tokens_seen": 1400963072
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004650912738214644,
+ "loss": 1.2784,
+ "theoretical_loss": 3.5659084574763673,
+ "tokens_seen": 1401028608
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046507522567703106,
+ "loss": 1.2199,
+ "theoretical_loss": 3.56589367669139,
+ "tokens_seen": 1401094144
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004650591775325978,
+ "loss": 1.3609,
+ "theoretical_loss": 3.565878896791338,
+ "tokens_seen": 1401159680
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046504312938816446,
+ "loss": 1.4024,
+ "theoretical_loss": 3.5658641177761186,
+ "tokens_seen": 1401225216
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004650270812437312,
+ "loss": 1.2853,
+ "theoretical_loss": 3.5658493396456366,
+ "tokens_seen": 1401290752
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046501103309929786,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5658345623997976,
+ "tokens_seen": 1401356288
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004649949849548646,
+ "loss": 1.3526,
+ "theoretical_loss": 3.565819786038508,
+ "tokens_seen": 1401421824
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046497893681043126,
+ "loss": 1.3236,
+ "theoretical_loss": 3.5658050105616725,
+ "tokens_seen": 1401487360
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000464962888665998,
+ "loss": 1.3269,
+ "theoretical_loss": 3.5657902359691978,
+ "tokens_seen": 1401552896
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046494684052156467,
+ "loss": 1.4531,
+ "theoretical_loss": 3.565775462260988,
+ "tokens_seen": 1401618432
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004649307923771314,
+ "loss": 1.3345,
+ "theoretical_loss": 3.565760689436951,
+ "tokens_seen": 1401683968
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046491474423269807,
+ "loss": 1.3002,
+ "theoretical_loss": 3.565745917496991,
+ "tokens_seen": 1401749504
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004648986960882648,
+ "loss": 1.2906,
+ "theoretical_loss": 3.5657311464410144,
+ "tokens_seen": 1401815040
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046488264794383147,
+ "loss": 1.3056,
+ "theoretical_loss": 3.5657163762689263,
+ "tokens_seen": 1401880576
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004648665997993982,
+ "loss": 1.243,
+ "theoretical_loss": 3.5657016069806335,
+ "tokens_seen": 1401946112
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000464850551654965,
+ "loss": 1.2916,
+ "theoretical_loss": 3.565686838576041,
+ "tokens_seen": 1402011648
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004648345035105316,
+ "loss": 1.4426,
+ "theoretical_loss": 3.5656720710550553,
+ "tokens_seen": 1402077184
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004648184553660984,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5656573044175817,
+ "tokens_seen": 1402142720
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.000464802407221665,
+ "loss": 1.2652,
+ "theoretical_loss": 3.5656425386635267,
+ "tokens_seen": 1402208256
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004647863590772318,
+ "loss": 1.5096,
+ "theoretical_loss": 3.565627773792795,
+ "tokens_seen": 1402273792
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004647703109327984,
+ "loss": 1.2868,
+ "theoretical_loss": 3.5656130098052934,
+ "tokens_seen": 1402339328
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.0004647542627883652,
+ "loss": 1.2618,
+ "theoretical_loss": 3.5655982467009277,
+ "tokens_seen": 1402404864
+ },
+ {
+ "epoch": 0.42,
+ "objective/train/docs_used": 470842,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3026866912841797,
+ "objective/train/theoretical_loss": 3.5655834844796033,
+ "objective/train/tokens_used": 1422930400,
+ "theoretical_loss": 3.5655834844796033,
+ "tokens_seen": 1402470400
+ },
+ {
+ "epoch": 0.42,
+ "learning_rate": 0.00046473821464393183,
+ "loss": 1.3822,
+ "theoretical_loss": 3.5655834844796033,
+ "tokens_seen": 1402470400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004647221664994986,
+ "loss": 1.3368,
+ "theoretical_loss": 3.5655687231412267,
+ "tokens_seen": 1402535936
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046470611835506523,
+ "loss": 1.346,
+ "theoretical_loss": 3.5655539626857036,
+ "tokens_seen": 1402601472
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046469007021063193,
+ "loss": 1.2459,
+ "theoretical_loss": 3.56553920311294,
+ "tokens_seen": 1402667008
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046467402206619863,
+ "loss": 1.2949,
+ "theoretical_loss": 3.5655244444228416,
+ "tokens_seen": 1402732544
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046465797392176533,
+ "loss": 1.3743,
+ "theoretical_loss": 3.5655096866153144,
+ "tokens_seen": 1402798080
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046464192577733203,
+ "loss": 1.3485,
+ "theoretical_loss": 3.5654949296902654,
+ "tokens_seen": 1402863616
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046462587763289873,
+ "loss": 1.3674,
+ "theoretical_loss": 3.5654801736475994,
+ "tokens_seen": 1402929152
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046460982948846543,
+ "loss": 1.2972,
+ "theoretical_loss": 3.5654654184872228,
+ "tokens_seen": 1402994688
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046459378134403213,
+ "loss": 1.3483,
+ "theoretical_loss": 3.5654506642090418,
+ "tokens_seen": 1403060224
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046457773319959883,
+ "loss": 1.2416,
+ "theoretical_loss": 3.565435910812962,
+ "tokens_seen": 1403125760
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046456168505516553,
+ "loss": 1.3794,
+ "theoretical_loss": 3.56542115829889,
+ "tokens_seen": 1403191296
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004645456369107322,
+ "loss": 1.3798,
+ "theoretical_loss": 3.5654064066667317,
+ "tokens_seen": 1403256832
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046452958876629894,
+ "loss": 1.3422,
+ "theoretical_loss": 3.565391655916393,
+ "tokens_seen": 1403322368
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004645135406218656,
+ "loss": 1.2489,
+ "theoretical_loss": 3.5653769060477805,
+ "tokens_seen": 1403387904
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046449749247743234,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5653621570608003,
+ "tokens_seen": 1403453440
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000464481444332999,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5653474089553576,
+ "tokens_seen": 1403518976
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046446539618856574,
+ "loss": 1.2985,
+ "theoretical_loss": 3.565332661731359,
+ "tokens_seen": 1403584512
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004644493480441324,
+ "loss": 1.2564,
+ "theoretical_loss": 3.565317915388712,
+ "tokens_seen": 1403650048
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046443329989969914,
+ "loss": 1.2824,
+ "theoretical_loss": 3.5653031699273203,
+ "tokens_seen": 1403715584
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004644172517552658,
+ "loss": 1.2888,
+ "theoretical_loss": 3.565288425347092,
+ "tokens_seen": 1403781120
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046440120361083254,
+ "loss": 1.2191,
+ "theoretical_loss": 3.5652736816479327,
+ "tokens_seen": 1403846656
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004643851554663992,
+ "loss": 1.3087,
+ "theoretical_loss": 3.5652589388297486,
+ "tokens_seen": 1403912192
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046436910732196594,
+ "loss": 1.2847,
+ "theoretical_loss": 3.565244196892446,
+ "tokens_seen": 1403977728
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004643530591775326,
+ "loss": 1.2152,
+ "theoretical_loss": 3.565229455835931,
+ "tokens_seen": 1404043264
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046433701103309934,
+ "loss": 1.2608,
+ "theoretical_loss": 3.565214715660111,
+ "tokens_seen": 1404108800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000464320962888666,
+ "loss": 1.3817,
+ "theoretical_loss": 3.56519997636489,
+ "tokens_seen": 1404174336
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046430491474423274,
+ "loss": 1.2184,
+ "theoretical_loss": 3.5651852379501765,
+ "tokens_seen": 1404239872
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004642888665997994,
+ "loss": 1.3529,
+ "theoretical_loss": 3.5651705004158756,
+ "tokens_seen": 1404305408
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046427281845536615,
+ "loss": 1.2785,
+ "theoretical_loss": 3.5651557637618936,
+ "tokens_seen": 1404370944
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004642567703109328,
+ "loss": 1.2324,
+ "theoretical_loss": 3.5651410279881377,
+ "tokens_seen": 1404436480
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046424072216649955,
+ "loss": 1.329,
+ "theoretical_loss": 3.5651262930945133,
+ "tokens_seen": 1404502016
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004642246740220662,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5651115590809273,
+ "tokens_seen": 1404567552
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046420862587763295,
+ "loss": 1.2996,
+ "theoretical_loss": 3.565096825947286,
+ "tokens_seen": 1404633088
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004641925777331996,
+ "loss": 1.297,
+ "theoretical_loss": 3.565082093693496,
+ "tokens_seen": 1404698624
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046417652958876635,
+ "loss": 1.3461,
+ "theoretical_loss": 3.5650673623194633,
+ "tokens_seen": 1404764160
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000464160481444333,
+ "loss": 1.366,
+ "theoretical_loss": 3.5650526318250946,
+ "tokens_seen": 1404829696
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046414443329989975,
+ "loss": 1.4017,
+ "theoretical_loss": 3.565037902210296,
+ "tokens_seen": 1404895232
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004641283851554664,
+ "loss": 1.1948,
+ "theoretical_loss": 3.5650231734749744,
+ "tokens_seen": 1404960768
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046411233701103315,
+ "loss": 1.3518,
+ "theoretical_loss": 3.565008445619036,
+ "tokens_seen": 1405026304
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004640962888665998,
+ "loss": 1.2568,
+ "theoretical_loss": 3.5649937186423877,
+ "tokens_seen": 1405091840
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046408024072216655,
+ "loss": 1.3657,
+ "theoretical_loss": 3.564978992544935,
+ "tokens_seen": 1405157376
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004640641925777332,
+ "loss": 1.4002,
+ "theoretical_loss": 3.5649642673265856,
+ "tokens_seen": 1405222912
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046404814443329996,
+ "loss": 1.2962,
+ "theoretical_loss": 3.564949542987245,
+ "tokens_seen": 1405288448
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004640320962888666,
+ "loss": 1.3722,
+ "theoretical_loss": 3.5649348195268207,
+ "tokens_seen": 1405353984
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046401604814443336,
+ "loss": 1.2351,
+ "theoretical_loss": 3.564920096945218,
+ "tokens_seen": 1405419520
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000464,
+ "loss": 1.328,
+ "theoretical_loss": 3.564905375242345,
+ "tokens_seen": 1405485056
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046398395185556676,
+ "loss": 1.3564,
+ "theoretical_loss": 3.5648906544181074,
+ "tokens_seen": 1405550592
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004639679037111334,
+ "loss": 1.2927,
+ "theoretical_loss": 3.5648759344724117,
+ "tokens_seen": 1405616128
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046395185556670016,
+ "loss": 1.2815,
+ "theoretical_loss": 3.5648612154051644,
+ "tokens_seen": 1405681664
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 471752,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.491056203842163,
+ "objective/train/theoretical_loss": 3.564846497216273,
+ "objective/train/tokens_used": 1426207200,
+ "theoretical_loss": 3.564846497216273,
+ "tokens_seen": 1405747200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004639358074222668,
+ "loss": 1.3665,
+ "theoretical_loss": 3.564846497216273,
+ "tokens_seen": 1405747200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004639197592778335,
+ "loss": 1.3182,
+ "theoretical_loss": 3.564831779905644,
+ "tokens_seen": 1405812736
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004639037111334002,
+ "loss": 1.3252,
+ "theoretical_loss": 3.564817063473183,
+ "tokens_seen": 1405878272
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004638876629889669,
+ "loss": 1.2465,
+ "theoretical_loss": 3.564802347918797,
+ "tokens_seen": 1405943808
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004638716148445336,
+ "loss": 1.3465,
+ "theoretical_loss": 3.5647876332423936,
+ "tokens_seen": 1406009344
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004638555667001003,
+ "loss": 1.2294,
+ "theoretical_loss": 3.564772919443879,
+ "tokens_seen": 1406074880
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000463839518555667,
+ "loss": 1.2853,
+ "theoretical_loss": 3.564758206523159,
+ "tokens_seen": 1406140416
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004638234704112337,
+ "loss": 1.3808,
+ "theoretical_loss": 3.5647434944801417,
+ "tokens_seen": 1406205952
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004638074222668004,
+ "loss": 1.2815,
+ "theoretical_loss": 3.564728783314733,
+ "tokens_seen": 1406271488
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004637913741223671,
+ "loss": 1.2508,
+ "theoretical_loss": 3.56471407302684,
+ "tokens_seen": 1406337024
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046377532597793376,
+ "loss": 1.3086,
+ "theoretical_loss": 3.56469936361637,
+ "tokens_seen": 1406402560
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004637592778335005,
+ "loss": 1.309,
+ "theoretical_loss": 3.5646846550832283,
+ "tokens_seen": 1406468096
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046374322968906716,
+ "loss": 1.3229,
+ "theoretical_loss": 3.5646699474273227,
+ "tokens_seen": 1406533632
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004637271815446339,
+ "loss": 1.2341,
+ "theoretical_loss": 3.5646552406485608,
+ "tokens_seen": 1406599168
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046371113340020056,
+ "loss": 1.3117,
+ "theoretical_loss": 3.5646405347468475,
+ "tokens_seen": 1406664704
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004636950852557673,
+ "loss": 1.3195,
+ "theoretical_loss": 3.5646258297220905,
+ "tokens_seen": 1406730240
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046367903711133396,
+ "loss": 1.3438,
+ "theoretical_loss": 3.5646111255741975,
+ "tokens_seen": 1406795776
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004636629889669007,
+ "loss": 1.2993,
+ "theoretical_loss": 3.564596422303074,
+ "tokens_seen": 1406861312
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046364694082246747,
+ "loss": 1.3483,
+ "theoretical_loss": 3.5645817199086283,
+ "tokens_seen": 1406926848
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004636308926780341,
+ "loss": 1.3506,
+ "theoretical_loss": 3.5645670183907665,
+ "tokens_seen": 1406992384
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046361484453360087,
+ "loss": 1.2594,
+ "theoretical_loss": 3.5645523177493947,
+ "tokens_seen": 1407057920
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635987963891675,
+ "loss": 1.2768,
+ "theoretical_loss": 3.5645376179844215,
+ "tokens_seen": 1407123456
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635827482447343,
+ "loss": 1.3181,
+ "theoretical_loss": 3.5645229190957526,
+ "tokens_seen": 1407188992
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635667001003009,
+ "loss": 1.3721,
+ "theoretical_loss": 3.5645082210832952,
+ "tokens_seen": 1407254528
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635506519558677,
+ "loss": 1.3782,
+ "theoretical_loss": 3.5644935239469566,
+ "tokens_seen": 1407320064
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635346038114343,
+ "loss": 1.2865,
+ "theoretical_loss": 3.564478827686644,
+ "tokens_seen": 1407385600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635185556670011,
+ "loss": 1.4146,
+ "theoretical_loss": 3.5644641323022634,
+ "tokens_seen": 1407451136
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004635025075225677,
+ "loss": 1.3308,
+ "theoretical_loss": 3.564449437793723,
+ "tokens_seen": 1407516672
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634864593781345,
+ "loss": 1.2313,
+ "theoretical_loss": 3.5644347441609288,
+ "tokens_seen": 1407582208
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634704112337011,
+ "loss": 1.2939,
+ "theoretical_loss": 3.564420051403788,
+ "tokens_seen": 1407647744
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634543630892679,
+ "loss": 1.3033,
+ "theoretical_loss": 3.5644053595222083,
+ "tokens_seen": 1407713280
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634383149448345,
+ "loss": 1.3209,
+ "theoretical_loss": 3.564390668516096,
+ "tokens_seen": 1407778816
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634222668004013,
+ "loss": 1.4386,
+ "theoretical_loss": 3.5643759783853595,
+ "tokens_seen": 1407844352
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004634062186559679,
+ "loss": 1.3385,
+ "theoretical_loss": 3.564361289129904,
+ "tokens_seen": 1407909888
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004633901705115347,
+ "loss": 1.3646,
+ "theoretical_loss": 3.564346600749638,
+ "tokens_seen": 1407975424
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046337412236710133,
+ "loss": 1.3764,
+ "theoretical_loss": 3.5643319132444686,
+ "tokens_seen": 1408040960
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004633580742226681,
+ "loss": 1.2875,
+ "theoretical_loss": 3.564317226614302,
+ "tokens_seen": 1408106496
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046334202607823473,
+ "loss": 1.1642,
+ "theoretical_loss": 3.564302540859046,
+ "tokens_seen": 1408172032
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004633259779338015,
+ "loss": 1.4003,
+ "theoretical_loss": 3.5642878559786073,
+ "tokens_seen": 1408237568
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046330992978936813,
+ "loss": 1.3716,
+ "theoretical_loss": 3.5642731719728937,
+ "tokens_seen": 1408303104
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046329388164493483,
+ "loss": 1.2402,
+ "theoretical_loss": 3.5642584888418125,
+ "tokens_seen": 1408368640
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046327783350050153,
+ "loss": 1.3014,
+ "theoretical_loss": 3.5642438065852704,
+ "tokens_seen": 1408434176
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046326178535606823,
+ "loss": 1.2789,
+ "theoretical_loss": 3.5642291252031746,
+ "tokens_seen": 1408499712
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046324573721163493,
+ "loss": 1.3755,
+ "theoretical_loss": 3.5642144446954322,
+ "tokens_seen": 1408565248
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046322968906720163,
+ "loss": 1.2905,
+ "theoretical_loss": 3.564199765061951,
+ "tokens_seen": 1408630784
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046321364092276833,
+ "loss": 1.4412,
+ "theoretical_loss": 3.564185086302638,
+ "tokens_seen": 1408696320
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046319759277833503,
+ "loss": 1.2412,
+ "theoretical_loss": 3.564170408417401,
+ "tokens_seen": 1408761856
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046318154463390174,
+ "loss": 1.2649,
+ "theoretical_loss": 3.564155731406146,
+ "tokens_seen": 1408827392
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046316549648946844,
+ "loss": 1.3384,
+ "theoretical_loss": 3.5641410552687813,
+ "tokens_seen": 1408892928
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004631494483450351,
+ "loss": 1.2401,
+ "theoretical_loss": 3.564126380005214,
+ "tokens_seen": 1408958464
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 473021,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3816249370574951,
+ "objective/train/theoretical_loss": 3.5641117056153515,
+ "objective/train/tokens_used": 1429484000,
+ "theoretical_loss": 3.5641117056153515,
+ "tokens_seen": 1409024000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046313340020060184,
+ "loss": 1.4074,
+ "theoretical_loss": 3.5641117056153515,
+ "tokens_seen": 1409024000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004631173520561685,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5640970320991014,
+ "tokens_seen": 1409089536
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046310130391173524,
+ "loss": 1.3182,
+ "theoretical_loss": 3.5640823594563704,
+ "tokens_seen": 1409155072
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004630852557673019,
+ "loss": 1.225,
+ "theoretical_loss": 3.564067687687066,
+ "tokens_seen": 1409220608
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046306920762286864,
+ "loss": 1.2488,
+ "theoretical_loss": 3.564053016791096,
+ "tokens_seen": 1409286144
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004630531594784353,
+ "loss": 1.3409,
+ "theoretical_loss": 3.564038346768368,
+ "tokens_seen": 1409351680
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046303711133400204,
+ "loss": 1.2559,
+ "theoretical_loss": 3.564023677618789,
+ "tokens_seen": 1409417216
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004630210631895687,
+ "loss": 1.3289,
+ "theoretical_loss": 3.564009009342266,
+ "tokens_seen": 1409482752
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046300501504513544,
+ "loss": 1.2364,
+ "theoretical_loss": 3.5639943419387072,
+ "tokens_seen": 1409548288
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004629889669007021,
+ "loss": 1.3942,
+ "theoretical_loss": 3.56397967540802,
+ "tokens_seen": 1409613824
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046297291875626884,
+ "loss": 1.4127,
+ "theoretical_loss": 3.5639650097501114,
+ "tokens_seen": 1409679360
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004629568706118355,
+ "loss": 1.3688,
+ "theoretical_loss": 3.5639503449648893,
+ "tokens_seen": 1409744896
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046294082246740224,
+ "loss": 1.3236,
+ "theoretical_loss": 3.5639356810522607,
+ "tokens_seen": 1409810432
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004629247743229689,
+ "loss": 1.1882,
+ "theoretical_loss": 3.5639210180121337,
+ "tokens_seen": 1409875968
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046290872617853565,
+ "loss": 1.2427,
+ "theoretical_loss": 3.5639063558444155,
+ "tokens_seen": 1409941504
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004628926780341023,
+ "loss": 1.364,
+ "theoretical_loss": 3.563891694549014,
+ "tokens_seen": 1410007040
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046287662988966905,
+ "loss": 1.3528,
+ "theoretical_loss": 3.563877034125837,
+ "tokens_seen": 1410072576
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004628605817452357,
+ "loss": 1.3439,
+ "theoretical_loss": 3.563862374574791,
+ "tokens_seen": 1410138112
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046284453360080245,
+ "loss": 1.3109,
+ "theoretical_loss": 3.5638477158957844,
+ "tokens_seen": 1410203648
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004628284854563691,
+ "loss": 1.3087,
+ "theoretical_loss": 3.5638330580887243,
+ "tokens_seen": 1410269184
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046281243731193585,
+ "loss": 1.4207,
+ "theoretical_loss": 3.5638184011535188,
+ "tokens_seen": 1410334720
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004627963891675025,
+ "loss": 1.3335,
+ "theoretical_loss": 3.563803745090075,
+ "tokens_seen": 1410400256
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046278034102306925,
+ "loss": 1.328,
+ "theoretical_loss": 3.563789089898301,
+ "tokens_seen": 1410465792
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004627642928786359,
+ "loss": 1.2656,
+ "theoretical_loss": 3.5637744355781047,
+ "tokens_seen": 1410531328
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046274824473420265,
+ "loss": 1.3268,
+ "theoretical_loss": 3.5637597821293934,
+ "tokens_seen": 1410596864
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004627321965897693,
+ "loss": 1.1607,
+ "theoretical_loss": 3.563745129552075,
+ "tokens_seen": 1410662400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046271614844533605,
+ "loss": 1.3328,
+ "theoretical_loss": 3.5637304778460566,
+ "tokens_seen": 1410727936
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004627001003009027,
+ "loss": 1.3885,
+ "theoretical_loss": 3.5637158270112463,
+ "tokens_seen": 1410793472
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046268405215646945,
+ "loss": 1.2589,
+ "theoretical_loss": 3.5637011770475517,
+ "tokens_seen": 1410859008
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004626680040120361,
+ "loss": 1.3258,
+ "theoretical_loss": 3.5636865279548813,
+ "tokens_seen": 1410924544
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046265195586760286,
+ "loss": 1.3359,
+ "theoretical_loss": 3.5636718797331417,
+ "tokens_seen": 1410990080
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004626359077231695,
+ "loss": 1.2994,
+ "theoretical_loss": 3.5636572323822415,
+ "tokens_seen": 1411055616
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046261985957873626,
+ "loss": 1.4495,
+ "theoretical_loss": 3.5636425859020884,
+ "tokens_seen": 1411121152
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004626038114343029,
+ "loss": 1.3747,
+ "theoretical_loss": 3.56362794029259,
+ "tokens_seen": 1411186688
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046258776328986966,
+ "loss": 1.2321,
+ "theoretical_loss": 3.5636132955536537,
+ "tokens_seen": 1411252224
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004625717151454363,
+ "loss": 1.3645,
+ "theoretical_loss": 3.5635986516851874,
+ "tokens_seen": 1411317760
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046255566700100306,
+ "loss": 1.301,
+ "theoretical_loss": 3.5635840086871005,
+ "tokens_seen": 1411383296
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004625396188565697,
+ "loss": 1.3656,
+ "theoretical_loss": 3.5635693665592987,
+ "tokens_seen": 1411448832
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004625235707121364,
+ "loss": 1.3231,
+ "theoretical_loss": 3.5635547253016915,
+ "tokens_seen": 1411514368
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004625075225677031,
+ "loss": 1.3584,
+ "theoretical_loss": 3.5635400849141856,
+ "tokens_seen": 1411579904
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004624914744232698,
+ "loss": 1.2762,
+ "theoretical_loss": 3.5635254453966896,
+ "tokens_seen": 1411645440
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004624754262788365,
+ "loss": 1.4195,
+ "theoretical_loss": 3.563510806749111,
+ "tokens_seen": 1411710976
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004624593781344032,
+ "loss": 1.3402,
+ "theoretical_loss": 3.5634961689713585,
+ "tokens_seen": 1411776512
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046244332998996996,
+ "loss": 1.1941,
+ "theoretical_loss": 3.563481532063339,
+ "tokens_seen": 1411842048
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004624272818455366,
+ "loss": 1.2568,
+ "theoretical_loss": 3.563466896024961,
+ "tokens_seen": 1411907584
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046241123370110337,
+ "loss": 1.2525,
+ "theoretical_loss": 3.563452260856133,
+ "tokens_seen": 1411973120
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046239518555667,
+ "loss": 1.3847,
+ "theoretical_loss": 3.5634376265567616,
+ "tokens_seen": 1412038656
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046237913741223677,
+ "loss": 1.3101,
+ "theoretical_loss": 3.563422993126756,
+ "tokens_seen": 1412104192
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004623630892678034,
+ "loss": 1.3066,
+ "theoretical_loss": 3.5634083605660236,
+ "tokens_seen": 1412169728
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046234704112337017,
+ "loss": 1.3575,
+ "theoretical_loss": 3.563393728874473,
+ "tokens_seen": 1412235264
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 474225,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4415783882141113,
+ "objective/train/theoretical_loss": 3.5633790980520113,
+ "objective/train/tokens_used": 1432760800,
+ "theoretical_loss": 3.5633790980520113,
+ "tokens_seen": 1412300800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004623309929789368,
+ "loss": 1.4305,
+ "theoretical_loss": 3.5633790980520113,
+ "tokens_seen": 1412300800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046231494483450357,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5633644680985475,
+ "tokens_seen": 1412366336
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004622988966900702,
+ "loss": 1.2806,
+ "theoretical_loss": 3.5633498390139895,
+ "tokens_seen": 1412431872
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046228284854563697,
+ "loss": 1.2758,
+ "theoretical_loss": 3.563335210798245,
+ "tokens_seen": 1412497408
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004622668004012036,
+ "loss": 1.2755,
+ "theoretical_loss": 3.563320583451222,
+ "tokens_seen": 1412562944
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046225075225677037,
+ "loss": 1.3708,
+ "theoretical_loss": 3.5633059569728287,
+ "tokens_seen": 1412628480
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000462234704112337,
+ "loss": 1.2688,
+ "theoretical_loss": 3.563291331362974,
+ "tokens_seen": 1412694016
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004622186559679038,
+ "loss": 1.2986,
+ "theoretical_loss": 3.563276706621565,
+ "tokens_seen": 1412759552
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004622026078234704,
+ "loss": 1.1771,
+ "theoretical_loss": 3.563262082748511,
+ "tokens_seen": 1412825088
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004621865596790372,
+ "loss": 1.2169,
+ "theoretical_loss": 3.5632474597437183,
+ "tokens_seen": 1412890624
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004621705115346038,
+ "loss": 1.2954,
+ "theoretical_loss": 3.5632328376070967,
+ "tokens_seen": 1412956160
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004621544633901706,
+ "loss": 1.4088,
+ "theoretical_loss": 3.563218216338554,
+ "tokens_seen": 1413021696
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004621384152457372,
+ "loss": 1.3772,
+ "theoretical_loss": 3.563203595937999,
+ "tokens_seen": 1413087232
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000462122367101304,
+ "loss": 1.3154,
+ "theoretical_loss": 3.5631889764053386,
+ "tokens_seen": 1413152768
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004621063189568706,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5631743577404817,
+ "tokens_seen": 1413218304
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004620902708124374,
+ "loss": 1.2884,
+ "theoretical_loss": 3.5631597399433366,
+ "tokens_seen": 1413283840
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000462074222668004,
+ "loss": 1.2522,
+ "theoretical_loss": 3.563145123013811,
+ "tokens_seen": 1413349376
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004620581745235708,
+ "loss": 1.2835,
+ "theoretical_loss": 3.563130506951814,
+ "tokens_seen": 1413414912
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004620421263791374,
+ "loss": 1.2298,
+ "theoretical_loss": 3.5631158917572536,
+ "tokens_seen": 1413480448
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004620260782347042,
+ "loss": 1.2495,
+ "theoretical_loss": 3.5631012774300386,
+ "tokens_seen": 1413545984
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046201003009027083,
+ "loss": 1.2452,
+ "theoretical_loss": 3.563086663970076,
+ "tokens_seen": 1413611520
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004619939819458376,
+ "loss": 1.351,
+ "theoretical_loss": 3.5630720513772753,
+ "tokens_seen": 1413677056
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046197793380140423,
+ "loss": 1.2684,
+ "theoretical_loss": 3.563057439651544,
+ "tokens_seen": 1413742592
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000461961885656971,
+ "loss": 1.2259,
+ "theoretical_loss": 3.5630428287927915,
+ "tokens_seen": 1413808128
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046194583751253763,
+ "loss": 1.2958,
+ "theoretical_loss": 3.5630282188009255,
+ "tokens_seen": 1413873664
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004619297893681044,
+ "loss": 1.2495,
+ "theoretical_loss": 3.5630136096758545,
+ "tokens_seen": 1413939200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046191374122367103,
+ "loss": 1.2337,
+ "theoretical_loss": 3.562999001417486,
+ "tokens_seen": 1414004736
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004618976930792378,
+ "loss": 1.4248,
+ "theoretical_loss": 3.5629843940257304,
+ "tokens_seen": 1414070272
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046188164493480443,
+ "loss": 1.3784,
+ "theoretical_loss": 3.562969787500495,
+ "tokens_seen": 1414135808
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046186559679037113,
+ "loss": 1.2026,
+ "theoretical_loss": 3.5629551818416876,
+ "tokens_seen": 1414201344
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046184954864593783,
+ "loss": 1.2322,
+ "theoretical_loss": 3.5629405770492175,
+ "tokens_seen": 1414266880
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046183350050150453,
+ "loss": 1.2609,
+ "theoretical_loss": 3.562925973122993,
+ "tokens_seen": 1414332416
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046181745235707123,
+ "loss": 1.3234,
+ "theoretical_loss": 3.562911370062923,
+ "tokens_seen": 1414397952
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046180140421263794,
+ "loss": 1.2756,
+ "theoretical_loss": 3.5628967678689154,
+ "tokens_seen": 1414463488
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046178535606820464,
+ "loss": 1.2495,
+ "theoretical_loss": 3.5628821665408785,
+ "tokens_seen": 1414529024
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046176930792377134,
+ "loss": 1.21,
+ "theoretical_loss": 3.5628675660787215,
+ "tokens_seen": 1414594560
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000461753259779338,
+ "loss": 1.2475,
+ "theoretical_loss": 3.5628529664823527,
+ "tokens_seen": 1414660096
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046173721163490474,
+ "loss": 1.2722,
+ "theoretical_loss": 3.562838367751681,
+ "tokens_seen": 1414725632
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004617211634904714,
+ "loss": 1.293,
+ "theoretical_loss": 3.562823769886614,
+ "tokens_seen": 1414791168
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046170511534603814,
+ "loss": 1.3017,
+ "theoretical_loss": 3.562809172887061,
+ "tokens_seen": 1414856704
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004616890672016048,
+ "loss": 1.3139,
+ "theoretical_loss": 3.56279457675293,
+ "tokens_seen": 1414922240
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046167301905717154,
+ "loss": 1.4239,
+ "theoretical_loss": 3.5627799814841308,
+ "tokens_seen": 1414987776
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004616569709127382,
+ "loss": 1.2464,
+ "theoretical_loss": 3.562765387080571,
+ "tokens_seen": 1415053312
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046164092276830494,
+ "loss": 1.3636,
+ "theoretical_loss": 3.5627507935421594,
+ "tokens_seen": 1415118848
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004616248746238716,
+ "loss": 1.3659,
+ "theoretical_loss": 3.5627362008688053,
+ "tokens_seen": 1415184384
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046160882647943834,
+ "loss": 1.3007,
+ "theoretical_loss": 3.562721609060416,
+ "tokens_seen": 1415249920
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000461592778335005,
+ "loss": 1.3657,
+ "theoretical_loss": 3.562707018116902,
+ "tokens_seen": 1415315456
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046157673019057174,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5626924280381704,
+ "tokens_seen": 1415380992
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004615606820461384,
+ "loss": 1.2507,
+ "theoretical_loss": 3.562677838824131,
+ "tokens_seen": 1415446528
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046154463390170515,
+ "loss": 1.2592,
+ "theoretical_loss": 3.5626632504746913,
+ "tokens_seen": 1415512064
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 475088,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1855875253677368,
+ "objective/train/theoretical_loss": 3.5626486629897616,
+ "objective/train/tokens_used": 1436037600,
+ "theoretical_loss": 3.5626486629897616,
+ "tokens_seen": 1415577600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004615285857572718,
+ "loss": 1.2318,
+ "theoretical_loss": 3.5626486629897616,
+ "tokens_seen": 1415577600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046151253761283855,
+ "loss": 1.283,
+ "theoretical_loss": 3.5626340763692497,
+ "tokens_seen": 1415643136
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004614964894684052,
+ "loss": 1.1989,
+ "theoretical_loss": 3.562619490613064,
+ "tokens_seen": 1415708672
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046148044132397195,
+ "loss": 1.2893,
+ "theoretical_loss": 3.5626049057211144,
+ "tokens_seen": 1415774208
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004614643931795386,
+ "loss": 1.2727,
+ "theoretical_loss": 3.5625903216933086,
+ "tokens_seen": 1415839744
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046144834503510535,
+ "loss": 1.154,
+ "theoretical_loss": 3.562575738529556,
+ "tokens_seen": 1415905280
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000461432296890672,
+ "loss": 1.3396,
+ "theoretical_loss": 3.5625611562297657,
+ "tokens_seen": 1415970816
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046141624874623875,
+ "loss": 1.3941,
+ "theoretical_loss": 3.562546574793845,
+ "tokens_seen": 1416036352
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004614002006018054,
+ "loss": 1.3347,
+ "theoretical_loss": 3.562531994221705,
+ "tokens_seen": 1416101888
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046138415245737215,
+ "loss": 1.2259,
+ "theoretical_loss": 3.5625174145132528,
+ "tokens_seen": 1416167424
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004613681043129388,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5625028356683983,
+ "tokens_seen": 1416232960
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046135205616850555,
+ "loss": 1.3344,
+ "theoretical_loss": 3.5624882576870496,
+ "tokens_seen": 1416298496
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004613360080240722,
+ "loss": 1.5102,
+ "theoretical_loss": 3.5624736805691164,
+ "tokens_seen": 1416364032
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046131995987963895,
+ "loss": 1.2662,
+ "theoretical_loss": 3.562459104314507,
+ "tokens_seen": 1416429568
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004613039117352056,
+ "loss": 1.3415,
+ "theoretical_loss": 3.562444528923131,
+ "tokens_seen": 1416495104
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046128786359077236,
+ "loss": 1.3411,
+ "theoretical_loss": 3.5624299543948963,
+ "tokens_seen": 1416560640
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000461271815446339,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5624153807297123,
+ "tokens_seen": 1416626176
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046125576730190576,
+ "loss": 1.3979,
+ "theoretical_loss": 3.5624008079274887,
+ "tokens_seen": 1416691712
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046123971915747246,
+ "loss": 1.2643,
+ "theoretical_loss": 3.5623862359881335,
+ "tokens_seen": 1416757248
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046122367101303916,
+ "loss": 1.3072,
+ "theoretical_loss": 3.5623716649115558,
+ "tokens_seen": 1416822784
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046120762286860586,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5623570946976657,
+ "tokens_seen": 1416888320
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046119157472417256,
+ "loss": 1.3149,
+ "theoretical_loss": 3.562342525346371,
+ "tokens_seen": 1416953856
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046117552657973926,
+ "loss": 1.3099,
+ "theoretical_loss": 3.562327956857581,
+ "tokens_seen": 1417019392
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046115947843530596,
+ "loss": 1.2656,
+ "theoretical_loss": 3.562313389231205,
+ "tokens_seen": 1417084928
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046114343029087266,
+ "loss": 1.1917,
+ "theoretical_loss": 3.5622988224671523,
+ "tokens_seen": 1417150464
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046112738214643936,
+ "loss": 1.3449,
+ "theoretical_loss": 3.5622842565653317,
+ "tokens_seen": 1417216000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046111133400200606,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5622696915256515,
+ "tokens_seen": 1417281536
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004610952858575727,
+ "loss": 1.4002,
+ "theoretical_loss": 3.5622551273480223,
+ "tokens_seen": 1417347072
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046107923771313946,
+ "loss": 1.3465,
+ "theoretical_loss": 3.562240564032352,
+ "tokens_seen": 1417412608
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004610631895687061,
+ "loss": 1.4118,
+ "theoretical_loss": 3.5622260015785505,
+ "tokens_seen": 1417478144
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046104714142427287,
+ "loss": 1.3078,
+ "theoretical_loss": 3.562211439986527,
+ "tokens_seen": 1417543680
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004610310932798395,
+ "loss": 1.34,
+ "theoretical_loss": 3.5621968792561898,
+ "tokens_seen": 1417609216
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046101504513540627,
+ "loss": 1.2498,
+ "theoretical_loss": 3.562182319387449,
+ "tokens_seen": 1417674752
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004609989969909729,
+ "loss": 1.3904,
+ "theoretical_loss": 3.5621677603802127,
+ "tokens_seen": 1417740288
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046098294884653967,
+ "loss": 1.3391,
+ "theoretical_loss": 3.5621532022343914,
+ "tokens_seen": 1417805824
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004609669007021063,
+ "loss": 1.3944,
+ "theoretical_loss": 3.5621386449498935,
+ "tokens_seen": 1417871360
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046095085255767307,
+ "loss": 1.25,
+ "theoretical_loss": 3.5621240885266285,
+ "tokens_seen": 1417936896
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004609348044132397,
+ "loss": 1.1472,
+ "theoretical_loss": 3.562109532964506,
+ "tokens_seen": 1418002432
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046091875626880647,
+ "loss": 1.2777,
+ "theoretical_loss": 3.562094978263435,
+ "tokens_seen": 1418067968
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004609027081243731,
+ "loss": 1.2624,
+ "theoretical_loss": 3.562080424423324,
+ "tokens_seen": 1418133504
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046088665997993987,
+ "loss": 1.3042,
+ "theoretical_loss": 3.5620658714440827,
+ "tokens_seen": 1418199040
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004608706118355065,
+ "loss": 1.214,
+ "theoretical_loss": 3.5620513193256214,
+ "tokens_seen": 1418264576
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046085456369107327,
+ "loss": 1.1787,
+ "theoretical_loss": 3.5620367680678484,
+ "tokens_seen": 1418330112
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004608385155466399,
+ "loss": 1.329,
+ "theoretical_loss": 3.562022217670673,
+ "tokens_seen": 1418395648
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004608224674022067,
+ "loss": 1.2944,
+ "theoretical_loss": 3.5620076681340054,
+ "tokens_seen": 1418461184
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004608064192577733,
+ "loss": 1.2893,
+ "theoretical_loss": 3.5619931194577537,
+ "tokens_seen": 1418526720
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607903711133401,
+ "loss": 1.3621,
+ "theoretical_loss": 3.561978571641828,
+ "tokens_seen": 1418592256
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607743229689067,
+ "loss": 1.2604,
+ "theoretical_loss": 3.561964024686138,
+ "tokens_seen": 1418657792
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607582748244735,
+ "loss": 1.2343,
+ "theoretical_loss": 3.5619494785905923,
+ "tokens_seen": 1418723328
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607422266800401,
+ "loss": 1.4144,
+ "theoretical_loss": 3.5619349333551016,
+ "tokens_seen": 1418788864
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 476156,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3921765089035034,
+ "objective/train/theoretical_loss": 3.5619203889795736,
+ "objective/train/tokens_used": 1439314400,
+ "theoretical_loss": 3.5619203889795736,
+ "tokens_seen": 1418854400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607261785356069,
+ "loss": 1.3558,
+ "theoretical_loss": 3.5619203889795736,
+ "tokens_seen": 1418854400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004607101303911735,
+ "loss": 1.3041,
+ "theoretical_loss": 3.5619058454639188,
+ "tokens_seen": 1418919936
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004606940822467403,
+ "loss": 1.3763,
+ "theoretical_loss": 3.5618913028080463,
+ "tokens_seen": 1418985472
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004606780341023069,
+ "loss": 1.3323,
+ "theoretical_loss": 3.561876761011866,
+ "tokens_seen": 1419051008
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004606619859578737,
+ "loss": 1.3684,
+ "theoretical_loss": 3.561862220075287,
+ "tokens_seen": 1419116544
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004606459378134403,
+ "loss": 1.1713,
+ "theoretical_loss": 3.561847679998219,
+ "tokens_seen": 1419182080
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004606298896690071,
+ "loss": 1.1765,
+ "theoretical_loss": 3.5618331407805712,
+ "tokens_seen": 1419247616
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046061384152457373,
+ "loss": 1.3827,
+ "theoretical_loss": 3.5618186024222536,
+ "tokens_seen": 1419313152
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004605977933801405,
+ "loss": 1.2884,
+ "theoretical_loss": 3.561804064923175,
+ "tokens_seen": 1419378688
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046058174523570713,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5617895282832457,
+ "tokens_seen": 1419444224
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004605656970912739,
+ "loss": 1.3333,
+ "theoretical_loss": 3.561774992502375,
+ "tokens_seen": 1419509760
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046054964894684053,
+ "loss": 1.286,
+ "theoretical_loss": 3.5617604575804727,
+ "tokens_seen": 1419575296
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004605336008024073,
+ "loss": 1.2231,
+ "theoretical_loss": 3.5617459235174476,
+ "tokens_seen": 1419640832
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046051755265797393,
+ "loss": 1.2878,
+ "theoretical_loss": 3.56173139031321,
+ "tokens_seen": 1419706368
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004605015045135407,
+ "loss": 1.2333,
+ "theoretical_loss": 3.561716857967669,
+ "tokens_seen": 1419771904
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046048545636910733,
+ "loss": 1.1826,
+ "theoretical_loss": 3.5617023264807353,
+ "tokens_seen": 1419837440
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046046940822467403,
+ "loss": 1.3458,
+ "theoretical_loss": 3.5616877958523174,
+ "tokens_seen": 1419902976
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046045336008024073,
+ "loss": 1.261,
+ "theoretical_loss": 3.561673266082325,
+ "tokens_seen": 1419968512
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046043731193580744,
+ "loss": 1.3043,
+ "theoretical_loss": 3.561658737170669,
+ "tokens_seen": 1420034048
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046042126379137414,
+ "loss": 1.2932,
+ "theoretical_loss": 3.5616442091172575,
+ "tokens_seen": 1420099584
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046040521564694084,
+ "loss": 1.2602,
+ "theoretical_loss": 3.5616296819220015,
+ "tokens_seen": 1420165120
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046038916750250754,
+ "loss": 1.2364,
+ "theoretical_loss": 3.5616151555848097,
+ "tokens_seen": 1420230656
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046037311935807424,
+ "loss": 1.3497,
+ "theoretical_loss": 3.561600630105592,
+ "tokens_seen": 1420296192
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046035707121364094,
+ "loss": 1.3755,
+ "theoretical_loss": 3.561586105484259,
+ "tokens_seen": 1420361728
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046034102306920764,
+ "loss": 1.2862,
+ "theoretical_loss": 3.56157158172072,
+ "tokens_seen": 1420427264
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004603249749247743,
+ "loss": 1.331,
+ "theoretical_loss": 3.561557058814884,
+ "tokens_seen": 1420492800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046030892678034104,
+ "loss": 1.2845,
+ "theoretical_loss": 3.5615425367666615,
+ "tokens_seen": 1420558336
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004602928786359077,
+ "loss": 1.2585,
+ "theoretical_loss": 3.5615280155759628,
+ "tokens_seen": 1420623872
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046027683049147444,
+ "loss": 1.3023,
+ "theoretical_loss": 3.5615134952426963,
+ "tokens_seen": 1420689408
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004602607823470411,
+ "loss": 1.294,
+ "theoretical_loss": 3.5614989757667734,
+ "tokens_seen": 1420754944
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046024473420260784,
+ "loss": 1.2831,
+ "theoretical_loss": 3.5614844571481026,
+ "tokens_seen": 1420820480
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004602286860581745,
+ "loss": 1.2303,
+ "theoretical_loss": 3.561469939386595,
+ "tokens_seen": 1420886016
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046021263791374124,
+ "loss": 1.3566,
+ "theoretical_loss": 3.561455422482159,
+ "tokens_seen": 1420951552
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004601965897693079,
+ "loss": 1.2763,
+ "theoretical_loss": 3.5614409064347057,
+ "tokens_seen": 1421017088
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046018054162487465,
+ "loss": 1.2519,
+ "theoretical_loss": 3.561426391244144,
+ "tokens_seen": 1421082624
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004601644934804413,
+ "loss": 1.2943,
+ "theoretical_loss": 3.561411876910385,
+ "tokens_seen": 1421148160
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046014844533600805,
+ "loss": 1.1506,
+ "theoretical_loss": 3.561397363433338,
+ "tokens_seen": 1421213696
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004601323971915747,
+ "loss": 1.4133,
+ "theoretical_loss": 3.5613828508129126,
+ "tokens_seen": 1421279232
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046011634904714145,
+ "loss": 1.337,
+ "theoretical_loss": 3.561368339049019,
+ "tokens_seen": 1421344768
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004601003009027081,
+ "loss": 1.3713,
+ "theoretical_loss": 3.5613538281415673,
+ "tokens_seen": 1421410304
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046008425275827485,
+ "loss": 1.3428,
+ "theoretical_loss": 3.561339318090467,
+ "tokens_seen": 1421475840
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004600682046138415,
+ "loss": 1.2585,
+ "theoretical_loss": 3.5613248088956286,
+ "tokens_seen": 1421541376
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046005215646940825,
+ "loss": 1.2722,
+ "theoretical_loss": 3.561310300556962,
+ "tokens_seen": 1421606912
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000460036108324975,
+ "loss": 1.1406,
+ "theoretical_loss": 3.5612957930743776,
+ "tokens_seen": 1421672448
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00046002006018054165,
+ "loss": 1.4369,
+ "theoretical_loss": 3.5612812864477847,
+ "tokens_seen": 1421737984
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004600040120361084,
+ "loss": 1.3137,
+ "theoretical_loss": 3.5612667806770935,
+ "tokens_seen": 1421803520
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045998796389167505,
+ "loss": 1.3503,
+ "theoretical_loss": 3.5612522757622145,
+ "tokens_seen": 1421869056
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004599719157472418,
+ "loss": 1.2199,
+ "theoretical_loss": 3.561237771703057,
+ "tokens_seen": 1421934592
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045995586760280845,
+ "loss": 1.3518,
+ "theoretical_loss": 3.561223268499532,
+ "tokens_seen": 1422000128
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004599398194583752,
+ "loss": 1.3655,
+ "theoretical_loss": 3.561208766151549,
+ "tokens_seen": 1422065664
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 477084,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3397164344787598,
+ "objective/train/theoretical_loss": 3.561194264659018,
+ "objective/train/tokens_used": 1442591200,
+ "theoretical_loss": 3.561194264659018,
+ "tokens_seen": 1422131200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045992377131394186,
+ "loss": 1.368,
+ "theoretical_loss": 3.561194264659018,
+ "tokens_seen": 1422131200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004599077231695086,
+ "loss": 1.342,
+ "theoretical_loss": 3.5611797640218494,
+ "tokens_seen": 1422196736
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045989167502507526,
+ "loss": 1.373,
+ "theoretical_loss": 3.5611652642399534,
+ "tokens_seen": 1422262272
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000459875626880642,
+ "loss": 1.3054,
+ "theoretical_loss": 3.56115076531324,
+ "tokens_seen": 1422327808
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045985957873620866,
+ "loss": 1.2597,
+ "theoretical_loss": 3.5611362672416194,
+ "tokens_seen": 1422393344
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045984353059177536,
+ "loss": 1.275,
+ "theoretical_loss": 3.5611217700250015,
+ "tokens_seen": 1422458880
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045982748244734206,
+ "loss": 1.3644,
+ "theoretical_loss": 3.561107273663297,
+ "tokens_seen": 1422524416
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045981143430290876,
+ "loss": 1.2652,
+ "theoretical_loss": 3.561092778156416,
+ "tokens_seen": 1422589952
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045979538615847546,
+ "loss": 1.3335,
+ "theoretical_loss": 3.5610782835042682,
+ "tokens_seen": 1422655488
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045977933801404216,
+ "loss": 1.2909,
+ "theoretical_loss": 3.5610637897067647,
+ "tokens_seen": 1422721024
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045976328986960886,
+ "loss": 1.2856,
+ "theoretical_loss": 3.561049296763815,
+ "tokens_seen": 1422786560
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045974724172517556,
+ "loss": 1.3185,
+ "theoretical_loss": 3.5610348046753293,
+ "tokens_seen": 1422852096
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045973119358074226,
+ "loss": 1.1983,
+ "theoretical_loss": 3.561020313441219,
+ "tokens_seen": 1422917632
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045971514543630896,
+ "loss": 1.2979,
+ "theoretical_loss": 3.561005823061393,
+ "tokens_seen": 1422983168
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004596990972918756,
+ "loss": 1.3143,
+ "theoretical_loss": 3.560991333535762,
+ "tokens_seen": 1423048704
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045968304914744236,
+ "loss": 1.335,
+ "theoretical_loss": 3.5609768448642365,
+ "tokens_seen": 1423114240
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000459667001003009,
+ "loss": 1.3229,
+ "theoretical_loss": 3.5609623570467273,
+ "tokens_seen": 1423179776
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045965095285857577,
+ "loss": 1.2293,
+ "theoretical_loss": 3.560947870083144,
+ "tokens_seen": 1423245312
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004596349047141424,
+ "loss": 1.342,
+ "theoretical_loss": 3.5609333839733965,
+ "tokens_seen": 1423310848
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045961885656970917,
+ "loss": 1.3353,
+ "theoretical_loss": 3.5609188987173965,
+ "tokens_seen": 1423376384
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004596028084252758,
+ "loss": 1.3138,
+ "theoretical_loss": 3.560904414315054,
+ "tokens_seen": 1423441920
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045958676028084257,
+ "loss": 1.3179,
+ "theoretical_loss": 3.5608899307662787,
+ "tokens_seen": 1423507456
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004595707121364092,
+ "loss": 1.2747,
+ "theoretical_loss": 3.5608754480709814,
+ "tokens_seen": 1423572992
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045955466399197597,
+ "loss": 1.398,
+ "theoretical_loss": 3.5608609662290727,
+ "tokens_seen": 1423638528
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004595386158475426,
+ "loss": 1.3593,
+ "theoretical_loss": 3.5608464852404627,
+ "tokens_seen": 1423704064
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045952256770310937,
+ "loss": 1.3231,
+ "theoretical_loss": 3.5608320051050626,
+ "tokens_seen": 1423769600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000459506519558676,
+ "loss": 1.3361,
+ "theoretical_loss": 3.560817525822782,
+ "tokens_seen": 1423835136
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045949047141424277,
+ "loss": 1.2657,
+ "theoretical_loss": 3.5608030473935313,
+ "tokens_seen": 1423900672
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004594744232698094,
+ "loss": 1.36,
+ "theoretical_loss": 3.5607885698172215,
+ "tokens_seen": 1423966208
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004594583751253762,
+ "loss": 1.2496,
+ "theoretical_loss": 3.560774093093763,
+ "tokens_seen": 1424031744
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004594423269809428,
+ "loss": 1.3415,
+ "theoretical_loss": 3.5607596172230664,
+ "tokens_seen": 1424097280
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004594262788365096,
+ "loss": 1.3427,
+ "theoretical_loss": 3.560745142205042,
+ "tokens_seen": 1424162816
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004594102306920762,
+ "loss": 1.2926,
+ "theoretical_loss": 3.5607306680396,
+ "tokens_seen": 1424228352
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000459394182547643,
+ "loss": 1.334,
+ "theoretical_loss": 3.560716194726652,
+ "tokens_seen": 1424293888
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004593781344032096,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5607017222661073,
+ "tokens_seen": 1424359424
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004593620862587764,
+ "loss": 1.2977,
+ "theoretical_loss": 3.560687250657878,
+ "tokens_seen": 1424424960
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000459346038114343,
+ "loss": 1.2586,
+ "theoretical_loss": 3.560672779901873,
+ "tokens_seen": 1424490496
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004593299899699098,
+ "loss": 1.335,
+ "theoretical_loss": 3.5606583099980043,
+ "tokens_seen": 1424556032
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004593139418254764,
+ "loss": 1.4184,
+ "theoretical_loss": 3.560643840946182,
+ "tokens_seen": 1424621568
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004592978936810432,
+ "loss": 1.3255,
+ "theoretical_loss": 3.560629372746316,
+ "tokens_seen": 1424687104
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004592818455366098,
+ "loss": 1.3755,
+ "theoretical_loss": 3.5606149053983183,
+ "tokens_seen": 1424752640
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004592657973921766,
+ "loss": 1.3917,
+ "theoretical_loss": 3.5606004389020987,
+ "tokens_seen": 1424818176
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045924974924774323,
+ "loss": 1.3441,
+ "theoretical_loss": 3.560585973257568,
+ "tokens_seen": 1424883712
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045923370110331,
+ "loss": 1.1777,
+ "theoretical_loss": 3.560571508464637,
+ "tokens_seen": 1424949248
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045921765295887663,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5605570445232164,
+ "tokens_seen": 1425014784
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004592016048144434,
+ "loss": 1.3868,
+ "theoretical_loss": 3.560542581433217,
+ "tokens_seen": 1425080320
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045918555667001003,
+ "loss": 1.2555,
+ "theoretical_loss": 3.560528119194549,
+ "tokens_seen": 1425145856
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004591695085255768,
+ "loss": 1.3703,
+ "theoretical_loss": 3.5605136578071237,
+ "tokens_seen": 1425211392
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045915346038114343,
+ "loss": 1.256,
+ "theoretical_loss": 3.560499197270852,
+ "tokens_seen": 1425276928
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004591374122367102,
+ "loss": 1.2993,
+ "theoretical_loss": 3.5604847375856448,
+ "tokens_seen": 1425342464
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 478424,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3589322566986084,
+ "objective/train/theoretical_loss": 3.5604702787514118,
+ "objective/train/tokens_used": 1445868000,
+ "theoretical_loss": 3.5604702787514118,
+ "tokens_seen": 1425408000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045912136409227683,
+ "loss": 1.2625,
+ "theoretical_loss": 3.5604702787514118,
+ "tokens_seen": 1425408000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004591053159478436,
+ "loss": 1.322,
+ "theoretical_loss": 3.5604558207680643,
+ "tokens_seen": 1425473536
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045908926780341023,
+ "loss": 1.3262,
+ "theoretical_loss": 3.5604413636355137,
+ "tokens_seen": 1425539072
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045907321965897693,
+ "loss": 1.2828,
+ "theoretical_loss": 3.56042690735367,
+ "tokens_seen": 1425604608
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045905717151454364,
+ "loss": 1.229,
+ "theoretical_loss": 3.560412451922445,
+ "tokens_seen": 1425670144
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045904112337011034,
+ "loss": 1.3612,
+ "theoretical_loss": 3.5603979973417488,
+ "tokens_seen": 1425735680
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045902507522567704,
+ "loss": 1.1432,
+ "theoretical_loss": 3.5603835436114926,
+ "tokens_seen": 1425801216
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045900902708124374,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5603690907315864,
+ "tokens_seen": 1425866752
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045899297893681044,
+ "loss": 1.2868,
+ "theoretical_loss": 3.5603546387019422,
+ "tokens_seen": 1425932288
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045897693079237714,
+ "loss": 1.2895,
+ "theoretical_loss": 3.560340187522471,
+ "tokens_seen": 1425997824
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045896088264794384,
+ "loss": 1.2537,
+ "theoretical_loss": 3.560325737193083,
+ "tokens_seen": 1426063360
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045894483450351054,
+ "loss": 1.2988,
+ "theoretical_loss": 3.5603112877136893,
+ "tokens_seen": 1426128896
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004589287863590772,
+ "loss": 1.3919,
+ "theoretical_loss": 3.560296839084201,
+ "tokens_seen": 1426194432
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045891273821464394,
+ "loss": 1.2831,
+ "theoretical_loss": 3.5602823913045283,
+ "tokens_seen": 1426259968
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004588966900702106,
+ "loss": 1.2351,
+ "theoretical_loss": 3.5602679443745835,
+ "tokens_seen": 1426325504
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045888064192577734,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5602534982942773,
+ "tokens_seen": 1426391040
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000458864593781344,
+ "loss": 1.3329,
+ "theoretical_loss": 3.56023905306352,
+ "tokens_seen": 1426456576
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045884854563691074,
+ "loss": 1.3195,
+ "theoretical_loss": 3.5602246086822227,
+ "tokens_seen": 1426522112
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004588324974924775,
+ "loss": 1.3218,
+ "theoretical_loss": 3.5602101651502966,
+ "tokens_seen": 1426587648
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045881644934804415,
+ "loss": 1.3029,
+ "theoretical_loss": 3.560195722467653,
+ "tokens_seen": 1426653184
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004588004012036109,
+ "loss": 1.348,
+ "theoretical_loss": 3.560181280634203,
+ "tokens_seen": 1426718720
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045878435305917755,
+ "loss": 1.3028,
+ "theoretical_loss": 3.560166839649857,
+ "tokens_seen": 1426784256
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004587683049147443,
+ "loss": 1.3608,
+ "theoretical_loss": 3.5601523995145268,
+ "tokens_seen": 1426849792
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045875225677031095,
+ "loss": 1.1981,
+ "theoretical_loss": 3.560137960228123,
+ "tokens_seen": 1426915328
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004587362086258777,
+ "loss": 1.2624,
+ "theoretical_loss": 3.5601235217905574,
+ "tokens_seen": 1426980864
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045872016048144435,
+ "loss": 1.2641,
+ "theoretical_loss": 3.56010908420174,
+ "tokens_seen": 1427046400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004587041123370111,
+ "loss": 1.3413,
+ "theoretical_loss": 3.5600946474615824,
+ "tokens_seen": 1427111936
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045868806419257775,
+ "loss": 1.362,
+ "theoretical_loss": 3.5600802115699968,
+ "tokens_seen": 1427177472
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004586720160481445,
+ "loss": 1.2466,
+ "theoretical_loss": 3.560065776526893,
+ "tokens_seen": 1427243008
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045865596790371115,
+ "loss": 1.3808,
+ "theoretical_loss": 3.5600513423321827,
+ "tokens_seen": 1427308544
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004586399197592779,
+ "loss": 1.2685,
+ "theoretical_loss": 3.5600369089857766,
+ "tokens_seen": 1427374080
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045862387161484455,
+ "loss": 1.2599,
+ "theoretical_loss": 3.5600224764875867,
+ "tokens_seen": 1427439616
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004586078234704113,
+ "loss": 1.3494,
+ "theoretical_loss": 3.5600080448375238,
+ "tokens_seen": 1427505152
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045859177532597795,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5599936140354993,
+ "tokens_seen": 1427570688
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004585757271815447,
+ "loss": 1.33,
+ "theoretical_loss": 3.559979184081424,
+ "tokens_seen": 1427636224
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045855967903711136,
+ "loss": 1.3565,
+ "theoretical_loss": 3.5599647549752094,
+ "tokens_seen": 1427701760
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004585436308926781,
+ "loss": 1.2961,
+ "theoretical_loss": 3.559950326716767,
+ "tokens_seen": 1427767296
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045852758274824476,
+ "loss": 1.222,
+ "theoretical_loss": 3.5599358993060077,
+ "tokens_seen": 1427832832
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004585115346038115,
+ "loss": 1.2649,
+ "theoretical_loss": 3.559921472742843,
+ "tokens_seen": 1427898368
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045849548645937816,
+ "loss": 1.2992,
+ "theoretical_loss": 3.5599070470271847,
+ "tokens_seen": 1427963904
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004584794383149449,
+ "loss": 1.239,
+ "theoretical_loss": 3.5598926221589435,
+ "tokens_seen": 1428029440
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045846339017051156,
+ "loss": 1.2707,
+ "theoretical_loss": 3.5598781981380307,
+ "tokens_seen": 1428094976
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045844734202607826,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5598637749643576,
+ "tokens_seen": 1428160512
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045843129388164496,
+ "loss": 1.3567,
+ "theoretical_loss": 3.559849352637836,
+ "tokens_seen": 1428226048
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045841524573721166,
+ "loss": 1.3475,
+ "theoretical_loss": 3.5598349311583766,
+ "tokens_seen": 1428291584
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045839919759277836,
+ "loss": 1.2968,
+ "theoretical_loss": 3.559820510525892,
+ "tokens_seen": 1428357120
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045838314944834506,
+ "loss": 1.2214,
+ "theoretical_loss": 3.5598060907402918,
+ "tokens_seen": 1428422656
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045836710130391176,
+ "loss": 1.2678,
+ "theoretical_loss": 3.559791671801489,
+ "tokens_seen": 1428488192
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045835105315947846,
+ "loss": 1.2237,
+ "theoretical_loss": 3.5597772537093944,
+ "tokens_seen": 1428553728
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045833500501504516,
+ "loss": 1.305,
+ "theoretical_loss": 3.5597628364639196,
+ "tokens_seen": 1428619264
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 479356,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.245415449142456,
+ "objective/train/theoretical_loss": 3.559748420064976,
+ "objective/train/tokens_used": 1449144800,
+ "theoretical_loss": 3.559748420064976,
+ "tokens_seen": 1428684800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045831895687061186,
+ "loss": 1.2871,
+ "theoretical_loss": 3.559748420064976,
+ "tokens_seen": 1428684800
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004583029087261785,
+ "loss": 1.2833,
+ "theoretical_loss": 3.559734004512475,
+ "tokens_seen": 1428750336
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045828686058174527,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5597195898063276,
+ "tokens_seen": 1428815872
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004582708124373119,
+ "loss": 1.2679,
+ "theoretical_loss": 3.5597051759464464,
+ "tokens_seen": 1428881408
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045825476429287867,
+ "loss": 1.3871,
+ "theoretical_loss": 3.5596907629327417,
+ "tokens_seen": 1428946944
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004582387161484453,
+ "loss": 1.3208,
+ "theoretical_loss": 3.559676350765126,
+ "tokens_seen": 1429012480
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045822266800401207,
+ "loss": 1.22,
+ "theoretical_loss": 3.5596619394435107,
+ "tokens_seen": 1429078016
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004582066198595787,
+ "loss": 1.2834,
+ "theoretical_loss": 3.5596475289678065,
+ "tokens_seen": 1429143552
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045819057171514547,
+ "loss": 1.2182,
+ "theoretical_loss": 3.559633119337926,
+ "tokens_seen": 1429209088
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004581745235707121,
+ "loss": 1.2952,
+ "theoretical_loss": 3.55961871055378,
+ "tokens_seen": 1429274624
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045815847542627887,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5596043026152806,
+ "tokens_seen": 1429340160
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004581424272818455,
+ "loss": 1.2538,
+ "theoretical_loss": 3.5595898955223393,
+ "tokens_seen": 1429405696
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045812637913741227,
+ "loss": 1.2469,
+ "theoretical_loss": 3.5595754892748674,
+ "tokens_seen": 1429471232
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004581103309929789,
+ "loss": 1.2872,
+ "theoretical_loss": 3.5595610838727767,
+ "tokens_seen": 1429536768
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580942828485457,
+ "loss": 1.3236,
+ "theoretical_loss": 3.559546679315979,
+ "tokens_seen": 1429602304
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580782347041123,
+ "loss": 1.3737,
+ "theoretical_loss": 3.559532275604386,
+ "tokens_seen": 1429667840
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580621865596791,
+ "loss": 1.379,
+ "theoretical_loss": 3.559517872737909,
+ "tokens_seen": 1429733376
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580461384152457,
+ "loss": 1.2007,
+ "theoretical_loss": 3.55950347071646,
+ "tokens_seen": 1429798912
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580300902708125,
+ "loss": 1.3352,
+ "theoretical_loss": 3.5594890695399504,
+ "tokens_seen": 1429864448
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004580140421263791,
+ "loss": 1.2399,
+ "theoretical_loss": 3.559474669208292,
+ "tokens_seen": 1429929984
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579979939819459,
+ "loss": 1.2412,
+ "theoretical_loss": 3.559460269721397,
+ "tokens_seen": 1429995520
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579819458375125,
+ "loss": 1.3351,
+ "theoretical_loss": 3.5594458710791765,
+ "tokens_seen": 1430061056
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579658976930793,
+ "loss": 1.2859,
+ "theoretical_loss": 3.5594314732815424,
+ "tokens_seen": 1430126592
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579498495486459,
+ "loss": 1.2754,
+ "theoretical_loss": 3.5594170763284065,
+ "tokens_seen": 1430192128
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579338014042127,
+ "loss": 1.2804,
+ "theoretical_loss": 3.559402680219681,
+ "tokens_seen": 1430257664
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579177532597793,
+ "loss": 1.3938,
+ "theoretical_loss": 3.559388284955277,
+ "tokens_seen": 1430323200
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004579017051153461,
+ "loss": 1.4642,
+ "theoretical_loss": 3.559373890535107,
+ "tokens_seen": 1430388736
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045788565697091273,
+ "loss": 1.3394,
+ "theoretical_loss": 3.5593594969590816,
+ "tokens_seen": 1430454272
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004578696088264795,
+ "loss": 1.2821,
+ "theoretical_loss": 3.559345104227114,
+ "tokens_seen": 1430519808
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045785356068204613,
+ "loss": 1.2923,
+ "theoretical_loss": 3.5593307123391154,
+ "tokens_seen": 1430585344
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004578375125376129,
+ "loss": 1.2905,
+ "theoretical_loss": 3.5593163212949976,
+ "tokens_seen": 1430650880
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045782146439317953,
+ "loss": 1.3761,
+ "theoretical_loss": 3.5593019310946725,
+ "tokens_seen": 1430716416
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004578054162487463,
+ "loss": 1.3591,
+ "theoretical_loss": 3.559287541738052,
+ "tokens_seen": 1430781952
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045778936810431293,
+ "loss": 1.2329,
+ "theoretical_loss": 3.5592731532250483,
+ "tokens_seen": 1430847488
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004577733199598797,
+ "loss": 1.2312,
+ "theoretical_loss": 3.559258765555573,
+ "tokens_seen": 1430913024
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045775727181544633,
+ "loss": 1.5046,
+ "theoretical_loss": 3.559244378729538,
+ "tokens_seen": 1430978560
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004577412236710131,
+ "loss": 1.3033,
+ "theoretical_loss": 3.5592299927468556,
+ "tokens_seen": 1431044096
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045772517552657973,
+ "loss": 1.3342,
+ "theoretical_loss": 3.5592156076074373,
+ "tokens_seen": 1431109632
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004577091273821465,
+ "loss": 1.2985,
+ "theoretical_loss": 3.559201223311195,
+ "tokens_seen": 1431175168
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045769307923771314,
+ "loss": 1.278,
+ "theoretical_loss": 3.5591868398580413,
+ "tokens_seen": 1431240704
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045767703109327984,
+ "loss": 1.2854,
+ "theoretical_loss": 3.559172457247887,
+ "tokens_seen": 1431306240
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045766098294884654,
+ "loss": 1.5248,
+ "theoretical_loss": 3.5591580754806458,
+ "tokens_seen": 1431371776
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045764493480441324,
+ "loss": 1.3153,
+ "theoretical_loss": 3.5591436945562283,
+ "tokens_seen": 1431437312
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045762888665998,
+ "loss": 1.3561,
+ "theoretical_loss": 3.5591293144745464,
+ "tokens_seen": 1431502848
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045761283851554664,
+ "loss": 1.3035,
+ "theoretical_loss": 3.559114935235513,
+ "tokens_seen": 1431568384
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004575967903711134,
+ "loss": 1.2852,
+ "theoretical_loss": 3.5591005568390406,
+ "tokens_seen": 1431633920
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045758074222668004,
+ "loss": 1.2484,
+ "theoretical_loss": 3.5590861792850403,
+ "tokens_seen": 1431699456
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004575646940822468,
+ "loss": 1.326,
+ "theoretical_loss": 3.559071802573424,
+ "tokens_seen": 1431764992
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045754864593781344,
+ "loss": 1.399,
+ "theoretical_loss": 3.5590574267041046,
+ "tokens_seen": 1431830528
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004575325977933802,
+ "loss": 1.4098,
+ "theoretical_loss": 3.5590430516769938,
+ "tokens_seen": 1431896064
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 480635,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3885066509246826,
+ "objective/train/theoretical_loss": 3.5590286774920035,
+ "objective/train/tokens_used": 1452421600,
+ "theoretical_loss": 3.5590286774920035,
+ "tokens_seen": 1431961600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045751654964894684,
+ "loss": 1.3568,
+ "theoretical_loss": 3.5590286774920035,
+ "tokens_seen": 1431961600
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004575005015045136,
+ "loss": 1.3758,
+ "theoretical_loss": 3.5590143041490463,
+ "tokens_seen": 1432027136
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045748445336008024,
+ "loss": 1.3635,
+ "theoretical_loss": 3.558999931648034,
+ "tokens_seen": 1432092672
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000457468405215647,
+ "loss": 1.2606,
+ "theoretical_loss": 3.5589855599888782,
+ "tokens_seen": 1432158208
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045745235707121364,
+ "loss": 1.2629,
+ "theoretical_loss": 3.5589711891714924,
+ "tokens_seen": 1432223744
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004574363089267804,
+ "loss": 1.2715,
+ "theoretical_loss": 3.5589568191957883,
+ "tokens_seen": 1432289280
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045742026078234705,
+ "loss": 1.4599,
+ "theoretical_loss": 3.5589424500616778,
+ "tokens_seen": 1432354816
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004574042126379138,
+ "loss": 1.4052,
+ "theoretical_loss": 3.558928081769073,
+ "tokens_seen": 1432420352
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045738816449348045,
+ "loss": 1.2824,
+ "theoretical_loss": 3.558913714317886,
+ "tokens_seen": 1432485888
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004573721163490472,
+ "loss": 1.3131,
+ "theoretical_loss": 3.5588993477080297,
+ "tokens_seen": 1432551424
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045735606820461385,
+ "loss": 1.3909,
+ "theoretical_loss": 3.558884981939416,
+ "tokens_seen": 1432616960
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004573400200601806,
+ "loss": 1.1918,
+ "theoretical_loss": 3.5588706170119577,
+ "tokens_seen": 1432682496
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045732397191574725,
+ "loss": 1.2211,
+ "theoretical_loss": 3.5588562529255663,
+ "tokens_seen": 1432748032
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000457307923771314,
+ "loss": 1.3774,
+ "theoretical_loss": 3.5588418896801537,
+ "tokens_seen": 1432813568
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045729187562688065,
+ "loss": 1.2349,
+ "theoretical_loss": 3.558827527275633,
+ "tokens_seen": 1432879104
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004572758274824474,
+ "loss": 1.2705,
+ "theoretical_loss": 3.558813165711917,
+ "tokens_seen": 1432944640
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045725977933801405,
+ "loss": 1.2108,
+ "theoretical_loss": 3.5587988049889168,
+ "tokens_seen": 1433010176
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004572437311935808,
+ "loss": 1.3224,
+ "theoretical_loss": 3.5587844451065456,
+ "tokens_seen": 1433075712
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045722768304914745,
+ "loss": 1.2045,
+ "theoretical_loss": 3.5587700860647153,
+ "tokens_seen": 1433141248
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004572116349047142,
+ "loss": 1.3315,
+ "theoretical_loss": 3.558755727863338,
+ "tokens_seen": 1433206784
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045719558676028085,
+ "loss": 1.3738,
+ "theoretical_loss": 3.558741370502327,
+ "tokens_seen": 1433272320
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004571795386158476,
+ "loss": 1.2957,
+ "theoretical_loss": 3.5587270139815943,
+ "tokens_seen": 1433337856
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045716349047141426,
+ "loss": 1.41,
+ "theoretical_loss": 3.5587126583010518,
+ "tokens_seen": 1433403392
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000457147442326981,
+ "loss": 1.3075,
+ "theoretical_loss": 3.5586983034606128,
+ "tokens_seen": 1433468928
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045713139418254766,
+ "loss": 1.2395,
+ "theoretical_loss": 3.558683949460189,
+ "tokens_seen": 1433534464
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004571153460381144,
+ "loss": 1.3525,
+ "theoretical_loss": 3.5586695962996933,
+ "tokens_seen": 1433600000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045709929789368106,
+ "loss": 1.2121,
+ "theoretical_loss": 3.5586552439790378,
+ "tokens_seen": 1433665536
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004570832497492478,
+ "loss": 1.3105,
+ "theoretical_loss": 3.5586408924981354,
+ "tokens_seen": 1433731072
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045706720160481446,
+ "loss": 1.3357,
+ "theoretical_loss": 3.558626541856898,
+ "tokens_seen": 1433796608
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045705115346038116,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5586121920552385,
+ "tokens_seen": 1433862144
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045703510531594786,
+ "loss": 1.3872,
+ "theoretical_loss": 3.5585978430930694,
+ "tokens_seen": 1433927680
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045701905717151456,
+ "loss": 1.3249,
+ "theoretical_loss": 3.5585834949703026,
+ "tokens_seen": 1433993216
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045700300902708126,
+ "loss": 1.3845,
+ "theoretical_loss": 3.558569147686852,
+ "tokens_seen": 1434058752
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045698696088264796,
+ "loss": 1.3745,
+ "theoretical_loss": 3.5585548012426287,
+ "tokens_seen": 1434124288
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045697091273821466,
+ "loss": 1.3304,
+ "theoretical_loss": 3.5585404556375466,
+ "tokens_seen": 1434189824
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045695486459378136,
+ "loss": 1.2257,
+ "theoretical_loss": 3.558526110871517,
+ "tokens_seen": 1434255360
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045693881644934807,
+ "loss": 1.3389,
+ "theoretical_loss": 3.558511766944453,
+ "tokens_seen": 1434320896
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045692276830491477,
+ "loss": 1.2873,
+ "theoretical_loss": 3.558497423856268,
+ "tokens_seen": 1434386432
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004569067201604814,
+ "loss": 1.4315,
+ "theoretical_loss": 3.558483081606873,
+ "tokens_seen": 1434451968
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045689067201604817,
+ "loss": 1.2989,
+ "theoretical_loss": 3.558468740196182,
+ "tokens_seen": 1434517504
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004568746238716148,
+ "loss": 1.3377,
+ "theoretical_loss": 3.5584543996241074,
+ "tokens_seen": 1434583040
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045685857572718157,
+ "loss": 1.2988,
+ "theoretical_loss": 3.558440059890561,
+ "tokens_seen": 1434648576
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004568425275827482,
+ "loss": 1.3134,
+ "theoretical_loss": 3.558425720995456,
+ "tokens_seen": 1434714112
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045682647943831497,
+ "loss": 1.4479,
+ "theoretical_loss": 3.558411382938706,
+ "tokens_seen": 1434779648
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004568104312938816,
+ "loss": 1.2924,
+ "theoretical_loss": 3.558397045720222,
+ "tokens_seen": 1434845184
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045679438314944837,
+ "loss": 1.3414,
+ "theoretical_loss": 3.5583827093399183,
+ "tokens_seen": 1434910720
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000456778335005015,
+ "loss": 1.2888,
+ "theoretical_loss": 3.5583683737977063,
+ "tokens_seen": 1434976256
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.00045676228686058177,
+ "loss": 1.3414,
+ "theoretical_loss": 3.5583540390934996,
+ "tokens_seen": 1435041792
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004567462387161484,
+ "loss": 1.2786,
+ "theoretical_loss": 3.55833970522721,
+ "tokens_seen": 1435107328
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004567301905717152,
+ "loss": 1.2949,
+ "theoretical_loss": 3.558325372198752,
+ "tokens_seen": 1435172864
+ },
+ {
+ "epoch": 0.43,
+ "objective/train/docs_used": 481622,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3161892890930176,
+ "objective/train/theoretical_loss": 3.5583110400080367,
+ "objective/train/tokens_used": 1455698400,
+ "theoretical_loss": 3.5583110400080367,
+ "tokens_seen": 1435238400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004567141424272818,
+ "loss": 1.379,
+ "theoretical_loss": 3.5583110400080367,
+ "tokens_seen": 1435238400
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004566980942828486,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5582967086549777,
+ "tokens_seen": 1435303936
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.0004566820461384152,
+ "loss": 1.2946,
+ "theoretical_loss": 3.558282378139487,
+ "tokens_seen": 1435369472
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 0.000456665997993982,
+ "loss": 1.3131,
+ "theoretical_loss": 3.5582680484614784,
+ "tokens_seen": 1435435008
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004566499498495486,
+ "loss": 1.2119,
+ "theoretical_loss": 3.5582537196208643,
+ "tokens_seen": 1435500544
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004566339017051154,
+ "loss": 1.3031,
+ "theoretical_loss": 3.5582393916175574,
+ "tokens_seen": 1435566080
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000456617853560682,
+ "loss": 1.3375,
+ "theoretical_loss": 3.558225064451471,
+ "tokens_seen": 1435631616
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004566018054162488,
+ "loss": 1.3074,
+ "theoretical_loss": 3.5582107381225176,
+ "tokens_seen": 1435697152
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004565857572718154,
+ "loss": 1.2224,
+ "theoretical_loss": 3.5581964126306103,
+ "tokens_seen": 1435762688
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004565697091273822,
+ "loss": 1.2397,
+ "theoretical_loss": 3.5581820879756614,
+ "tokens_seen": 1435828224
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004565536609829488,
+ "loss": 1.3009,
+ "theoretical_loss": 3.5581677641575844,
+ "tokens_seen": 1435893760
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004565376128385156,
+ "loss": 1.2711,
+ "theoretical_loss": 3.558153441176292,
+ "tokens_seen": 1435959296
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045652156469408223,
+ "loss": 1.307,
+ "theoretical_loss": 3.558139119031697,
+ "tokens_seen": 1436024832
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000456505516549649,
+ "loss": 1.437,
+ "theoretical_loss": 3.558124797723713,
+ "tokens_seen": 1436090368
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045648946840521563,
+ "loss": 1.3788,
+ "theoretical_loss": 3.558110477252252,
+ "tokens_seen": 1436155904
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004564734202607824,
+ "loss": 1.3812,
+ "theoretical_loss": 3.5580961576172276,
+ "tokens_seen": 1436221440
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045645737211634903,
+ "loss": 1.2538,
+ "theoretical_loss": 3.558081838818553,
+ "tokens_seen": 1436286976
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004564413239719158,
+ "loss": 1.3411,
+ "theoretical_loss": 3.5580675208561403,
+ "tokens_seen": 1436352512
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004564252758274825,
+ "loss": 1.2705,
+ "theoretical_loss": 3.5580532037299033,
+ "tokens_seen": 1436418048
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004564092276830492,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5580388874397544,
+ "tokens_seen": 1436483584
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004563931795386159,
+ "loss": 1.3112,
+ "theoretical_loss": 3.5580245719856074,
+ "tokens_seen": 1436549120
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004563771313941826,
+ "loss": 1.2366,
+ "theoretical_loss": 3.558010257367375,
+ "tokens_seen": 1436614656
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004563610832497493,
+ "loss": 1.2951,
+ "theoretical_loss": 3.5579959435849693,
+ "tokens_seen": 1436680192
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000456345035105316,
+ "loss": 1.3306,
+ "theoretical_loss": 3.557981630638305,
+ "tokens_seen": 1436745728
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004563289869608827,
+ "loss": 1.421,
+ "theoretical_loss": 3.557967318527295,
+ "tokens_seen": 1436811264
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004563129388164494,
+ "loss": 1.2585,
+ "theoretical_loss": 3.557953007251851,
+ "tokens_seen": 1436876800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004562968906720161,
+ "loss": 1.3057,
+ "theoretical_loss": 3.5579386968118865,
+ "tokens_seen": 1436942336
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045628084252758274,
+ "loss": 1.2843,
+ "theoretical_loss": 3.557924387207316,
+ "tokens_seen": 1437007872
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004562647943831495,
+ "loss": 1.3638,
+ "theoretical_loss": 3.5579100784380513,
+ "tokens_seen": 1437073408
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045624874623871614,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5578957705040057,
+ "tokens_seen": 1437138944
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004562326980942829,
+ "loss": 1.2938,
+ "theoretical_loss": 3.557881463405093,
+ "tokens_seen": 1437204480
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045621664994984954,
+ "loss": 1.2259,
+ "theoretical_loss": 3.557867157141226,
+ "tokens_seen": 1437270016
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004562006018054163,
+ "loss": 1.3679,
+ "theoretical_loss": 3.557852851712318,
+ "tokens_seen": 1437335552
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045618455366098294,
+ "loss": 1.3252,
+ "theoretical_loss": 3.557838547118282,
+ "tokens_seen": 1437401088
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004561685055165497,
+ "loss": 1.3509,
+ "theoretical_loss": 3.5578242433590317,
+ "tokens_seen": 1437466624
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045615245737211634,
+ "loss": 1.283,
+ "theoretical_loss": 3.5578099404344794,
+ "tokens_seen": 1437532160
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004561364092276831,
+ "loss": 1.3375,
+ "theoretical_loss": 3.5577956383445386,
+ "tokens_seen": 1437597696
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045612036108324974,
+ "loss": 1.2761,
+ "theoretical_loss": 3.557781337089123,
+ "tokens_seen": 1437663232
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004561043129388165,
+ "loss": 1.2984,
+ "theoretical_loss": 3.557767036668146,
+ "tokens_seen": 1437728768
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045608826479438314,
+ "loss": 1.3462,
+ "theoretical_loss": 3.557752737081521,
+ "tokens_seen": 1437794304
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004560722166499499,
+ "loss": 1.2343,
+ "theoretical_loss": 3.55773843832916,
+ "tokens_seen": 1437859840
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045605616850551655,
+ "loss": 1.36,
+ "theoretical_loss": 3.5577241404109774,
+ "tokens_seen": 1437925376
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004560401203610833,
+ "loss": 1.3435,
+ "theoretical_loss": 3.5577098433268866,
+ "tokens_seen": 1437990912
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045602407221664995,
+ "loss": 1.4761,
+ "theoretical_loss": 3.5576955470768,
+ "tokens_seen": 1438056448
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004560080240722167,
+ "loss": 1.3683,
+ "theoretical_loss": 3.5576812516606324,
+ "tokens_seen": 1438121984
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045599197592778335,
+ "loss": 1.329,
+ "theoretical_loss": 3.557666957078296,
+ "tokens_seen": 1438187520
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004559759277833501,
+ "loss": 1.3379,
+ "theoretical_loss": 3.557652663329704,
+ "tokens_seen": 1438253056
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045595987963891675,
+ "loss": 1.4506,
+ "theoretical_loss": 3.5576383704147707,
+ "tokens_seen": 1438318592
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004559438314944835,
+ "loss": 1.2574,
+ "theoretical_loss": 3.5576240783334097,
+ "tokens_seen": 1438384128
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045592778335005015,
+ "loss": 1.2259,
+ "theoretical_loss": 3.557609787085533,
+ "tokens_seen": 1438449664
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 482849,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2713879346847534,
+ "objective/train/theoretical_loss": 3.5575954966710546,
+ "objective/train/tokens_used": 1458975200,
+ "theoretical_loss": 3.5575954966710546,
+ "tokens_seen": 1438515200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004559117352056169,
+ "loss": 1.2522,
+ "theoretical_loss": 3.5575954966710546,
+ "tokens_seen": 1438515200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045589568706118355,
+ "loss": 1.2682,
+ "theoretical_loss": 3.5575812070898887,
+ "tokens_seen": 1438580736
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004558796389167503,
+ "loss": 1.3228,
+ "theoretical_loss": 3.557566918341948,
+ "tokens_seen": 1438646272
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045586359077231695,
+ "loss": 1.361,
+ "theoretical_loss": 3.5575526304271463,
+ "tokens_seen": 1438711808
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004558475426278837,
+ "loss": 1.2867,
+ "theoretical_loss": 3.5575383433453966,
+ "tokens_seen": 1438777344
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045583149448345035,
+ "loss": 1.2955,
+ "theoretical_loss": 3.557524057096613,
+ "tokens_seen": 1438842880
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004558154463390171,
+ "loss": 1.3134,
+ "theoretical_loss": 3.5575097716807087,
+ "tokens_seen": 1438908416
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045579939819458376,
+ "loss": 1.2428,
+ "theoretical_loss": 3.557495487097597,
+ "tokens_seen": 1438973952
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004557833500501505,
+ "loss": 1.3008,
+ "theoretical_loss": 3.557481203347192,
+ "tokens_seen": 1439039488
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045576730190571716,
+ "loss": 1.3362,
+ "theoretical_loss": 3.5574669204294067,
+ "tokens_seen": 1439105024
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004557512537612839,
+ "loss": 1.298,
+ "theoretical_loss": 3.557452638344155,
+ "tokens_seen": 1439170560
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045573520561685056,
+ "loss": 1.2211,
+ "theoretical_loss": 3.5574383570913497,
+ "tokens_seen": 1439236096
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004557191574724173,
+ "loss": 1.3366,
+ "theoretical_loss": 3.5574240766709058,
+ "tokens_seen": 1439301632
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045570310932798396,
+ "loss": 1.3071,
+ "theoretical_loss": 3.5574097970827356,
+ "tokens_seen": 1439367168
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004556870611835507,
+ "loss": 1.2954,
+ "theoretical_loss": 3.5573955183267536,
+ "tokens_seen": 1439432704
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045567101303911736,
+ "loss": 1.4258,
+ "theoretical_loss": 3.5573812404028726,
+ "tokens_seen": 1439498240
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004556549648946841,
+ "loss": 1.3103,
+ "theoretical_loss": 3.557366963311007,
+ "tokens_seen": 1439563776
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045563891675025076,
+ "loss": 1.3403,
+ "theoretical_loss": 3.5573526870510697,
+ "tokens_seen": 1439629312
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045562286860581746,
+ "loss": 1.3593,
+ "theoretical_loss": 3.5573384116229745,
+ "tokens_seen": 1439694848
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045560682046138416,
+ "loss": 1.3412,
+ "theoretical_loss": 3.5573241370266357,
+ "tokens_seen": 1439760384
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045559077231695086,
+ "loss": 1.3049,
+ "theoretical_loss": 3.557309863261967,
+ "tokens_seen": 1439825920
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045557472417251756,
+ "loss": 1.2632,
+ "theoretical_loss": 3.5572955903288808,
+ "tokens_seen": 1439891456
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045555867602808427,
+ "loss": 1.2165,
+ "theoretical_loss": 3.557281318227292,
+ "tokens_seen": 1439956992
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045554262788365097,
+ "loss": 1.2038,
+ "theoretical_loss": 3.557267046957114,
+ "tokens_seen": 1440022528
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045552657973921767,
+ "loss": 1.4544,
+ "theoretical_loss": 3.5572527765182604,
+ "tokens_seen": 1440088064
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004555105315947843,
+ "loss": 1.3677,
+ "theoretical_loss": 3.557238506910646,
+ "tokens_seen": 1440153600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045549448345035107,
+ "loss": 1.3547,
+ "theoretical_loss": 3.5572242381341823,
+ "tokens_seen": 1440219136
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004554784353059177,
+ "loss": 1.3557,
+ "theoretical_loss": 3.557209970188785,
+ "tokens_seen": 1440284672
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045546238716148447,
+ "loss": 1.2771,
+ "theoretical_loss": 3.5571957030743677,
+ "tokens_seen": 1440350208
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004554463390170511,
+ "loss": 1.2832,
+ "theoretical_loss": 3.557181436790843,
+ "tokens_seen": 1440415744
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045543029087261787,
+ "loss": 1.3751,
+ "theoretical_loss": 3.5571671713381257,
+ "tokens_seen": 1440481280
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004554142427281845,
+ "loss": 1.2117,
+ "theoretical_loss": 3.5571529067161296,
+ "tokens_seen": 1440546816
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045539819458375127,
+ "loss": 1.3891,
+ "theoretical_loss": 3.5571386429247687,
+ "tokens_seen": 1440612352
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553821464393179,
+ "loss": 1.3857,
+ "theoretical_loss": 3.557124379963956,
+ "tokens_seen": 1440677888
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553660982948847,
+ "loss": 1.4717,
+ "theoretical_loss": 3.5571101178336058,
+ "tokens_seen": 1440743424
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553500501504513,
+ "loss": 1.3517,
+ "theoretical_loss": 3.557095856533632,
+ "tokens_seen": 1440808960
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553340020060181,
+ "loss": 1.1584,
+ "theoretical_loss": 3.5570815960639486,
+ "tokens_seen": 1440874496
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553179538615847,
+ "loss": 1.3169,
+ "theoretical_loss": 3.5570673364244696,
+ "tokens_seen": 1440940032
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004553019057171515,
+ "loss": 1.4056,
+ "theoretical_loss": 3.557053077615109,
+ "tokens_seen": 1441005568
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004552858575727181,
+ "loss": 1.2782,
+ "theoretical_loss": 3.5570388196357796,
+ "tokens_seen": 1441071104
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004552698094282849,
+ "loss": 1.338,
+ "theoretical_loss": 3.5570245624863968,
+ "tokens_seen": 1441136640
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045525376128385163,
+ "loss": 1.2896,
+ "theoretical_loss": 3.557010306166874,
+ "tokens_seen": 1441202176
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004552377131394183,
+ "loss": 1.3373,
+ "theoretical_loss": 3.5569960506771245,
+ "tokens_seen": 1441267712
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045522166499498503,
+ "loss": 1.4077,
+ "theoretical_loss": 3.5569817960170633,
+ "tokens_seen": 1441333248
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004552056168505517,
+ "loss": 1.2653,
+ "theoretical_loss": 3.5569675421866043,
+ "tokens_seen": 1441398784
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045518956870611843,
+ "loss": 1.2131,
+ "theoretical_loss": 3.5569532891856603,
+ "tokens_seen": 1441464320
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004551735205616851,
+ "loss": 1.2864,
+ "theoretical_loss": 3.556939037014147,
+ "tokens_seen": 1441529856
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045515747241725184,
+ "loss": 1.2779,
+ "theoretical_loss": 3.5569247856719772,
+ "tokens_seen": 1441595392
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004551414242728185,
+ "loss": 1.339,
+ "theoretical_loss": 3.5569105351590657,
+ "tokens_seen": 1441660928
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045512537612838524,
+ "loss": 1.3434,
+ "theoretical_loss": 3.556896285475326,
+ "tokens_seen": 1441726464
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 483810,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2746837139129639,
+ "objective/train/theoretical_loss": 3.5568820366206726,
+ "objective/train/tokens_used": 1462252000,
+ "theoretical_loss": 3.5568820366206726,
+ "tokens_seen": 1441792000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004551093279839519,
+ "loss": 1.3041,
+ "theoretical_loss": 3.5568820366206726,
+ "tokens_seen": 1441792000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045509327983951864,
+ "loss": 1.2942,
+ "theoretical_loss": 3.556867788595019,
+ "tokens_seen": 1441857536
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004550772316950853,
+ "loss": 1.2462,
+ "theoretical_loss": 3.5568535413982794,
+ "tokens_seen": 1441923072
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045506118355065204,
+ "loss": 1.4316,
+ "theoretical_loss": 3.556839295030369,
+ "tokens_seen": 1441988608
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004550451354062187,
+ "loss": 1.3575,
+ "theoretical_loss": 3.5568250494912004,
+ "tokens_seen": 1442054144
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045502908726178544,
+ "loss": 1.3017,
+ "theoretical_loss": 3.556810804780689,
+ "tokens_seen": 1442119680
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004550130391173521,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5567965608987473,
+ "tokens_seen": 1442185216
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549969909729188,
+ "loss": 1.2188,
+ "theoretical_loss": 3.5567823178452915,
+ "tokens_seen": 1442250752
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549809428284855,
+ "loss": 1.2008,
+ "theoretical_loss": 3.5567680756202344,
+ "tokens_seen": 1442316288
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549648946840522,
+ "loss": 1.2739,
+ "theoretical_loss": 3.5567538342234903,
+ "tokens_seen": 1442381824
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549488465396189,
+ "loss": 1.18,
+ "theoretical_loss": 3.556739593654974,
+ "tokens_seen": 1442447360
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549327983951856,
+ "loss": 1.2894,
+ "theoretical_loss": 3.556725353914599,
+ "tokens_seen": 1442512896
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004549167502507523,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5567111150022805,
+ "tokens_seen": 1442578432
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000454900702106319,
+ "loss": 1.4224,
+ "theoretical_loss": 3.5566968769179317,
+ "tokens_seen": 1442643968
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004548846539618857,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5566826396614672,
+ "tokens_seen": 1442709504
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004548686058174524,
+ "loss": 1.3673,
+ "theoretical_loss": 3.556668403232801,
+ "tokens_seen": 1442775040
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045485255767301904,
+ "loss": 1.2996,
+ "theoretical_loss": 3.556654167631848,
+ "tokens_seen": 1442840576
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004548365095285858,
+ "loss": 1.3492,
+ "theoretical_loss": 3.5566399328585225,
+ "tokens_seen": 1442906112
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045482046138415244,
+ "loss": 1.3996,
+ "theoretical_loss": 3.556625698912738,
+ "tokens_seen": 1442971648
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004548044132397192,
+ "loss": 1.3516,
+ "theoretical_loss": 3.5566114657944095,
+ "tokens_seen": 1443037184
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045478836509528584,
+ "loss": 1.3208,
+ "theoretical_loss": 3.5565972335034512,
+ "tokens_seen": 1443102720
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004547723169508526,
+ "loss": 1.2532,
+ "theoretical_loss": 3.556583002039777,
+ "tokens_seen": 1443168256
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045475626880641924,
+ "loss": 1.2962,
+ "theoretical_loss": 3.5565687714033016,
+ "tokens_seen": 1443233792
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000454740220661986,
+ "loss": 1.2418,
+ "theoretical_loss": 3.5565545415939397,
+ "tokens_seen": 1443299328
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045472417251755264,
+ "loss": 1.279,
+ "theoretical_loss": 3.556540312611605,
+ "tokens_seen": 1443364864
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004547081243731194,
+ "loss": 1.3042,
+ "theoretical_loss": 3.5565260844562117,
+ "tokens_seen": 1443430400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045469207622868605,
+ "loss": 1.3724,
+ "theoretical_loss": 3.556511857127675,
+ "tokens_seen": 1443495936
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004546760280842528,
+ "loss": 1.2836,
+ "theoretical_loss": 3.5564976306259095,
+ "tokens_seen": 1443561472
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045465997993981945,
+ "loss": 1.3978,
+ "theoretical_loss": 3.556483404950829,
+ "tokens_seen": 1443627008
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004546439317953862,
+ "loss": 1.3526,
+ "theoretical_loss": 3.556469180102348,
+ "tokens_seen": 1443692544
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045462788365095285,
+ "loss": 1.3248,
+ "theoretical_loss": 3.5564549560803806,
+ "tokens_seen": 1443758080
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004546118355065196,
+ "loss": 1.2878,
+ "theoretical_loss": 3.556440732884842,
+ "tokens_seen": 1443823616
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045459578736208625,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5564265105156463,
+ "tokens_seen": 1443889152
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000454579739217653,
+ "loss": 1.2843,
+ "theoretical_loss": 3.556412288972708,
+ "tokens_seen": 1443954688
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045456369107321965,
+ "loss": 1.3958,
+ "theoretical_loss": 3.556398068255942,
+ "tokens_seen": 1444020224
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004545476429287864,
+ "loss": 1.3342,
+ "theoretical_loss": 3.556383848365262,
+ "tokens_seen": 1444085760
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045453159478435305,
+ "loss": 1.4241,
+ "theoretical_loss": 3.556369629300583,
+ "tokens_seen": 1444151296
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004545155466399198,
+ "loss": 1.2689,
+ "theoretical_loss": 3.5563554110618196,
+ "tokens_seen": 1444216832
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045449949849548645,
+ "loss": 1.3332,
+ "theoretical_loss": 3.556341193648886,
+ "tokens_seen": 1444282368
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004544834503510532,
+ "loss": 1.2805,
+ "theoretical_loss": 3.5563269770616976,
+ "tokens_seen": 1444347904
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045446740220661985,
+ "loss": 1.3438,
+ "theoretical_loss": 3.556312761300168,
+ "tokens_seen": 1444413440
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004544513540621866,
+ "loss": 1.3181,
+ "theoretical_loss": 3.5562985463642125,
+ "tokens_seen": 1444478976
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045443530591775326,
+ "loss": 1.3549,
+ "theoretical_loss": 3.556284332253745,
+ "tokens_seen": 1444544512
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045441925777332,
+ "loss": 1.3919,
+ "theoretical_loss": 3.5562701189686807,
+ "tokens_seen": 1444610048
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045440320962888666,
+ "loss": 1.3744,
+ "theoretical_loss": 3.556255906508934,
+ "tokens_seen": 1444675584
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004543871614844534,
+ "loss": 1.2959,
+ "theoretical_loss": 3.5562416948744193,
+ "tokens_seen": 1444741120
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045437111334002006,
+ "loss": 1.1631,
+ "theoretical_loss": 3.5562274840650523,
+ "tokens_seen": 1444806656
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004543550651955868,
+ "loss": 1.4316,
+ "theoretical_loss": 3.556213274080746,
+ "tokens_seen": 1444872192
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045433901705115346,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5561990649214166,
+ "tokens_seen": 1444937728
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004543229689067202,
+ "loss": 1.3261,
+ "theoretical_loss": 3.5561848565869782,
+ "tokens_seen": 1445003264
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 485005,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2077436447143555,
+ "objective/train/theoretical_loss": 3.556170649077345,
+ "objective/train/tokens_used": 1465528800,
+ "theoretical_loss": 3.556170649077345,
+ "tokens_seen": 1445068800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045430692076228686,
+ "loss": 1.299,
+ "theoretical_loss": 3.556170649077345,
+ "tokens_seen": 1445068800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004542908726178536,
+ "loss": 1.3906,
+ "theoretical_loss": 3.5561564423924326,
+ "tokens_seen": 1445134336
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045427482447342026,
+ "loss": 1.3759,
+ "theoretical_loss": 3.556142236532155,
+ "tokens_seen": 1445199872
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000454258776328987,
+ "loss": 1.1993,
+ "theoretical_loss": 3.556128031496428,
+ "tokens_seen": 1445265408
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045424272818455366,
+ "loss": 1.3354,
+ "theoretical_loss": 3.5561138272851647,
+ "tokens_seen": 1445330944
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045422668004012036,
+ "loss": 1.3714,
+ "theoretical_loss": 3.556099623898281,
+ "tokens_seen": 1445396480
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045421063189568706,
+ "loss": 1.3137,
+ "theoretical_loss": 3.5560854213356916,
+ "tokens_seen": 1445462016
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045419458375125377,
+ "loss": 1.3455,
+ "theoretical_loss": 3.556071219597311,
+ "tokens_seen": 1445527552
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045417853560682047,
+ "loss": 1.3484,
+ "theoretical_loss": 3.5560570186830542,
+ "tokens_seen": 1445593088
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045416248746238717,
+ "loss": 1.3403,
+ "theoretical_loss": 3.556042818592836,
+ "tokens_seen": 1445658624
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045414643931795387,
+ "loss": 1.2567,
+ "theoretical_loss": 3.5560286193265713,
+ "tokens_seen": 1445724160
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045413039117352057,
+ "loss": 1.3307,
+ "theoretical_loss": 3.556014420884175,
+ "tokens_seen": 1445789696
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045411434302908727,
+ "loss": 1.3201,
+ "theoretical_loss": 3.5560002232655616,
+ "tokens_seen": 1445855232
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045409829488465397,
+ "loss": 1.2798,
+ "theoretical_loss": 3.555986026470646,
+ "tokens_seen": 1445920768
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004540822467402206,
+ "loss": 1.2078,
+ "theoretical_loss": 3.5559718304993435,
+ "tokens_seen": 1445986304
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045406619859578737,
+ "loss": 1.3716,
+ "theoretical_loss": 3.5559576353515685,
+ "tokens_seen": 1446051840
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004540501504513541,
+ "loss": 1.3648,
+ "theoretical_loss": 3.555943441027236,
+ "tokens_seen": 1446117376
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045403410230692077,
+ "loss": 1.2856,
+ "theoretical_loss": 3.5559292475262616,
+ "tokens_seen": 1446182912
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004540180541624875,
+ "loss": 1.4444,
+ "theoretical_loss": 3.5559150548485596,
+ "tokens_seen": 1446248448
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045400200601805417,
+ "loss": 1.3344,
+ "theoretical_loss": 3.555900862994045,
+ "tokens_seen": 1446313984
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045398595787362093,
+ "loss": 1.2735,
+ "theoretical_loss": 3.5558866719626328,
+ "tokens_seen": 1446379520
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004539699097291876,
+ "loss": 1.3613,
+ "theoretical_loss": 3.555872481754238,
+ "tokens_seen": 1446445056
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045395386158475433,
+ "loss": 1.3414,
+ "theoretical_loss": 3.555858292368775,
+ "tokens_seen": 1446510592
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000453937813440321,
+ "loss": 1.3846,
+ "theoretical_loss": 3.5558441038061606,
+ "tokens_seen": 1446576128
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045392176529588773,
+ "loss": 1.2806,
+ "theoretical_loss": 3.5558299160663074,
+ "tokens_seen": 1446641664
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004539057171514544,
+ "loss": 1.3078,
+ "theoretical_loss": 3.555815729149132,
+ "tokens_seen": 1446707200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045388966900702113,
+ "loss": 1.435,
+ "theoretical_loss": 3.555801543054549,
+ "tokens_seen": 1446772736
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004538736208625878,
+ "loss": 1.3391,
+ "theoretical_loss": 3.5557873577824735,
+ "tokens_seen": 1446838272
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045385757271815453,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5557731733328204,
+ "tokens_seen": 1446903808
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004538415245737212,
+ "loss": 1.3014,
+ "theoretical_loss": 3.555758989705505,
+ "tokens_seen": 1446969344
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045382547642928793,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5557448069004423,
+ "tokens_seen": 1447034880
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004538094282848546,
+ "loss": 1.2944,
+ "theoretical_loss": 3.555730624917547,
+ "tokens_seen": 1447100416
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045379338014042134,
+ "loss": 1.2632,
+ "theoretical_loss": 3.555716443756735,
+ "tokens_seen": 1447165952
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000453777331995988,
+ "loss": 1.3009,
+ "theoretical_loss": 3.555702263417921,
+ "tokens_seen": 1447231488
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045376128385155474,
+ "loss": 1.3152,
+ "theoretical_loss": 3.55568808390102,
+ "tokens_seen": 1447297024
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004537452357071214,
+ "loss": 1.3284,
+ "theoretical_loss": 3.555673905205947,
+ "tokens_seen": 1447362560
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045372918756268814,
+ "loss": 1.4173,
+ "theoretical_loss": 3.555659727332618,
+ "tokens_seen": 1447428096
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004537131394182548,
+ "loss": 1.4036,
+ "theoretical_loss": 3.555645550280947,
+ "tokens_seen": 1447493632
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045369709127382154,
+ "loss": 1.3324,
+ "theoretical_loss": 3.5556313740508503,
+ "tokens_seen": 1447559168
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004536810431293882,
+ "loss": 1.347,
+ "theoretical_loss": 3.5556171986422425,
+ "tokens_seen": 1447624704
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045366499498495494,
+ "loss": 1.3555,
+ "theoretical_loss": 3.5556030240550385,
+ "tokens_seen": 1447690240
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004536489468405216,
+ "loss": 1.26,
+ "theoretical_loss": 3.5555888502891544,
+ "tokens_seen": 1447755776
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045363289869608834,
+ "loss": 1.3767,
+ "theoretical_loss": 3.555574677344505,
+ "tokens_seen": 1447821312
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000453616850551655,
+ "loss": 1.3768,
+ "theoretical_loss": 3.5555605052210053,
+ "tokens_seen": 1447886848
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004536008024072217,
+ "loss": 1.3414,
+ "theoretical_loss": 3.5555463339185707,
+ "tokens_seen": 1447952384
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535847542627884,
+ "loss": 1.3862,
+ "theoretical_loss": 3.555532163437116,
+ "tokens_seen": 1448017920
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535687061183551,
+ "loss": 1.4174,
+ "theoretical_loss": 3.555517993776558,
+ "tokens_seen": 1448083456
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535526579739218,
+ "loss": 1.3539,
+ "theoretical_loss": 3.5555038249368103,
+ "tokens_seen": 1448148992
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535366098294885,
+ "loss": 1.3942,
+ "theoretical_loss": 3.555489656917789,
+ "tokens_seen": 1448214528
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535205616850552,
+ "loss": 1.3464,
+ "theoretical_loss": 3.5554754897194103,
+ "tokens_seen": 1448280064
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 485883,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.440194845199585,
+ "objective/train/theoretical_loss": 3.5554613233415875,
+ "objective/train/tokens_used": 1468805600,
+ "theoretical_loss": 3.5554613233415875,
+ "tokens_seen": 1448345600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004535045135406219,
+ "loss": 1.3957,
+ "theoretical_loss": 3.5554613233415875,
+ "tokens_seen": 1448345600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004534884653961886,
+ "loss": 1.1841,
+ "theoretical_loss": 3.5554471577842373,
+ "tokens_seen": 1448411136
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004534724172517553,
+ "loss": 1.2246,
+ "theoretical_loss": 3.5554329930472752,
+ "tokens_seen": 1448476672
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045345636910732194,
+ "loss": 1.3249,
+ "theoretical_loss": 3.5554188291306157,
+ "tokens_seen": 1448542208
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004534403209628887,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5554046660341747,
+ "tokens_seen": 1448607744
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045342427281845534,
+ "loss": 1.3259,
+ "theoretical_loss": 3.5553905037578675,
+ "tokens_seen": 1448673280
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004534082246740221,
+ "loss": 1.3397,
+ "theoretical_loss": 3.55537634230161,
+ "tokens_seen": 1448738816
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045339217652958874,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5553621816653167,
+ "tokens_seen": 1448804352
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004533761283851555,
+ "loss": 1.2989,
+ "theoretical_loss": 3.5553480218489035,
+ "tokens_seen": 1448869888
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045336008024072214,
+ "loss": 1.3951,
+ "theoretical_loss": 3.5553338628522866,
+ "tokens_seen": 1448935424
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004533440320962889,
+ "loss": 1.2479,
+ "theoretical_loss": 3.55531970467538,
+ "tokens_seen": 1449000960
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045332798395185555,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5553055473181,
+ "tokens_seen": 1449066496
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004533119358074223,
+ "loss": 1.3126,
+ "theoretical_loss": 3.5552913907803623,
+ "tokens_seen": 1449132032
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045329588766298895,
+ "loss": 1.4397,
+ "theoretical_loss": 3.5552772350620816,
+ "tokens_seen": 1449197568
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004532798395185557,
+ "loss": 1.284,
+ "theoretical_loss": 3.555263080163174,
+ "tokens_seen": 1449263104
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045326379137412235,
+ "loss": 1.3189,
+ "theoretical_loss": 3.5552489260835554,
+ "tokens_seen": 1449328640
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004532477432296891,
+ "loss": 1.2923,
+ "theoretical_loss": 3.55523477282314,
+ "tokens_seen": 1449394176
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045323169508525575,
+ "loss": 1.384,
+ "theoretical_loss": 3.555220620381845,
+ "tokens_seen": 1449459712
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004532156469408225,
+ "loss": 1.3066,
+ "theoretical_loss": 3.555206468759585,
+ "tokens_seen": 1449525248
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045319959879638915,
+ "loss": 1.329,
+ "theoretical_loss": 3.5551923179562754,
+ "tokens_seen": 1449590784
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004531835506519559,
+ "loss": 1.2898,
+ "theoretical_loss": 3.5551781679718326,
+ "tokens_seen": 1449656320
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045316750250752255,
+ "loss": 1.4265,
+ "theoretical_loss": 3.5551640188061713,
+ "tokens_seen": 1449721856
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004531514543630893,
+ "loss": 1.2649,
+ "theoretical_loss": 3.555149870459208,
+ "tokens_seen": 1449787392
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045313540621865595,
+ "loss": 1.3998,
+ "theoretical_loss": 3.5551357229308573,
+ "tokens_seen": 1449852928
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004531193580742227,
+ "loss": 1.2399,
+ "theoretical_loss": 3.5551215762210355,
+ "tokens_seen": 1449918464
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045310330992978935,
+ "loss": 1.2495,
+ "theoretical_loss": 3.5551074303296586,
+ "tokens_seen": 1449984000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004530872617853561,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5550932852566413,
+ "tokens_seen": 1450049536
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045307121364092276,
+ "loss": 1.3231,
+ "theoretical_loss": 3.5550791410019,
+ "tokens_seen": 1450115072
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004530551654964895,
+ "loss": 1.2919,
+ "theoretical_loss": 3.55506499756535,
+ "tokens_seen": 1450180608
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045303911735205616,
+ "loss": 1.4347,
+ "theoretical_loss": 3.555050854946907,
+ "tokens_seen": 1450246144
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004530230692076229,
+ "loss": 1.3137,
+ "theoretical_loss": 3.5550367131464875,
+ "tokens_seen": 1450311680
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045300702106318956,
+ "loss": 1.2355,
+ "theoretical_loss": 3.555022572164006,
+ "tokens_seen": 1450377216
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004529909729187563,
+ "loss": 1.2252,
+ "theoretical_loss": 3.5550084319993793,
+ "tokens_seen": 1450442752
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045297492477432296,
+ "loss": 1.2676,
+ "theoretical_loss": 3.5549942926525224,
+ "tokens_seen": 1450508288
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004529588766298897,
+ "loss": 1.3313,
+ "theoretical_loss": 3.554980154123351,
+ "tokens_seen": 1450573824
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045294282848545636,
+ "loss": 1.2972,
+ "theoretical_loss": 3.554966016411782,
+ "tokens_seen": 1450639360
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004529267803410231,
+ "loss": 1.2923,
+ "theoretical_loss": 3.5549518795177297,
+ "tokens_seen": 1450704896
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045291073219658976,
+ "loss": 1.3751,
+ "theoretical_loss": 3.554937743441111,
+ "tokens_seen": 1450770432
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004528946840521565,
+ "loss": 1.347,
+ "theoretical_loss": 3.5549236081818414,
+ "tokens_seen": 1450835968
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045287863590772316,
+ "loss": 1.3123,
+ "theoretical_loss": 3.554909473739836,
+ "tokens_seen": 1450901504
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004528625877632899,
+ "loss": 1.3425,
+ "theoretical_loss": 3.554895340115012,
+ "tokens_seen": 1450967040
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004528465396188566,
+ "loss": 1.3134,
+ "theoretical_loss": 3.554881207307284,
+ "tokens_seen": 1451032576
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045283049147442326,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5548670753165688,
+ "tokens_seen": 1451098112
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045281444332999,
+ "loss": 1.1967,
+ "theoretical_loss": 3.554852944142781,
+ "tokens_seen": 1451163648
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045279839518555667,
+ "loss": 1.1971,
+ "theoretical_loss": 3.5548388137858384,
+ "tokens_seen": 1451229184
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004527823470411234,
+ "loss": 1.2516,
+ "theoretical_loss": 3.5548246842456557,
+ "tokens_seen": 1451294720
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045276629889669007,
+ "loss": 1.2867,
+ "theoretical_loss": 3.554810555522148,
+ "tokens_seen": 1451360256
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004527502507522568,
+ "loss": 1.3382,
+ "theoretical_loss": 3.554796427615233,
+ "tokens_seen": 1451425792
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045273420260782347,
+ "loss": 1.2944,
+ "theoretical_loss": 3.554782300524826,
+ "tokens_seen": 1451491328
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004527181544633902,
+ "loss": 1.3326,
+ "theoretical_loss": 3.5547681742508423,
+ "tokens_seen": 1451556864
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 487012,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2259671688079834,
+ "objective/train/theoretical_loss": 3.554754048793198,
+ "objective/train/tokens_used": 1472082400,
+ "theoretical_loss": 3.554754048793198,
+ "tokens_seen": 1451622400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045270210631895687,
+ "loss": 1.2535,
+ "theoretical_loss": 3.554754048793198,
+ "tokens_seen": 1451622400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004526860581745236,
+ "loss": 1.3089,
+ "theoretical_loss": 3.55473992415181,
+ "tokens_seen": 1451687936
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045267001003009027,
+ "loss": 1.313,
+ "theoretical_loss": 3.5547258003265934,
+ "tokens_seen": 1451753472
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000452653961885657,
+ "loss": 1.2195,
+ "theoretical_loss": 3.5547116773174645,
+ "tokens_seen": 1451819008
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045263791374122367,
+ "loss": 1.2883,
+ "theoretical_loss": 3.554697555124339,
+ "tokens_seen": 1451884544
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045262186559679043,
+ "loss": 1.335,
+ "theoretical_loss": 3.5546834337471336,
+ "tokens_seen": 1451950080
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004526058174523571,
+ "loss": 1.3661,
+ "theoretical_loss": 3.554669313185764,
+ "tokens_seen": 1452015616
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045258976930792383,
+ "loss": 1.3341,
+ "theoretical_loss": 3.5546551934401456,
+ "tokens_seen": 1452081152
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004525737211634905,
+ "loss": 1.3922,
+ "theoretical_loss": 3.5546410745101955,
+ "tokens_seen": 1452146688
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045255767301905723,
+ "loss": 1.3309,
+ "theoretical_loss": 3.554626956395829,
+ "tokens_seen": 1452212224
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004525416248746239,
+ "loss": 1.3254,
+ "theoretical_loss": 3.554612839096963,
+ "tokens_seen": 1452277760
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045252557673019063,
+ "loss": 1.2518,
+ "theoretical_loss": 3.5545987226135125,
+ "tokens_seen": 1452343296
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004525095285857573,
+ "loss": 1.2557,
+ "theoretical_loss": 3.5545846069453946,
+ "tokens_seen": 1452408832
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045249348044132403,
+ "loss": 1.3505,
+ "theoretical_loss": 3.554570492092525,
+ "tokens_seen": 1452474368
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004524774322968907,
+ "loss": 1.2946,
+ "theoretical_loss": 3.55455637805482,
+ "tokens_seen": 1452539904
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045246138415245743,
+ "loss": 1.2938,
+ "theoretical_loss": 3.5545422648321954,
+ "tokens_seen": 1452605440
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004524453360080241,
+ "loss": 1.4115,
+ "theoretical_loss": 3.5545281524245675,
+ "tokens_seen": 1452670976
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045242928786359083,
+ "loss": 1.256,
+ "theoretical_loss": 3.5545140408318527,
+ "tokens_seen": 1452736512
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004524132397191575,
+ "loss": 1.3221,
+ "theoretical_loss": 3.554499930053967,
+ "tokens_seen": 1452802048
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045239719157472424,
+ "loss": 1.3062,
+ "theoretical_loss": 3.5544858200908265,
+ "tokens_seen": 1452867584
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004523811434302909,
+ "loss": 1.3071,
+ "theoretical_loss": 3.5544717109423476,
+ "tokens_seen": 1452933120
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045236509528585764,
+ "loss": 1.275,
+ "theoretical_loss": 3.5544576026084465,
+ "tokens_seen": 1452998656
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004523490471414243,
+ "loss": 1.2948,
+ "theoretical_loss": 3.554443495089039,
+ "tokens_seen": 1453064192
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045233299899699104,
+ "loss": 1.3114,
+ "theoretical_loss": 3.554429388384042,
+ "tokens_seen": 1453129728
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004523169508525577,
+ "loss": 1.2334,
+ "theoretical_loss": 3.554415282493372,
+ "tokens_seen": 1453195264
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045230090270812444,
+ "loss": 1.3717,
+ "theoretical_loss": 3.5544011774169446,
+ "tokens_seen": 1453260800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004522848545636911,
+ "loss": 1.3352,
+ "theoretical_loss": 3.554387073154676,
+ "tokens_seen": 1453326336
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045226880641925784,
+ "loss": 1.3201,
+ "theoretical_loss": 3.554372969706482,
+ "tokens_seen": 1453391872
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004522527582748245,
+ "loss": 1.1697,
+ "theoretical_loss": 3.5543588670722808,
+ "tokens_seen": 1453457408
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045223671013039124,
+ "loss": 1.3069,
+ "theoretical_loss": 3.554344765251987,
+ "tokens_seen": 1453522944
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004522206619859579,
+ "loss": 1.2158,
+ "theoretical_loss": 3.5543306642455175,
+ "tokens_seen": 1453588480
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004522046138415246,
+ "loss": 1.2928,
+ "theoretical_loss": 3.5543165640527885,
+ "tokens_seen": 1453654016
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004521885656970913,
+ "loss": 1.2231,
+ "theoretical_loss": 3.554302464673717,
+ "tokens_seen": 1453719552
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000452172517552658,
+ "loss": 1.3529,
+ "theoretical_loss": 3.554288366108219,
+ "tokens_seen": 1453785088
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004521564694082247,
+ "loss": 1.3699,
+ "theoretical_loss": 3.55427426835621,
+ "tokens_seen": 1453850624
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004521404212637914,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5542601714176074,
+ "tokens_seen": 1453916160
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004521243731193581,
+ "loss": 1.2796,
+ "theoretical_loss": 3.554246075292328,
+ "tokens_seen": 1453981696
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004521083249749248,
+ "loss": 1.2764,
+ "theoretical_loss": 3.554231979980287,
+ "tokens_seen": 1454047232
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004520922768304915,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5542178854814015,
+ "tokens_seen": 1454112768
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004520762286860582,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5542037917955875,
+ "tokens_seen": 1454178304
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045206018054162484,
+ "loss": 1.3271,
+ "theoretical_loss": 3.5541896989227624,
+ "tokens_seen": 1454243840
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004520441323971916,
+ "loss": 1.3196,
+ "theoretical_loss": 3.5541756068628416,
+ "tokens_seen": 1454309376
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045202808425275824,
+ "loss": 1.2465,
+ "theoretical_loss": 3.5541615156157422,
+ "tokens_seen": 1454374912
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000452012036108325,
+ "loss": 1.3355,
+ "theoretical_loss": 3.5541474251813807,
+ "tokens_seen": 1454440448
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045199598796389164,
+ "loss": 1.3128,
+ "theoretical_loss": 3.5541333355596736,
+ "tokens_seen": 1454505984
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004519799398194584,
+ "loss": 1.266,
+ "theoretical_loss": 3.5541192467505365,
+ "tokens_seen": 1454571520
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045196389167502504,
+ "loss": 1.2017,
+ "theoretical_loss": 3.554105158753887,
+ "tokens_seen": 1454637056
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004519478435305918,
+ "loss": 1.2959,
+ "theoretical_loss": 3.5540910715696423,
+ "tokens_seen": 1454702592
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045193179538615845,
+ "loss": 1.2276,
+ "theoretical_loss": 3.5540769851977174,
+ "tokens_seen": 1454768128
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004519157472417252,
+ "loss": 1.2987,
+ "theoretical_loss": 3.554062899638029,
+ "tokens_seen": 1454833664
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 487903,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4405250549316406,
+ "objective/train/theoretical_loss": 3.5540488148904945,
+ "objective/train/tokens_used": 1475359200,
+ "theoretical_loss": 3.5540488148904945,
+ "tokens_seen": 1454899200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045189969909729185,
+ "loss": 1.313,
+ "theoretical_loss": 3.5540488148904945,
+ "tokens_seen": 1454899200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004518836509528586,
+ "loss": 1.3153,
+ "theoretical_loss": 3.5540347309550304,
+ "tokens_seen": 1454964736
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045186760280842525,
+ "loss": 1.3729,
+ "theoretical_loss": 3.554020647831553,
+ "tokens_seen": 1455030272
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000451851554663992,
+ "loss": 1.2942,
+ "theoretical_loss": 3.5540065655199786,
+ "tokens_seen": 1455095808
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045183550651955865,
+ "loss": 1.2284,
+ "theoretical_loss": 3.553992484020224,
+ "tokens_seen": 1455161344
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004518194583751254,
+ "loss": 1.3839,
+ "theoretical_loss": 3.5539784033322066,
+ "tokens_seen": 1455226880
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045180341023069205,
+ "loss": 1.2961,
+ "theoretical_loss": 3.5539643234558422,
+ "tokens_seen": 1455292416
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004517873620862588,
+ "loss": 1.28,
+ "theoretical_loss": 3.553950244391048,
+ "tokens_seen": 1455357952
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045177131394182545,
+ "loss": 1.2359,
+ "theoretical_loss": 3.55393616613774,
+ "tokens_seen": 1455423488
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004517552657973922,
+ "loss": 1.2961,
+ "theoretical_loss": 3.5539220886958356,
+ "tokens_seen": 1455489024
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045173921765295885,
+ "loss": 1.2413,
+ "theoretical_loss": 3.5539080120652513,
+ "tokens_seen": 1455554560
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004517231695085256,
+ "loss": 1.3649,
+ "theoretical_loss": 3.5538939362459034,
+ "tokens_seen": 1455620096
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045170712136409226,
+ "loss": 1.2859,
+ "theoretical_loss": 3.553879861237709,
+ "tokens_seen": 1455685632
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000451691073219659,
+ "loss": 1.287,
+ "theoretical_loss": 3.553865787040585,
+ "tokens_seen": 1455751168
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045167502507522566,
+ "loss": 1.3115,
+ "theoretical_loss": 3.5538517136544483,
+ "tokens_seen": 1455816704
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004516589769307924,
+ "loss": 1.2647,
+ "theoretical_loss": 3.553837641079215,
+ "tokens_seen": 1455882240
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045164292878635917,
+ "loss": 1.2529,
+ "theoretical_loss": 3.553823569314802,
+ "tokens_seen": 1455947776
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004516268806419258,
+ "loss": 1.3434,
+ "theoretical_loss": 3.5538094983611264,
+ "tokens_seen": 1456013312
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045161083249749257,
+ "loss": 1.1867,
+ "theoretical_loss": 3.553795428218105,
+ "tokens_seen": 1456078848
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004515947843530592,
+ "loss": 1.387,
+ "theoretical_loss": 3.5537813588856544,
+ "tokens_seen": 1456144384
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004515787362086259,
+ "loss": 1.3508,
+ "theoretical_loss": 3.553767290363692,
+ "tokens_seen": 1456209920
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004515626880641926,
+ "loss": 1.3123,
+ "theoretical_loss": 3.5537532226521336,
+ "tokens_seen": 1456275456
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004515466399197593,
+ "loss": 1.3477,
+ "theoretical_loss": 3.553739155750897,
+ "tokens_seen": 1456340992
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000451530591775326,
+ "loss": 1.3485,
+ "theoretical_loss": 3.553725089659898,
+ "tokens_seen": 1456406528
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004515145436308927,
+ "loss": 1.3496,
+ "theoretical_loss": 3.5537110243790546,
+ "tokens_seen": 1456472064
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004514984954864594,
+ "loss": 1.4036,
+ "theoretical_loss": 3.553696959908283,
+ "tokens_seen": 1456537600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004514824473420261,
+ "loss": 1.3022,
+ "theoretical_loss": 3.553682896247501,
+ "tokens_seen": 1456603136
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004514663991975928,
+ "loss": 1.2692,
+ "theoretical_loss": 3.5536688333966246,
+ "tokens_seen": 1456668672
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004514503510531595,
+ "loss": 1.1335,
+ "theoretical_loss": 3.5536547713555704,
+ "tokens_seen": 1456734208
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045143430290872617,
+ "loss": 1.3056,
+ "theoretical_loss": 3.5536407101242564,
+ "tokens_seen": 1456799744
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004514182547642929,
+ "loss": 1.2785,
+ "theoretical_loss": 3.553626649702599,
+ "tokens_seen": 1456865280
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045140220661985957,
+ "loss": 1.2605,
+ "theoretical_loss": 3.5536125900905153,
+ "tokens_seen": 1456930816
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004513861584754263,
+ "loss": 1.2189,
+ "theoretical_loss": 3.553598531287922,
+ "tokens_seen": 1456996352
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045137011033099297,
+ "loss": 1.2632,
+ "theoretical_loss": 3.5535844732947366,
+ "tokens_seen": 1457061888
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004513540621865597,
+ "loss": 1.3776,
+ "theoretical_loss": 3.553570416110875,
+ "tokens_seen": 1457127424
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045133801404212637,
+ "loss": 1.3405,
+ "theoretical_loss": 3.553556359736256,
+ "tokens_seen": 1457192960
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004513219658976931,
+ "loss": 1.3776,
+ "theoretical_loss": 3.553542304170795,
+ "tokens_seen": 1457258496
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045130591775325977,
+ "loss": 1.3449,
+ "theoretical_loss": 3.55352824941441,
+ "tokens_seen": 1457324032
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004512898696088265,
+ "loss": 1.2835,
+ "theoretical_loss": 3.5535141954670175,
+ "tokens_seen": 1457389568
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045127382146439317,
+ "loss": 1.3431,
+ "theoretical_loss": 3.5535001423285344,
+ "tokens_seen": 1457455104
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004512577733199599,
+ "loss": 1.3769,
+ "theoretical_loss": 3.5534860899988785,
+ "tokens_seen": 1457520640
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004512417251755266,
+ "loss": 1.3933,
+ "theoretical_loss": 3.5534720384779668,
+ "tokens_seen": 1457586176
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045122567703109333,
+ "loss": 1.3153,
+ "theoretical_loss": 3.5534579877657153,
+ "tokens_seen": 1457651712
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045120962888666,
+ "loss": 1.3405,
+ "theoretical_loss": 3.5534439378620424,
+ "tokens_seen": 1457717248
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045119358074222673,
+ "loss": 1.2997,
+ "theoretical_loss": 3.553429888766865,
+ "tokens_seen": 1457782784
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004511775325977934,
+ "loss": 1.3291,
+ "theoretical_loss": 3.553415840480099,
+ "tokens_seen": 1457848320
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045116148445336013,
+ "loss": 1.3659,
+ "theoretical_loss": 3.5534017930016635,
+ "tokens_seen": 1457913856
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004511454363089268,
+ "loss": 1.2452,
+ "theoretical_loss": 3.5533877463314743,
+ "tokens_seen": 1457979392
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045112938816449353,
+ "loss": 1.4578,
+ "theoretical_loss": 3.553373700469449,
+ "tokens_seen": 1458044928
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004511133400200602,
+ "loss": 1.2215,
+ "theoretical_loss": 3.5533596554155045,
+ "tokens_seen": 1458110464
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 489171,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.413554310798645,
+ "objective/train/theoretical_loss": 3.5533456111695583,
+ "objective/train/tokens_used": 1478636000,
+ "theoretical_loss": 3.5533456111695583,
+ "tokens_seen": 1458176000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045109729187562693,
+ "loss": 1.3237,
+ "theoretical_loss": 3.5533456111695583,
+ "tokens_seen": 1458176000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004510812437311936,
+ "loss": 1.3097,
+ "theoretical_loss": 3.5533315677315276,
+ "tokens_seen": 1458241536
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045106519558676033,
+ "loss": 1.3404,
+ "theoretical_loss": 3.5533175251013294,
+ "tokens_seen": 1458307072
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000451049147442327,
+ "loss": 1.2486,
+ "theoretical_loss": 3.553303483278881,
+ "tokens_seen": 1458372608
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045103309929789374,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5532894422640995,
+ "tokens_seen": 1458438144
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004510170511534604,
+ "loss": 1.3998,
+ "theoretical_loss": 3.5532754020569026,
+ "tokens_seen": 1458503680
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045100100300902714,
+ "loss": 1.3538,
+ "theoretical_loss": 3.5532613626572074,
+ "tokens_seen": 1458569216
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004509849548645938,
+ "loss": 1.3451,
+ "theoretical_loss": 3.553247324064931,
+ "tokens_seen": 1458634752
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045096890672016054,
+ "loss": 1.3197,
+ "theoretical_loss": 3.5532332862799914,
+ "tokens_seen": 1458700288
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004509528585757272,
+ "loss": 1.3852,
+ "theoretical_loss": 3.5532192493023045,
+ "tokens_seen": 1458765824
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045093681043129394,
+ "loss": 1.2242,
+ "theoretical_loss": 3.5532052131317884,
+ "tokens_seen": 1458831360
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004509207622868606,
+ "loss": 1.2423,
+ "theoretical_loss": 3.553191177768361,
+ "tokens_seen": 1458896896
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045090471414242734,
+ "loss": 1.3741,
+ "theoretical_loss": 3.553177143211938,
+ "tokens_seen": 1458962432
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000450888665997994,
+ "loss": 1.3684,
+ "theoretical_loss": 3.5531631094624387,
+ "tokens_seen": 1459027968
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045087261785356074,
+ "loss": 1.3882,
+ "theoretical_loss": 3.5531490765197797,
+ "tokens_seen": 1459093504
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004508565697091274,
+ "loss": 1.3671,
+ "theoretical_loss": 3.553135044383878,
+ "tokens_seen": 1459159040
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045084052156469414,
+ "loss": 1.3389,
+ "theoretical_loss": 3.553121013054651,
+ "tokens_seen": 1459224576
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004508244734202608,
+ "loss": 1.3312,
+ "theoretical_loss": 3.5531069825320167,
+ "tokens_seen": 1459290112
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004508084252758275,
+ "loss": 1.3107,
+ "theoretical_loss": 3.553092952815892,
+ "tokens_seen": 1459355648
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004507923771313942,
+ "loss": 1.335,
+ "theoretical_loss": 3.5530789239061944,
+ "tokens_seen": 1459421184
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004507763289869609,
+ "loss": 1.3087,
+ "theoretical_loss": 3.5530648958028417,
+ "tokens_seen": 1459486720
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004507602808425276,
+ "loss": 1.2736,
+ "theoretical_loss": 3.5530508685057507,
+ "tokens_seen": 1459552256
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004507442326980943,
+ "loss": 1.3638,
+ "theoretical_loss": 3.5530368420148397,
+ "tokens_seen": 1459617792
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000450728184553661,
+ "loss": 1.3524,
+ "theoretical_loss": 3.5530228163300253,
+ "tokens_seen": 1459683328
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004507121364092277,
+ "loss": 1.3522,
+ "theoretical_loss": 3.5530087914512256,
+ "tokens_seen": 1459748864
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004506960882647944,
+ "loss": 1.2369,
+ "theoretical_loss": 3.5529947673783573,
+ "tokens_seen": 1459814400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004506800401203611,
+ "loss": 1.2734,
+ "theoretical_loss": 3.5529807441113395,
+ "tokens_seen": 1459879936
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045066399197592774,
+ "loss": 1.3226,
+ "theoretical_loss": 3.5529667216500878,
+ "tokens_seen": 1459945472
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004506479438314945,
+ "loss": 1.3201,
+ "theoretical_loss": 3.5529526999945213,
+ "tokens_seen": 1460011008
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045063189568706114,
+ "loss": 1.3557,
+ "theoretical_loss": 3.5529386791445567,
+ "tokens_seen": 1460076544
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004506158475426279,
+ "loss": 1.3174,
+ "theoretical_loss": 3.552924659100112,
+ "tokens_seen": 1460142080
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045059979939819454,
+ "loss": 1.3624,
+ "theoretical_loss": 3.552910639861104,
+ "tokens_seen": 1460207616
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004505837512537613,
+ "loss": 1.308,
+ "theoretical_loss": 3.5528966214274513,
+ "tokens_seen": 1460273152
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045056770310932795,
+ "loss": 1.2808,
+ "theoretical_loss": 3.5528826037990706,
+ "tokens_seen": 1460338688
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004505516549648947,
+ "loss": 1.2842,
+ "theoretical_loss": 3.55286858697588,
+ "tokens_seen": 1460404224
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045053560682046135,
+ "loss": 1.2757,
+ "theoretical_loss": 3.552854570957797,
+ "tokens_seen": 1460469760
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004505195586760281,
+ "loss": 1.2751,
+ "theoretical_loss": 3.5528405557447393,
+ "tokens_seen": 1460535296
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045050351053159475,
+ "loss": 1.3406,
+ "theoretical_loss": 3.552826541336625,
+ "tokens_seen": 1460600832
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004504874623871615,
+ "loss": 1.2714,
+ "theoretical_loss": 3.5528125277333706,
+ "tokens_seen": 1460666368
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045047141424272815,
+ "loss": 1.3913,
+ "theoretical_loss": 3.552798514934895,
+ "tokens_seen": 1460731904
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004504553660982949,
+ "loss": 1.386,
+ "theoretical_loss": 3.5527845029411145,
+ "tokens_seen": 1460797440
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045043931795386166,
+ "loss": 1.2745,
+ "theoretical_loss": 3.5527704917519483,
+ "tokens_seen": 1460862976
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004504232698094283,
+ "loss": 1.2067,
+ "theoretical_loss": 3.5527564813673127,
+ "tokens_seen": 1460928512
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045040722166499506,
+ "loss": 1.2071,
+ "theoretical_loss": 3.552742471787127,
+ "tokens_seen": 1460994048
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004503911735205617,
+ "loss": 1.4215,
+ "theoretical_loss": 3.5527284630113076,
+ "tokens_seen": 1461059584
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045037512537612846,
+ "loss": 1.4245,
+ "theoretical_loss": 3.5527144550397725,
+ "tokens_seen": 1461125120
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004503590772316951,
+ "loss": 1.24,
+ "theoretical_loss": 3.5527004478724398,
+ "tokens_seen": 1461190656
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045034302908726186,
+ "loss": 1.1814,
+ "theoretical_loss": 3.552686441509227,
+ "tokens_seen": 1461256192
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004503269809428285,
+ "loss": 1.3061,
+ "theoretical_loss": 3.5526724359500523,
+ "tokens_seen": 1461321728
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045031093279839526,
+ "loss": 1.3079,
+ "theoretical_loss": 3.552658431194833,
+ "tokens_seen": 1461387264
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 490450,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2836272716522217,
+ "objective/train/theoretical_loss": 3.552644427243487,
+ "objective/train/tokens_used": 1481912800,
+ "theoretical_loss": 3.552644427243487,
+ "tokens_seen": 1461452800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004502948846539619,
+ "loss": 1.2955,
+ "theoretical_loss": 3.552644427243487,
+ "tokens_seen": 1461452800
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045027883650952867,
+ "loss": 1.4117,
+ "theoretical_loss": 3.5526304240959323,
+ "tokens_seen": 1461518336
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004502627883650953,
+ "loss": 1.2808,
+ "theoretical_loss": 3.5526164217520866,
+ "tokens_seen": 1461583872
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045024674022066207,
+ "loss": 1.2112,
+ "theoretical_loss": 3.5526024202118673,
+ "tokens_seen": 1461649408
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004502306920762287,
+ "loss": 1.3372,
+ "theoretical_loss": 3.5525884194751933,
+ "tokens_seen": 1461714944
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045021464393179547,
+ "loss": 1.3712,
+ "theoretical_loss": 3.5525744195419815,
+ "tokens_seen": 1461780480
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501985957873621,
+ "loss": 1.2657,
+ "theoretical_loss": 3.55256042041215,
+ "tokens_seen": 1461846016
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501825476429288,
+ "loss": 1.236,
+ "theoretical_loss": 3.5525464220856176,
+ "tokens_seen": 1461911552
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501664994984955,
+ "loss": 1.3177,
+ "theoretical_loss": 3.5525324245623007,
+ "tokens_seen": 1461977088
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501504513540622,
+ "loss": 1.3865,
+ "theoretical_loss": 3.5525184278421182,
+ "tokens_seen": 1462042624
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501344032096289,
+ "loss": 1.2844,
+ "theoretical_loss": 3.552504431924987,
+ "tokens_seen": 1462108160
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501183550651956,
+ "loss": 1.2484,
+ "theoretical_loss": 3.552490436810827,
+ "tokens_seen": 1462173696
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004501023069207623,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5524764424995543,
+ "tokens_seen": 1462239232
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000450086258776329,
+ "loss": 1.3489,
+ "theoretical_loss": 3.552462448991087,
+ "tokens_seen": 1462304768
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004500702106318957,
+ "loss": 1.3081,
+ "theoretical_loss": 3.5524484562853447,
+ "tokens_seen": 1462370304
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004500541624874624,
+ "loss": 1.3625,
+ "theoretical_loss": 3.5524344643822436,
+ "tokens_seen": 1462435840
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045003811434302907,
+ "loss": 1.2521,
+ "theoretical_loss": 3.552420473281702,
+ "tokens_seen": 1462501376
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004500220661985958,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5524064829836384,
+ "tokens_seen": 1462566912
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00045000601805416247,
+ "loss": 1.283,
+ "theoretical_loss": 3.5523924934879707,
+ "tokens_seen": 1462632448
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004499899699097292,
+ "loss": 1.3991,
+ "theoretical_loss": 3.552378504794617,
+ "tokens_seen": 1462697984
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044997392176529587,
+ "loss": 1.3523,
+ "theoretical_loss": 3.552364516903495,
+ "tokens_seen": 1462763520
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004499578736208626,
+ "loss": 1.2914,
+ "theoretical_loss": 3.552350529814523,
+ "tokens_seen": 1462829056
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044994182547642927,
+ "loss": 1.2439,
+ "theoretical_loss": 3.552336543527619,
+ "tokens_seen": 1462894592
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000449925777331996,
+ "loss": 1.1822,
+ "theoretical_loss": 3.552322558042701,
+ "tokens_seen": 1462960128
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044990972918756267,
+ "loss": 1.2918,
+ "theoretical_loss": 3.5523085733596877,
+ "tokens_seen": 1463025664
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004498936810431294,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5522945894784965,
+ "tokens_seen": 1463091200
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004498776328986961,
+ "loss": 1.186,
+ "theoretical_loss": 3.5522806063990457,
+ "tokens_seen": 1463156736
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044986158475426283,
+ "loss": 1.3742,
+ "theoretical_loss": 3.5522666241212533,
+ "tokens_seen": 1463222272
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004498455366098295,
+ "loss": 1.2982,
+ "theoretical_loss": 3.552252642645038,
+ "tokens_seen": 1463287808
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044982948846539623,
+ "loss": 1.394,
+ "theoretical_loss": 3.552238661970317,
+ "tokens_seen": 1463353344
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004498134403209629,
+ "loss": 1.1875,
+ "theoretical_loss": 3.552224682097009,
+ "tokens_seen": 1463418880
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044979739217652963,
+ "loss": 1.3574,
+ "theoretical_loss": 3.5522107030250325,
+ "tokens_seen": 1463484416
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004497813440320963,
+ "loss": 1.2868,
+ "theoretical_loss": 3.552196724754305,
+ "tokens_seen": 1463549952
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044976529588766303,
+ "loss": 1.335,
+ "theoretical_loss": 3.5521827472847454,
+ "tokens_seen": 1463615488
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004497492477432297,
+ "loss": 1.2571,
+ "theoretical_loss": 3.5521687706162712,
+ "tokens_seen": 1463681024
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044973319959879643,
+ "loss": 1.2657,
+ "theoretical_loss": 3.5521547947488017,
+ "tokens_seen": 1463746560
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004497171514543631,
+ "loss": 1.2836,
+ "theoretical_loss": 3.5521408196822537,
+ "tokens_seen": 1463812096
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044970110330992983,
+ "loss": 1.2512,
+ "theoretical_loss": 3.552126845416546,
+ "tokens_seen": 1463877632
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004496850551654965,
+ "loss": 1.3302,
+ "theoretical_loss": 3.5521128719515978,
+ "tokens_seen": 1463943168
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044966900702106324,
+ "loss": 1.2591,
+ "theoretical_loss": 3.5520988992873264,
+ "tokens_seen": 1464008704
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004496529588766299,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5520849274236497,
+ "tokens_seen": 1464074240
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044963691073219664,
+ "loss": 1.3326,
+ "theoretical_loss": 3.5520709563604873,
+ "tokens_seen": 1464139776
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004496208625877633,
+ "loss": 1.3833,
+ "theoretical_loss": 3.552056986097756,
+ "tokens_seen": 1464205312
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044960481444333004,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5520430166353756,
+ "tokens_seen": 1464270848
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004495887662988967,
+ "loss": 1.3173,
+ "theoretical_loss": 3.5520290479732637,
+ "tokens_seen": 1464336384
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044957271815446344,
+ "loss": 1.2926,
+ "theoretical_loss": 3.552015080111338,
+ "tokens_seen": 1464401920
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004495566700100301,
+ "loss": 1.2533,
+ "theoretical_loss": 3.552001113049518,
+ "tokens_seen": 1464467456
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044954062186559684,
+ "loss": 1.1938,
+ "theoretical_loss": 3.551987146787722,
+ "tokens_seen": 1464532992
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004495245737211635,
+ "loss": 1.2698,
+ "theoretical_loss": 3.5519731813258675,
+ "tokens_seen": 1464598528
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044950852557673024,
+ "loss": 1.1891,
+ "theoretical_loss": 3.5519592166638727,
+ "tokens_seen": 1464664064
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 491422,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3803954124450684,
+ "objective/train/theoretical_loss": 3.5519452528016577,
+ "objective/train/tokens_used": 1485189600,
+ "theoretical_loss": 3.5519452528016577,
+ "tokens_seen": 1464729600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004494924774322969,
+ "loss": 1.3201,
+ "theoretical_loss": 3.5519452528016577,
+ "tokens_seen": 1464729600
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044947642928786364,
+ "loss": 1.2366,
+ "theoretical_loss": 3.5519312897391395,
+ "tokens_seen": 1464795136
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004494603811434303,
+ "loss": 1.3456,
+ "theoretical_loss": 3.551917327476237,
+ "tokens_seen": 1464860672
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044944433299899704,
+ "loss": 1.2998,
+ "theoretical_loss": 3.5519033660128683,
+ "tokens_seen": 1464926208
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004494282848545637,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5518894053489523,
+ "tokens_seen": 1464991744
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004494122367101304,
+ "loss": 1.3636,
+ "theoretical_loss": 3.551875445484407,
+ "tokens_seen": 1465057280
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493961885656971,
+ "loss": 1.3595,
+ "theoretical_loss": 3.551861486419152,
+ "tokens_seen": 1465122816
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493801404212638,
+ "loss": 1.3471,
+ "theoretical_loss": 3.551847528153104,
+ "tokens_seen": 1465188352
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493640922768305,
+ "loss": 1.2771,
+ "theoretical_loss": 3.551833570686182,
+ "tokens_seen": 1465253888
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493480441323972,
+ "loss": 1.3855,
+ "theoretical_loss": 3.5518196140183056,
+ "tokens_seen": 1465319424
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493319959879639,
+ "loss": 1.3506,
+ "theoretical_loss": 3.5518056581493926,
+ "tokens_seen": 1465384960
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004493159478435306,
+ "loss": 1.2656,
+ "theoretical_loss": 3.5517917030793615,
+ "tokens_seen": 1465450496
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004492998996990973,
+ "loss": 1.3191,
+ "theoretical_loss": 3.5517777488081315,
+ "tokens_seen": 1465516032
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000449283851554664,
+ "loss": 1.3627,
+ "theoretical_loss": 3.55176379533562,
+ "tokens_seen": 1465581568
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044926780341023064,
+ "loss": 1.2993,
+ "theoretical_loss": 3.5517498426617458,
+ "tokens_seen": 1465647104
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004492517552657974,
+ "loss": 1.2695,
+ "theoretical_loss": 3.5517358907864285,
+ "tokens_seen": 1465712640
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044923570712136415,
+ "loss": 1.3515,
+ "theoretical_loss": 3.5517219397095854,
+ "tokens_seen": 1465778176
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004492196589769308,
+ "loss": 1.2987,
+ "theoretical_loss": 3.5517079894311365,
+ "tokens_seen": 1465843712
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044920361083249755,
+ "loss": 1.3813,
+ "theoretical_loss": 3.551694039950999,
+ "tokens_seen": 1465909248
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004491875626880642,
+ "loss": 1.2322,
+ "theoretical_loss": 3.551680091269092,
+ "tokens_seen": 1465974784
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044917151454363096,
+ "loss": 1.2963,
+ "theoretical_loss": 3.551666143385335,
+ "tokens_seen": 1466040320
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004491554663991976,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5516521962996457,
+ "tokens_seen": 1466105856
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044913941825476436,
+ "loss": 1.1739,
+ "theoretical_loss": 3.551638250011943,
+ "tokens_seen": 1466171392
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000449123370110331,
+ "loss": 1.3681,
+ "theoretical_loss": 3.5516243045221456,
+ "tokens_seen": 1466236928
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044910732196589776,
+ "loss": 1.357,
+ "theoretical_loss": 3.551610359830172,
+ "tokens_seen": 1466302464
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004490912738214644,
+ "loss": 1.2856,
+ "theoretical_loss": 3.5515964159359417,
+ "tokens_seen": 1466368000
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044907522567703116,
+ "loss": 1.3846,
+ "theoretical_loss": 3.5515824728393723,
+ "tokens_seen": 1466433536
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004490591775325978,
+ "loss": 1.2296,
+ "theoretical_loss": 3.551568530540383,
+ "tokens_seen": 1466499072
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044904312938816456,
+ "loss": 1.3763,
+ "theoretical_loss": 3.551554589038893,
+ "tokens_seen": 1466564608
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004490270812437312,
+ "loss": 1.3708,
+ "theoretical_loss": 3.55154064833482,
+ "tokens_seen": 1466630144
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044901103309929796,
+ "loss": 1.3562,
+ "theoretical_loss": 3.551526708428084,
+ "tokens_seen": 1466695680
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004489949849548646,
+ "loss": 1.3149,
+ "theoretical_loss": 3.5515127693186033,
+ "tokens_seen": 1466761216
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044897893681043136,
+ "loss": 1.2556,
+ "theoretical_loss": 3.551498831006296,
+ "tokens_seen": 1466826752
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000448962888665998,
+ "loss": 1.2372,
+ "theoretical_loss": 3.5514848934910814,
+ "tokens_seen": 1466892288
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044894684052156476,
+ "loss": 1.2254,
+ "theoretical_loss": 3.5514709567728784,
+ "tokens_seen": 1466957824
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004489307923771314,
+ "loss": 1.321,
+ "theoretical_loss": 3.551457020851606,
+ "tokens_seen": 1467023360
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044891474423269817,
+ "loss": 1.224,
+ "theoretical_loss": 3.5514430857271826,
+ "tokens_seen": 1467088896
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004488986960882648,
+ "loss": 1.3188,
+ "theoretical_loss": 3.5514291513995273,
+ "tokens_seen": 1467154432
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044888264794383157,
+ "loss": 1.2998,
+ "theoretical_loss": 3.551415217868559,
+ "tokens_seen": 1467219968
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004488665997993982,
+ "loss": 1.2508,
+ "theoretical_loss": 3.5514012851341965,
+ "tokens_seen": 1467285504
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044885055165496497,
+ "loss": 1.3836,
+ "theoretical_loss": 3.5513873531963585,
+ "tokens_seen": 1467351040
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004488345035105316,
+ "loss": 1.2725,
+ "theoretical_loss": 3.551373422054964,
+ "tokens_seen": 1467416576
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044881845536609837,
+ "loss": 1.269,
+ "theoretical_loss": 3.551359491709932,
+ "tokens_seen": 1467482112
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000448802407221665,
+ "loss": 1.3287,
+ "theoretical_loss": 3.551345562161181,
+ "tokens_seen": 1467547648
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044878635907723177,
+ "loss": 1.2812,
+ "theoretical_loss": 3.5513316334086307,
+ "tokens_seen": 1467613184
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004487703109327984,
+ "loss": 1.2979,
+ "theoretical_loss": 3.5513177054522,
+ "tokens_seen": 1467678720
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004487542627883651,
+ "loss": 1.3208,
+ "theoretical_loss": 3.551303778291807,
+ "tokens_seen": 1467744256
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004487382146439318,
+ "loss": 1.3323,
+ "theoretical_loss": 3.551289851927371,
+ "tokens_seen": 1467809792
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004487221664994985,
+ "loss": 1.2646,
+ "theoretical_loss": 3.5512759263588114,
+ "tokens_seen": 1467875328
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004487061183550652,
+ "loss": 1.3031,
+ "theoretical_loss": 3.5512620015860463,
+ "tokens_seen": 1467940864
+ },
+ {
+ "epoch": 0.44,
+ "objective/train/docs_used": 492285,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1124306917190552,
+ "objective/train/theoretical_loss": 3.551248077608996,
+ "objective/train/tokens_used": 1488466400,
+ "theoretical_loss": 3.551248077608996,
+ "tokens_seen": 1468006400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004486900702106319,
+ "loss": 1.2451,
+ "theoretical_loss": 3.551248077608996,
+ "tokens_seen": 1468006400
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004486740220661986,
+ "loss": 1.2949,
+ "theoretical_loss": 3.5512341544275783,
+ "tokens_seen": 1468071936
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004486579739217653,
+ "loss": 1.2772,
+ "theoretical_loss": 3.5512202320417128,
+ "tokens_seen": 1468137472
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.000448641925777332,
+ "loss": 1.3196,
+ "theoretical_loss": 3.551206310451319,
+ "tokens_seen": 1468203008
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004486258776328987,
+ "loss": 1.3246,
+ "theoretical_loss": 3.5511923896563147,
+ "tokens_seen": 1468268544
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044860982948846537,
+ "loss": 1.3011,
+ "theoretical_loss": 3.5511784696566204,
+ "tokens_seen": 1468334080
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.0004485937813440321,
+ "loss": 1.2091,
+ "theoretical_loss": 3.5511645504521536,
+ "tokens_seen": 1468399616
+ },
+ {
+ "epoch": 0.44,
+ "learning_rate": 0.00044857773319959877,
+ "loss": 1.2992,
+ "theoretical_loss": 3.5511506320428348,
+ "tokens_seen": 1468465152
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004485616850551655,
+ "loss": 1.2837,
+ "theoretical_loss": 3.551136714428582,
+ "tokens_seen": 1468530688
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044854563691073217,
+ "loss": 1.2785,
+ "theoretical_loss": 3.551122797609315,
+ "tokens_seen": 1468596224
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004485295887662989,
+ "loss": 1.2684,
+ "theoretical_loss": 3.551108881584953,
+ "tokens_seen": 1468661760
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004485135406218656,
+ "loss": 1.2748,
+ "theoretical_loss": 3.5510949663554148,
+ "tokens_seen": 1468727296
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044849749247743233,
+ "loss": 1.3099,
+ "theoretical_loss": 3.5510810519206197,
+ "tokens_seen": 1468792832
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000448481444332999,
+ "loss": 1.3105,
+ "theoretical_loss": 3.5510671382804864,
+ "tokens_seen": 1468858368
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044846539618856573,
+ "loss": 1.1873,
+ "theoretical_loss": 3.5510532254349343,
+ "tokens_seen": 1468923904
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004484493480441324,
+ "loss": 1.3178,
+ "theoretical_loss": 3.551039313383883,
+ "tokens_seen": 1468989440
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044843329989969913,
+ "loss": 1.3457,
+ "theoretical_loss": 3.5510254021272516,
+ "tokens_seen": 1469054976
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004484172517552658,
+ "loss": 1.3076,
+ "theoretical_loss": 3.5510114916649584,
+ "tokens_seen": 1469120512
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044840120361083253,
+ "loss": 1.332,
+ "theoretical_loss": 3.5509975819969233,
+ "tokens_seen": 1469186048
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004483851554663992,
+ "loss": 1.372,
+ "theoretical_loss": 3.550983673123066,
+ "tokens_seen": 1469251584
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044836910732196593,
+ "loss": 1.2497,
+ "theoretical_loss": 3.5509697650433054,
+ "tokens_seen": 1469317120
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004483530591775326,
+ "loss": 1.338,
+ "theoretical_loss": 3.55095585775756,
+ "tokens_seen": 1469382656
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044833701103309933,
+ "loss": 1.1832,
+ "theoretical_loss": 3.5509419512657496,
+ "tokens_seen": 1469448192
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000448320962888666,
+ "loss": 1.2157,
+ "theoretical_loss": 3.550928045567794,
+ "tokens_seen": 1469513728
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044830491474423274,
+ "loss": 1.222,
+ "theoretical_loss": 3.550914140663612,
+ "tokens_seen": 1469579264
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004482888665997994,
+ "loss": 1.2205,
+ "theoretical_loss": 3.550900236553123,
+ "tokens_seen": 1469644800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044827281845536614,
+ "loss": 1.2296,
+ "theoretical_loss": 3.5508863332362455,
+ "tokens_seen": 1469710336
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004482567703109328,
+ "loss": 1.3189,
+ "theoretical_loss": 3.5508724307129,
+ "tokens_seen": 1469775872
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044824072216649954,
+ "loss": 1.2367,
+ "theoretical_loss": 3.5508585289830052,
+ "tokens_seen": 1469841408
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004482246740220662,
+ "loss": 1.288,
+ "theoretical_loss": 3.5508446280464807,
+ "tokens_seen": 1469906944
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044820862587763294,
+ "loss": 1.2607,
+ "theoretical_loss": 3.5508307279032456,
+ "tokens_seen": 1469972480
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004481925777331996,
+ "loss": 1.1821,
+ "theoretical_loss": 3.5508168285532196,
+ "tokens_seen": 1470038016
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044817652958876634,
+ "loss": 1.3742,
+ "theoretical_loss": 3.5508029299963217,
+ "tokens_seen": 1470103552
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000448160481444333,
+ "loss": 1.3117,
+ "theoretical_loss": 3.5507890322324718,
+ "tokens_seen": 1470169088
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044814443329989974,
+ "loss": 1.1871,
+ "theoretical_loss": 3.5507751352615884,
+ "tokens_seen": 1470234624
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004481283851554664,
+ "loss": 1.2141,
+ "theoretical_loss": 3.5507612390835916,
+ "tokens_seen": 1470300160
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044811233701103314,
+ "loss": 1.2809,
+ "theoretical_loss": 3.550747343698401,
+ "tokens_seen": 1470365696
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004480962888665998,
+ "loss": 1.3651,
+ "theoretical_loss": 3.5507334491059357,
+ "tokens_seen": 1470431232
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044808024072216654,
+ "loss": 1.2923,
+ "theoretical_loss": 3.550719555306115,
+ "tokens_seen": 1470496768
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004480641925777332,
+ "loss": 1.3626,
+ "theoretical_loss": 3.550705662298859,
+ "tokens_seen": 1470562304
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044804814443329995,
+ "loss": 1.2943,
+ "theoretical_loss": 3.550691770084086,
+ "tokens_seen": 1470627840
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044803209628886665,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5506778786617166,
+ "tokens_seen": 1470693376
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044801604814443335,
+ "loss": 1.1644,
+ "theoretical_loss": 3.55066398803167,
+ "tokens_seen": 1470758912
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044800000000000005,
+ "loss": 1.3377,
+ "theoretical_loss": 3.5506500981938656,
+ "tokens_seen": 1470824448
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004479839518555667,
+ "loss": 1.4048,
+ "theoretical_loss": 3.5506362091482226,
+ "tokens_seen": 1470889984
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044796790371113345,
+ "loss": 1.2446,
+ "theoretical_loss": 3.550622320894661,
+ "tokens_seen": 1470955520
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004479518555667001,
+ "loss": 1.2955,
+ "theoretical_loss": 3.5506084334331005,
+ "tokens_seen": 1471021056
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044793580742226685,
+ "loss": 1.2552,
+ "theoretical_loss": 3.5505945467634596,
+ "tokens_seen": 1471086592
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004479197592778335,
+ "loss": 1.2722,
+ "theoretical_loss": 3.550580660885659,
+ "tokens_seen": 1471152128
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044790371113340025,
+ "loss": 1.3282,
+ "theoretical_loss": 3.550566775799618,
+ "tokens_seen": 1471217664
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 493595,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.409515619277954,
+ "objective/train/theoretical_loss": 3.550552891505256,
+ "objective/train/tokens_used": 1491743200,
+ "theoretical_loss": 3.550552891505256,
+ "tokens_seen": 1471283200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004478876629889669,
+ "loss": 1.272,
+ "theoretical_loss": 3.550552891505256,
+ "tokens_seen": 1471283200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044787161484453365,
+ "loss": 1.2476,
+ "theoretical_loss": 3.5505390080024926,
+ "tokens_seen": 1471348736
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004478555667001003,
+ "loss": 1.3048,
+ "theoretical_loss": 3.550525125291248,
+ "tokens_seen": 1471414272
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044783951855566705,
+ "loss": 1.2262,
+ "theoretical_loss": 3.5505112433714405,
+ "tokens_seen": 1471479808
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004478234704112337,
+ "loss": 1.2819,
+ "theoretical_loss": 3.550497362242991,
+ "tokens_seen": 1471545344
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044780742226680046,
+ "loss": 1.2758,
+ "theoretical_loss": 3.5504834819058186,
+ "tokens_seen": 1471610880
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004477913741223671,
+ "loss": 1.2836,
+ "theoretical_loss": 3.550469602359843,
+ "tokens_seen": 1471676416
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044777532597793386,
+ "loss": 1.3172,
+ "theoretical_loss": 3.5504557236049843,
+ "tokens_seen": 1471741952
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004477592778335005,
+ "loss": 1.3781,
+ "theoretical_loss": 3.5504418456411613,
+ "tokens_seen": 1471807488
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044774322968906726,
+ "loss": 1.4068,
+ "theoretical_loss": 3.550427968468294,
+ "tokens_seen": 1471873024
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004477271815446339,
+ "loss": 1.2669,
+ "theoretical_loss": 3.550414092086303,
+ "tokens_seen": 1471938560
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044771113340020066,
+ "loss": 1.241,
+ "theoretical_loss": 3.550400216495107,
+ "tokens_seen": 1472004096
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004476950852557673,
+ "loss": 1.318,
+ "theoretical_loss": 3.550386341694626,
+ "tokens_seen": 1472069632
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044767903711133406,
+ "loss": 1.1884,
+ "theoretical_loss": 3.55037246768478,
+ "tokens_seen": 1472135168
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004476629889669007,
+ "loss": 1.3407,
+ "theoretical_loss": 3.5503585944654885,
+ "tokens_seen": 1472200704
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044764694082246746,
+ "loss": 1.4194,
+ "theoretical_loss": 3.550344722036671,
+ "tokens_seen": 1472266240
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004476308926780341,
+ "loss": 1.2354,
+ "theoretical_loss": 3.5503308503982476,
+ "tokens_seen": 1472331776
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044761484453360086,
+ "loss": 1.3369,
+ "theoretical_loss": 3.5503169795501384,
+ "tokens_seen": 1472397312
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004475987963891675,
+ "loss": 1.3405,
+ "theoretical_loss": 3.550303109492263,
+ "tokens_seen": 1472462848
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044758274824473426,
+ "loss": 1.3397,
+ "theoretical_loss": 3.55028924022454,
+ "tokens_seen": 1472528384
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004475667001003009,
+ "loss": 1.2805,
+ "theoretical_loss": 3.550275371746891,
+ "tokens_seen": 1472593920
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044755065195586767,
+ "loss": 1.2799,
+ "theoretical_loss": 3.5502615040592356,
+ "tokens_seen": 1472659456
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004475346038114343,
+ "loss": 1.3203,
+ "theoretical_loss": 3.5502476371614926,
+ "tokens_seen": 1472724992
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044751855566700107,
+ "loss": 1.2079,
+ "theoretical_loss": 3.5502337710535827,
+ "tokens_seen": 1472790528
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004475025075225677,
+ "loss": 1.3942,
+ "theoretical_loss": 3.5502199057354256,
+ "tokens_seen": 1472856064
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044748645937813447,
+ "loss": 1.3379,
+ "theoretical_loss": 3.550206041206941,
+ "tokens_seen": 1472921600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004474704112337011,
+ "loss": 1.2404,
+ "theoretical_loss": 3.5501921774680483,
+ "tokens_seen": 1472987136
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044745436308926787,
+ "loss": 1.2891,
+ "theoretical_loss": 3.5501783145186687,
+ "tokens_seen": 1473052672
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004474383149448345,
+ "loss": 1.2657,
+ "theoretical_loss": 3.5501644523587212,
+ "tokens_seen": 1473118208
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044742226680040127,
+ "loss": 1.3169,
+ "theoretical_loss": 3.5501505909881264,
+ "tokens_seen": 1473183744
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004474062186559679,
+ "loss": 1.2235,
+ "theoretical_loss": 3.5501367304068028,
+ "tokens_seen": 1473249280
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044739017051153467,
+ "loss": 1.2876,
+ "theoretical_loss": 3.550122870614672,
+ "tokens_seen": 1473314816
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004473741223671013,
+ "loss": 1.2713,
+ "theoretical_loss": 3.5501090116116534,
+ "tokens_seen": 1473380352
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000447358074222668,
+ "loss": 1.2368,
+ "theoretical_loss": 3.550095153397667,
+ "tokens_seen": 1473445888
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004473420260782347,
+ "loss": 1.2598,
+ "theoretical_loss": 3.550081295972632,
+ "tokens_seen": 1473511424
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004473259779338014,
+ "loss": 1.2551,
+ "theoretical_loss": 3.550067439336469,
+ "tokens_seen": 1473576960
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004473099297893681,
+ "loss": 1.3723,
+ "theoretical_loss": 3.550053583489098,
+ "tokens_seen": 1473642496
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004472938816449348,
+ "loss": 1.2455,
+ "theoretical_loss": 3.5500397284304395,
+ "tokens_seen": 1473708032
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004472778335005015,
+ "loss": 1.3906,
+ "theoretical_loss": 3.550025874160413,
+ "tokens_seen": 1473773568
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004472617853560682,
+ "loss": 1.3666,
+ "theoretical_loss": 3.550012020678939,
+ "tokens_seen": 1473839104
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004472457372116349,
+ "loss": 1.3299,
+ "theoretical_loss": 3.549998167985937,
+ "tokens_seen": 1473904640
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004472296890672016,
+ "loss": 1.3547,
+ "theoretical_loss": 3.5499843160813276,
+ "tokens_seen": 1473970176
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044721364092276827,
+ "loss": 1.2667,
+ "theoretical_loss": 3.54997046496503,
+ "tokens_seen": 1474035712
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000447197592778335,
+ "loss": 1.2703,
+ "theoretical_loss": 3.549956614636965,
+ "tokens_seen": 1474101248
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044718154463390167,
+ "loss": 1.2809,
+ "theoretical_loss": 3.549942765097053,
+ "tokens_seen": 1474166784
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004471654964894684,
+ "loss": 1.314,
+ "theoretical_loss": 3.5499289163452126,
+ "tokens_seen": 1474232320
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044714944834503507,
+ "loss": 1.3864,
+ "theoretical_loss": 3.549915068381366,
+ "tokens_seen": 1474297856
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044713340020060183,
+ "loss": 1.2958,
+ "theoretical_loss": 3.549901221205432,
+ "tokens_seen": 1474363392
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004471173520561685,
+ "loss": 1.2209,
+ "theoretical_loss": 3.549887374817331,
+ "tokens_seen": 1474428928
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044710130391173523,
+ "loss": 1.1318,
+ "theoretical_loss": 3.5498735292169834,
+ "tokens_seen": 1474494464
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 494397,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3106197118759155,
+ "objective/train/theoretical_loss": 3.5498596844043093,
+ "objective/train/tokens_used": 1495020000,
+ "theoretical_loss": 3.5498596844043093,
+ "tokens_seen": 1474560000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004470852557673019,
+ "loss": 1.1911,
+ "theoretical_loss": 3.5498596844043093,
+ "tokens_seen": 1474560000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044706920762286863,
+ "loss": 1.2254,
+ "theoretical_loss": 3.549845840379229,
+ "tokens_seen": 1474625536
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004470531594784353,
+ "loss": 1.3583,
+ "theoretical_loss": 3.549831997141662,
+ "tokens_seen": 1474691072
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044703711133400203,
+ "loss": 1.267,
+ "theoretical_loss": 3.549818154691529,
+ "tokens_seen": 1474756608
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004470210631895687,
+ "loss": 1.2699,
+ "theoretical_loss": 3.5498043130287504,
+ "tokens_seen": 1474822144
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044700501504513543,
+ "loss": 1.2423,
+ "theoretical_loss": 3.549790472153246,
+ "tokens_seen": 1474887680
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004469889669007021,
+ "loss": 1.3835,
+ "theoretical_loss": 3.549776632064937,
+ "tokens_seen": 1474953216
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044697291875626883,
+ "loss": 1.3148,
+ "theoretical_loss": 3.5497627927637425,
+ "tokens_seen": 1475018752
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004469568706118355,
+ "loss": 1.3392,
+ "theoretical_loss": 3.5497489542495835,
+ "tokens_seen": 1475084288
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044694082246740224,
+ "loss": 1.3905,
+ "theoretical_loss": 3.5497351165223794,
+ "tokens_seen": 1475149824
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004469247743229689,
+ "loss": 1.2646,
+ "theoretical_loss": 3.5497212795820516,
+ "tokens_seen": 1475215360
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044690872617853564,
+ "loss": 1.3821,
+ "theoretical_loss": 3.54970744342852,
+ "tokens_seen": 1475280896
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004468926780341023,
+ "loss": 1.2743,
+ "theoretical_loss": 3.5496936080617045,
+ "tokens_seen": 1475346432
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044687662988966904,
+ "loss": 1.2287,
+ "theoretical_loss": 3.5496797734815257,
+ "tokens_seen": 1475411968
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004468605817452357,
+ "loss": 1.2634,
+ "theoretical_loss": 3.5496659396879044,
+ "tokens_seen": 1475477504
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044684453360080244,
+ "loss": 1.3642,
+ "theoretical_loss": 3.5496521066807603,
+ "tokens_seen": 1475543040
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004468284854563692,
+ "loss": 1.3364,
+ "theoretical_loss": 3.5496382744600137,
+ "tokens_seen": 1475608576
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044681243731193584,
+ "loss": 1.2063,
+ "theoretical_loss": 3.549624443025585,
+ "tokens_seen": 1475674112
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004467963891675026,
+ "loss": 1.2851,
+ "theoretical_loss": 3.5496106123773954,
+ "tokens_seen": 1475739648
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044678034102306924,
+ "loss": 1.2888,
+ "theoretical_loss": 3.5495967825153647,
+ "tokens_seen": 1475805184
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000446764292878636,
+ "loss": 1.3448,
+ "theoretical_loss": 3.5495829534394128,
+ "tokens_seen": 1475870720
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044674824473420264,
+ "loss": 1.3652,
+ "theoretical_loss": 3.5495691251494614,
+ "tokens_seen": 1475936256
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044673219658976934,
+ "loss": 1.2462,
+ "theoretical_loss": 3.54955529764543,
+ "tokens_seen": 1476001792
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044671614844533604,
+ "loss": 1.4395,
+ "theoretical_loss": 3.549541470927239,
+ "tokens_seen": 1476067328
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044670010030090274,
+ "loss": 1.3211,
+ "theoretical_loss": 3.549527644994809,
+ "tokens_seen": 1476132864
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044668405215646945,
+ "loss": 1.2914,
+ "theoretical_loss": 3.5495138198480607,
+ "tokens_seen": 1476198400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044666800401203615,
+ "loss": 1.2559,
+ "theoretical_loss": 3.549499995486914,
+ "tokens_seen": 1476263936
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044665195586760285,
+ "loss": 1.2727,
+ "theoretical_loss": 3.5494861719112905,
+ "tokens_seen": 1476329472
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044663590772316955,
+ "loss": 1.3265,
+ "theoretical_loss": 3.549472349121109,
+ "tokens_seen": 1476395008
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044661985957873625,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5494585271162915,
+ "tokens_seen": 1476460544
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044660381143430295,
+ "loss": 1.2819,
+ "theoretical_loss": 3.549444705896758,
+ "tokens_seen": 1476526080
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004465877632898696,
+ "loss": 1.3293,
+ "theoretical_loss": 3.5494308854624292,
+ "tokens_seen": 1476591616
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044657171514543635,
+ "loss": 1.3372,
+ "theoretical_loss": 3.5494170658132256,
+ "tokens_seen": 1476657152
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000446555667001003,
+ "loss": 1.295,
+ "theoretical_loss": 3.5494032469490673,
+ "tokens_seen": 1476722688
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044653961885656975,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5493894288698753,
+ "tokens_seen": 1476788224
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004465235707121364,
+ "loss": 1.265,
+ "theoretical_loss": 3.54937561157557,
+ "tokens_seen": 1476853760
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044650752256770315,
+ "loss": 1.3506,
+ "theoretical_loss": 3.5493617950660727,
+ "tokens_seen": 1476919296
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004464914744232698,
+ "loss": 1.2906,
+ "theoretical_loss": 3.549347979341303,
+ "tokens_seen": 1476984832
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044647542627883655,
+ "loss": 1.3151,
+ "theoretical_loss": 3.5493341644011815,
+ "tokens_seen": 1477050368
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004464593781344032,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5493203502456296,
+ "tokens_seen": 1477115904
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044644332998996995,
+ "loss": 1.3257,
+ "theoretical_loss": 3.5493065368745675,
+ "tokens_seen": 1477181440
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004464272818455366,
+ "loss": 1.2564,
+ "theoretical_loss": 3.549292724287916,
+ "tokens_seen": 1477246976
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044641123370110336,
+ "loss": 1.199,
+ "theoretical_loss": 3.5492789124855957,
+ "tokens_seen": 1477312512
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044639518555667,
+ "loss": 1.3125,
+ "theoretical_loss": 3.5492651014675274,
+ "tokens_seen": 1477378048
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044637913741223676,
+ "loss": 1.4284,
+ "theoretical_loss": 3.5492512912336314,
+ "tokens_seen": 1477443584
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004463630892678034,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5492374817838286,
+ "tokens_seen": 1477509120
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044634704112337016,
+ "loss": 1.2588,
+ "theoretical_loss": 3.54922367311804,
+ "tokens_seen": 1477574656
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004463309929789368,
+ "loss": 1.2301,
+ "theoretical_loss": 3.549209865236185,
+ "tokens_seen": 1477640192
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044631494483450356,
+ "loss": 1.2794,
+ "theoretical_loss": 3.549196058138187,
+ "tokens_seen": 1477705728
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004462988966900702,
+ "loss": 1.279,
+ "theoretical_loss": 3.549182251823964,
+ "tokens_seen": 1477771264
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 495688,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2247810363769531,
+ "objective/train/theoretical_loss": 3.549168446293438,
+ "objective/train/tokens_used": 1498296800,
+ "theoretical_loss": 3.549168446293438,
+ "tokens_seen": 1477836800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044628284854563696,
+ "loss": 1.2837,
+ "theoretical_loss": 3.549168446293438,
+ "tokens_seen": 1477836800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004462668004012036,
+ "loss": 1.2858,
+ "theoretical_loss": 3.54915464154653,
+ "tokens_seen": 1477902336
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044625075225677036,
+ "loss": 1.3078,
+ "theoretical_loss": 3.54914083758316,
+ "tokens_seen": 1477967872
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000446234704112337,
+ "loss": 1.229,
+ "theoretical_loss": 3.5491270344032495,
+ "tokens_seen": 1478033408
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044621865596790376,
+ "loss": 1.3989,
+ "theoretical_loss": 3.549113232006719,
+ "tokens_seen": 1478098944
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004462026078234704,
+ "loss": 1.3031,
+ "theoretical_loss": 3.549099430393489,
+ "tokens_seen": 1478164480
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044618655967903716,
+ "loss": 1.2627,
+ "theoretical_loss": 3.5490856295634807,
+ "tokens_seen": 1478230016
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004461705115346038,
+ "loss": 1.3746,
+ "theoretical_loss": 3.549071829516615,
+ "tokens_seen": 1478295552
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044615446339017057,
+ "loss": 1.305,
+ "theoretical_loss": 3.5490580302528123,
+ "tokens_seen": 1478361088
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004461384152457372,
+ "loss": 1.2692,
+ "theoretical_loss": 3.549044231771994,
+ "tokens_seen": 1478426624
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044612236710130397,
+ "loss": 1.2892,
+ "theoretical_loss": 3.549030434074081,
+ "tokens_seen": 1478492160
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004461063189568706,
+ "loss": 1.2861,
+ "theoretical_loss": 3.5490166371589935,
+ "tokens_seen": 1478557696
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044609027081243737,
+ "loss": 1.3145,
+ "theoretical_loss": 3.549002841026653,
+ "tokens_seen": 1478623232
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000446074222668004,
+ "loss": 1.2981,
+ "theoretical_loss": 3.54898904567698,
+ "tokens_seen": 1478688768
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044605817452357077,
+ "loss": 1.3612,
+ "theoretical_loss": 3.548975251109896,
+ "tokens_seen": 1478754304
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004460421263791374,
+ "loss": 1.3104,
+ "theoretical_loss": 3.548961457325321,
+ "tokens_seen": 1478819840
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044602607823470417,
+ "loss": 1.2714,
+ "theoretical_loss": 3.548947664323177,
+ "tokens_seen": 1478885376
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004460100300902708,
+ "loss": 1.2712,
+ "theoretical_loss": 3.548933872103384,
+ "tokens_seen": 1478950912
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044599398194583757,
+ "loss": 1.3135,
+ "theoretical_loss": 3.548920080665863,
+ "tokens_seen": 1479016448
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004459779338014042,
+ "loss": 1.318,
+ "theoretical_loss": 3.548906290010536,
+ "tokens_seen": 1479081984
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004459618856569709,
+ "loss": 1.3726,
+ "theoretical_loss": 3.548892500137323,
+ "tokens_seen": 1479147520
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004459458375125376,
+ "loss": 1.3166,
+ "theoretical_loss": 3.5488787110461457,
+ "tokens_seen": 1479213056
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004459297893681043,
+ "loss": 1.3793,
+ "theoretical_loss": 3.5488649227369247,
+ "tokens_seen": 1479278592
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000445913741223671,
+ "loss": 1.4259,
+ "theoretical_loss": 3.548851135209581,
+ "tokens_seen": 1479344128
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458976930792377,
+ "loss": 1.3343,
+ "theoretical_loss": 3.548837348464035,
+ "tokens_seen": 1479409664
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458816449348044,
+ "loss": 1.2835,
+ "theoretical_loss": 3.548823562500209,
+ "tokens_seen": 1479475200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458655967903711,
+ "loss": 1.3106,
+ "theoretical_loss": 3.5488097773180236,
+ "tokens_seen": 1479540736
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458495486459378,
+ "loss": 1.5154,
+ "theoretical_loss": 3.5487959929173996,
+ "tokens_seen": 1479606272
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458335005015045,
+ "loss": 1.3065,
+ "theoretical_loss": 3.548782209298258,
+ "tokens_seen": 1479671808
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044581745235707117,
+ "loss": 1.3041,
+ "theoretical_loss": 3.54876842646052,
+ "tokens_seen": 1479737344
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004458014042126379,
+ "loss": 1.2106,
+ "theoretical_loss": 3.5487546444041076,
+ "tokens_seen": 1479802880
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044578535606820457,
+ "loss": 1.3655,
+ "theoretical_loss": 3.5487408631289403,
+ "tokens_seen": 1479868416
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044576930792377133,
+ "loss": 1.3113,
+ "theoretical_loss": 3.54872708263494,
+ "tokens_seen": 1479933952
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000445753259779338,
+ "loss": 1.3416,
+ "theoretical_loss": 3.548713302922028,
+ "tokens_seen": 1479999488
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044573721163490473,
+ "loss": 1.3106,
+ "theoretical_loss": 3.5486995239901256,
+ "tokens_seen": 1480065024
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004457211634904714,
+ "loss": 1.3696,
+ "theoretical_loss": 3.5486857458391534,
+ "tokens_seen": 1480130560
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044570511534603813,
+ "loss": 1.3216,
+ "theoretical_loss": 3.5486719684690327,
+ "tokens_seen": 1480196096
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004456890672016048,
+ "loss": 1.2756,
+ "theoretical_loss": 3.548658191879685,
+ "tokens_seen": 1480261632
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044567301905717153,
+ "loss": 1.3497,
+ "theoretical_loss": 3.5486444160710313,
+ "tokens_seen": 1480327168
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004456569709127382,
+ "loss": 1.325,
+ "theoretical_loss": 3.548630641042992,
+ "tokens_seen": 1480392704
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044564092276830493,
+ "loss": 1.2649,
+ "theoretical_loss": 3.54861686679549,
+ "tokens_seen": 1480458240
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004456248746238717,
+ "loss": 1.333,
+ "theoretical_loss": 3.5486030933284454,
+ "tokens_seen": 1480523776
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044560882647943833,
+ "loss": 1.3912,
+ "theoretical_loss": 3.5485893206417796,
+ "tokens_seen": 1480589312
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004455927783350051,
+ "loss": 1.3423,
+ "theoretical_loss": 3.5485755487354136,
+ "tokens_seen": 1480654848
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044557673019057173,
+ "loss": 1.3831,
+ "theoretical_loss": 3.5485617776092693,
+ "tokens_seen": 1480720384
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004455606820461385,
+ "loss": 1.2653,
+ "theoretical_loss": 3.5485480072632676,
+ "tokens_seen": 1480785920
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044554463390170514,
+ "loss": 1.2235,
+ "theoretical_loss": 3.5485342376973295,
+ "tokens_seen": 1480851456
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004455285857572719,
+ "loss": 1.3191,
+ "theoretical_loss": 3.5485204689113767,
+ "tokens_seen": 1480916992
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044551253761283854,
+ "loss": 1.3674,
+ "theoretical_loss": 3.548506700905331,
+ "tokens_seen": 1480982528
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004454964894684053,
+ "loss": 1.4616,
+ "theoretical_loss": 3.5484929336791122,
+ "tokens_seen": 1481048064
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 496572,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3179576396942139,
+ "objective/train/theoretical_loss": 3.5484791672326423,
+ "objective/train/tokens_used": 1501573600,
+ "theoretical_loss": 3.5484791672326423,
+ "tokens_seen": 1481113600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044548044132397194,
+ "loss": 1.2927,
+ "theoretical_loss": 3.5484791672326423,
+ "tokens_seen": 1481113600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004454643931795387,
+ "loss": 1.3033,
+ "theoretical_loss": 3.548465401565844,
+ "tokens_seen": 1481179136
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044544834503510534,
+ "loss": 1.3021,
+ "theoretical_loss": 3.548451636678637,
+ "tokens_seen": 1481244672
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004454322968906721,
+ "loss": 1.2335,
+ "theoretical_loss": 3.5484378725709425,
+ "tokens_seen": 1481310208
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044541624874623874,
+ "loss": 1.3487,
+ "theoretical_loss": 3.548424109242683,
+ "tokens_seen": 1481375744
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004454002006018055,
+ "loss": 1.2137,
+ "theoretical_loss": 3.54841034669378,
+ "tokens_seen": 1481441280
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044538415245737214,
+ "loss": 1.4135,
+ "theoretical_loss": 3.548396584924154,
+ "tokens_seen": 1481506816
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004453681043129389,
+ "loss": 1.2251,
+ "theoretical_loss": 3.5483828239337267,
+ "tokens_seen": 1481572352
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044535205616850554,
+ "loss": 1.4295,
+ "theoretical_loss": 3.5483690637224194,
+ "tokens_seen": 1481637888
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044533600802407224,
+ "loss": 1.3799,
+ "theoretical_loss": 3.548355304290154,
+ "tokens_seen": 1481703424
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044531995987963895,
+ "loss": 1.3247,
+ "theoretical_loss": 3.548341545636851,
+ "tokens_seen": 1481768960
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044530391173520565,
+ "loss": 1.2748,
+ "theoretical_loss": 3.5483277877624326,
+ "tokens_seen": 1481834496
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044528786359077235,
+ "loss": 1.2615,
+ "theoretical_loss": 3.5483140306668206,
+ "tokens_seen": 1481900032
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044527181544633905,
+ "loss": 1.3339,
+ "theoretical_loss": 3.548300274349936,
+ "tokens_seen": 1481965568
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044525576730190575,
+ "loss": 1.2702,
+ "theoretical_loss": 3.5482865188117,
+ "tokens_seen": 1482031104
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044523971915747245,
+ "loss": 1.4656,
+ "theoretical_loss": 3.5482727640520344,
+ "tokens_seen": 1482096640
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044522367101303915,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5482590100708604,
+ "tokens_seen": 1482162176
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044520762286860585,
+ "loss": 1.2492,
+ "theoretical_loss": 3.5482452568681007,
+ "tokens_seen": 1482227712
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004451915747241725,
+ "loss": 1.236,
+ "theoretical_loss": 3.5482315044436747,
+ "tokens_seen": 1482293248
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044517552657973925,
+ "loss": 1.2805,
+ "theoretical_loss": 3.5482177527975063,
+ "tokens_seen": 1482358784
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004451594784353059,
+ "loss": 1.3446,
+ "theoretical_loss": 3.5482040019295153,
+ "tokens_seen": 1482424320
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044514343029087265,
+ "loss": 1.3252,
+ "theoretical_loss": 3.548190251839624,
+ "tokens_seen": 1482489856
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004451273821464393,
+ "loss": 1.3548,
+ "theoretical_loss": 3.5481765025277543,
+ "tokens_seen": 1482555392
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044511133400200605,
+ "loss": 1.397,
+ "theoretical_loss": 3.548162753993827,
+ "tokens_seen": 1482620928
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004450952858575727,
+ "loss": 1.2837,
+ "theoretical_loss": 3.548149006237764,
+ "tokens_seen": 1482686464
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044507923771313945,
+ "loss": 1.3748,
+ "theoretical_loss": 3.548135259259487,
+ "tokens_seen": 1482752000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004450631895687061,
+ "loss": 1.2917,
+ "theoretical_loss": 3.548121513058918,
+ "tokens_seen": 1482817536
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044504714142427286,
+ "loss": 1.3039,
+ "theoretical_loss": 3.548107767635978,
+ "tokens_seen": 1482883072
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004450310932798395,
+ "loss": 1.2809,
+ "theoretical_loss": 3.548094022990589,
+ "tokens_seen": 1482948608
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044501504513540626,
+ "loss": 1.3502,
+ "theoretical_loss": 3.548080279122672,
+ "tokens_seen": 1483014144
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004449989969909729,
+ "loss": 1.1451,
+ "theoretical_loss": 3.5480665360321497,
+ "tokens_seen": 1483079680
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044498294884653966,
+ "loss": 1.2604,
+ "theoretical_loss": 3.548052793718943,
+ "tokens_seen": 1483145216
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004449669007021063,
+ "loss": 1.2212,
+ "theoretical_loss": 3.548039052182974,
+ "tokens_seen": 1483210752
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044495085255767306,
+ "loss": 1.3196,
+ "theoretical_loss": 3.5480253114241647,
+ "tokens_seen": 1483276288
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004449348044132397,
+ "loss": 1.2527,
+ "theoretical_loss": 3.5480115714424354,
+ "tokens_seen": 1483341824
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044491875626880646,
+ "loss": 1.1341,
+ "theoretical_loss": 3.5479978322377095,
+ "tokens_seen": 1483407360
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004449027081243731,
+ "loss": 1.3636,
+ "theoretical_loss": 3.547984093809908,
+ "tokens_seen": 1483472896
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044488665997993986,
+ "loss": 1.2996,
+ "theoretical_loss": 3.5479703561589524,
+ "tokens_seen": 1483538432
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004448706118355065,
+ "loss": 1.3028,
+ "theoretical_loss": 3.547956619284765,
+ "tokens_seen": 1483603968
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044485456369107326,
+ "loss": 1.3174,
+ "theoretical_loss": 3.5479428831872672,
+ "tokens_seen": 1483669504
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004448385155466399,
+ "loss": 1.4079,
+ "theoretical_loss": 3.5479291478663812,
+ "tokens_seen": 1483735040
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044482246740220666,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5479154133220283,
+ "tokens_seen": 1483800576
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004448064192577733,
+ "loss": 1.2796,
+ "theoretical_loss": 3.5479016795541307,
+ "tokens_seen": 1483866112
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044479037111334007,
+ "loss": 1.3138,
+ "theoretical_loss": 3.5478879465626094,
+ "tokens_seen": 1483931648
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004447743229689067,
+ "loss": 1.3321,
+ "theoretical_loss": 3.547874214347387,
+ "tokens_seen": 1483997184
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044475827482447347,
+ "loss": 1.3997,
+ "theoretical_loss": 3.547860482908386,
+ "tokens_seen": 1484062720
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004447422266800401,
+ "loss": 1.3465,
+ "theoretical_loss": 3.5478467522455266,
+ "tokens_seen": 1484128256
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044472617853560687,
+ "loss": 1.2211,
+ "theoretical_loss": 3.5478330223587315,
+ "tokens_seen": 1484193792
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004447101303911735,
+ "loss": 1.2692,
+ "theoretical_loss": 3.5478192932479233,
+ "tokens_seen": 1484259328
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044469408224674027,
+ "loss": 1.3578,
+ "theoretical_loss": 3.5478055649130225,
+ "tokens_seen": 1484324864
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 497900,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2213529348373413,
+ "objective/train/theoretical_loss": 3.5477918373539516,
+ "objective/train/tokens_used": 1504850400,
+ "theoretical_loss": 3.5477918373539516,
+ "tokens_seen": 1484390400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004446780341023069,
+ "loss": 1.3075,
+ "theoretical_loss": 3.5477918373539516,
+ "tokens_seen": 1484390400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044466198595787367,
+ "loss": 1.3878,
+ "theoretical_loss": 3.5477781105706327,
+ "tokens_seen": 1484455936
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004446459378134403,
+ "loss": 1.3598,
+ "theoretical_loss": 3.5477643845629876,
+ "tokens_seen": 1484521472
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044462988966900707,
+ "loss": 1.3405,
+ "theoretical_loss": 3.547750659330938,
+ "tokens_seen": 1484587008
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004446138415245737,
+ "loss": 1.3402,
+ "theoretical_loss": 3.547736934874406,
+ "tokens_seen": 1484652544
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445977933801405,
+ "loss": 1.1972,
+ "theoretical_loss": 3.547723211193314,
+ "tokens_seen": 1484718080
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445817452357071,
+ "loss": 1.2758,
+ "theoretical_loss": 3.547709488287583,
+ "tokens_seen": 1484783616
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445656970912738,
+ "loss": 1.2796,
+ "theoretical_loss": 3.547695766157136,
+ "tokens_seen": 1484849152
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445496489468405,
+ "loss": 1.2446,
+ "theoretical_loss": 3.547682044801894,
+ "tokens_seen": 1484914688
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445336008024072,
+ "loss": 1.3446,
+ "theoretical_loss": 3.54766832422178,
+ "tokens_seen": 1484980224
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445175526579739,
+ "loss": 1.299,
+ "theoretical_loss": 3.547654604416715,
+ "tokens_seen": 1485045760
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004445015045135406,
+ "loss": 1.1966,
+ "theoretical_loss": 3.547640885386622,
+ "tokens_seen": 1485111296
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004444854563691073,
+ "loss": 1.2917,
+ "theoretical_loss": 3.547627167131422,
+ "tokens_seen": 1485176832
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000444469408224674,
+ "loss": 1.23,
+ "theoretical_loss": 3.5476134496510383,
+ "tokens_seen": 1485242368
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004444533600802407,
+ "loss": 1.2974,
+ "theoretical_loss": 3.5475997329453914,
+ "tokens_seen": 1485307904
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004444373119358074,
+ "loss": 1.3243,
+ "theoretical_loss": 3.547586017014405,
+ "tokens_seen": 1485373440
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004444212637913742,
+ "loss": 1.2422,
+ "theoretical_loss": 3.5475723018579997,
+ "tokens_seen": 1485438976
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004444052156469408,
+ "loss": 1.1543,
+ "theoretical_loss": 3.5475585874760984,
+ "tokens_seen": 1485504512
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004443891675025076,
+ "loss": 1.3314,
+ "theoretical_loss": 3.547544873868623,
+ "tokens_seen": 1485570048
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044437311935807423,
+ "loss": 1.2806,
+ "theoretical_loss": 3.5475311610354963,
+ "tokens_seen": 1485635584
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000444357071213641,
+ "loss": 1.3674,
+ "theoretical_loss": 3.547517448976639,
+ "tokens_seen": 1485701120
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044434102306920763,
+ "loss": 1.3682,
+ "theoretical_loss": 3.547503737691975,
+ "tokens_seen": 1485766656
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004443249749247744,
+ "loss": 1.342,
+ "theoretical_loss": 3.547490027181424,
+ "tokens_seen": 1485832192
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044430892678034103,
+ "loss": 1.2189,
+ "theoretical_loss": 3.547476317444911,
+ "tokens_seen": 1485897728
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004442928786359078,
+ "loss": 1.279,
+ "theoretical_loss": 3.547462608482356,
+ "tokens_seen": 1485963264
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044427683049147443,
+ "loss": 1.3384,
+ "theoretical_loss": 3.5474489002936824,
+ "tokens_seen": 1486028800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004442607823470412,
+ "loss": 1.411,
+ "theoretical_loss": 3.5474351928788117,
+ "tokens_seen": 1486094336
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044424473420260783,
+ "loss": 1.4026,
+ "theoretical_loss": 3.547421486237666,
+ "tokens_seen": 1486159872
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004442286860581746,
+ "loss": 1.3369,
+ "theoretical_loss": 3.5474077803701682,
+ "tokens_seen": 1486225408
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044421263791374123,
+ "loss": 1.3357,
+ "theoretical_loss": 3.5473940752762405,
+ "tokens_seen": 1486290944
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000444196589769308,
+ "loss": 1.336,
+ "theoretical_loss": 3.5473803709558043,
+ "tokens_seen": 1486356480
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044418054162487464,
+ "loss": 1.2241,
+ "theoretical_loss": 3.5473666674087827,
+ "tokens_seen": 1486422016
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004441644934804414,
+ "loss": 1.276,
+ "theoretical_loss": 3.547352964635097,
+ "tokens_seen": 1486487552
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044414844533600804,
+ "loss": 1.3731,
+ "theoretical_loss": 3.547339262634671,
+ "tokens_seen": 1486553088
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004441323971915748,
+ "loss": 1.3268,
+ "theoretical_loss": 3.5473255614074257,
+ "tokens_seen": 1486618624
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044411634904714144,
+ "loss": 1.3405,
+ "theoretical_loss": 3.5473118609532834,
+ "tokens_seen": 1486684160
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004441003009027082,
+ "loss": 1.3448,
+ "theoretical_loss": 3.547298161272167,
+ "tokens_seen": 1486749696
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044408425275827484,
+ "loss": 1.3078,
+ "theoretical_loss": 3.5472844623639985,
+ "tokens_seen": 1486815232
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004440682046138416,
+ "loss": 1.3369,
+ "theoretical_loss": 3.5472707642287,
+ "tokens_seen": 1486880768
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044405215646940824,
+ "loss": 1.1991,
+ "theoretical_loss": 3.5472570668661945,
+ "tokens_seen": 1486946304
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000444036108324975,
+ "loss": 1.2801,
+ "theoretical_loss": 3.547243370276404,
+ "tokens_seen": 1487011840
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044402006018054164,
+ "loss": 1.4029,
+ "theoretical_loss": 3.547229674459251,
+ "tokens_seen": 1487077376
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004440040120361084,
+ "loss": 1.3469,
+ "theoretical_loss": 3.547215979414657,
+ "tokens_seen": 1487142912
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044398796389167504,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5472022851425455,
+ "tokens_seen": 1487208448
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004439719157472418,
+ "loss": 1.2682,
+ "theoretical_loss": 3.547188591642838,
+ "tokens_seen": 1487273984
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044395586760280844,
+ "loss": 1.3008,
+ "theoretical_loss": 3.5471748989154577,
+ "tokens_seen": 1487339520
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044393981945837515,
+ "loss": 1.3619,
+ "theoretical_loss": 3.5471612069603267,
+ "tokens_seen": 1487405056
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044392377131394185,
+ "loss": 1.2172,
+ "theoretical_loss": 3.5471475157773673,
+ "tokens_seen": 1487470592
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044390772316950855,
+ "loss": 1.3858,
+ "theoretical_loss": 3.547133825366502,
+ "tokens_seen": 1487536128
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044389167502507525,
+ "loss": 1.321,
+ "theoretical_loss": 3.547120135727653,
+ "tokens_seen": 1487601664
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 498821,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4604065418243408,
+ "objective/train/theoretical_loss": 3.5471064468607434,
+ "objective/train/tokens_used": 1508127200,
+ "theoretical_loss": 3.5471064468607434,
+ "tokens_seen": 1487667200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044387562688064195,
+ "loss": 1.4423,
+ "theoretical_loss": 3.5471064468607434,
+ "tokens_seen": 1487667200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044385957873620865,
+ "loss": 1.1887,
+ "theoretical_loss": 3.547092758765695,
+ "tokens_seen": 1487732736
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044384353059177535,
+ "loss": 1.1948,
+ "theoretical_loss": 3.5470790714424307,
+ "tokens_seen": 1487798272
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044382748244734205,
+ "loss": 1.2935,
+ "theoretical_loss": 3.547065384890873,
+ "tokens_seen": 1487863808
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044381143430290875,
+ "loss": 1.2731,
+ "theoretical_loss": 3.547051699110944,
+ "tokens_seen": 1487929344
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004437953861584754,
+ "loss": 1.2875,
+ "theoretical_loss": 3.547038014102567,
+ "tokens_seen": 1487994880
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044377933801404215,
+ "loss": 1.2711,
+ "theoretical_loss": 3.547024329865663,
+ "tokens_seen": 1488060416
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004437632898696088,
+ "loss": 1.2718,
+ "theoretical_loss": 3.547010646400156,
+ "tokens_seen": 1488125952
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044374724172517555,
+ "loss": 1.4034,
+ "theoretical_loss": 3.5469969637059684,
+ "tokens_seen": 1488191488
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004437311935807422,
+ "loss": 1.341,
+ "theoretical_loss": 3.546983281783022,
+ "tokens_seen": 1488257024
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044371514543630895,
+ "loss": 1.2758,
+ "theoretical_loss": 3.54696960063124,
+ "tokens_seen": 1488322560
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004436990972918756,
+ "loss": 1.2209,
+ "theoretical_loss": 3.5469559202505447,
+ "tokens_seen": 1488388096
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044368304914744236,
+ "loss": 1.4072,
+ "theoretical_loss": 3.546942240640859,
+ "tokens_seen": 1488453632
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000443667001003009,
+ "loss": 1.3209,
+ "theoretical_loss": 3.546928561802105,
+ "tokens_seen": 1488519168
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044365095285857576,
+ "loss": 1.2755,
+ "theoretical_loss": 3.546914883734206,
+ "tokens_seen": 1488584704
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004436349047141424,
+ "loss": 1.1996,
+ "theoretical_loss": 3.5469012064370835,
+ "tokens_seen": 1488650240
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044361885656970916,
+ "loss": 1.3091,
+ "theoretical_loss": 3.5468875299106615,
+ "tokens_seen": 1488715776
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004436028084252758,
+ "loss": 1.3795,
+ "theoretical_loss": 3.5468738541548617,
+ "tokens_seen": 1488781312
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044358676028084256,
+ "loss": 1.2914,
+ "theoretical_loss": 3.546860179169607,
+ "tokens_seen": 1488846848
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004435707121364092,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5468465049548206,
+ "tokens_seen": 1488912384
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044355466399197596,
+ "loss": 1.3453,
+ "theoretical_loss": 3.5468328315104243,
+ "tokens_seen": 1488977920
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004435386158475426,
+ "loss": 1.3484,
+ "theoretical_loss": 3.546819158836341,
+ "tokens_seen": 1489043456
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044352256770310936,
+ "loss": 1.3253,
+ "theoretical_loss": 3.546805486932494,
+ "tokens_seen": 1489108992
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000443506519558676,
+ "loss": 1.2508,
+ "theoretical_loss": 3.546791815798805,
+ "tokens_seen": 1489174528
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044349047141424276,
+ "loss": 1.3773,
+ "theoretical_loss": 3.546778145435198,
+ "tokens_seen": 1489240064
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004434744232698094,
+ "loss": 1.2105,
+ "theoretical_loss": 3.546764475841595,
+ "tokens_seen": 1489305600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044345837512537616,
+ "loss": 1.3785,
+ "theoretical_loss": 3.5467508070179186,
+ "tokens_seen": 1489371136
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004434423269809428,
+ "loss": 1.3551,
+ "theoretical_loss": 3.546737138964092,
+ "tokens_seen": 1489436672
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044342627883650957,
+ "loss": 1.2329,
+ "theoretical_loss": 3.5467234716800373,
+ "tokens_seen": 1489502208
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004434102306920762,
+ "loss": 1.3633,
+ "theoretical_loss": 3.5467098051656785,
+ "tokens_seen": 1489567744
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044339418254764297,
+ "loss": 1.3787,
+ "theoretical_loss": 3.546696139420937,
+ "tokens_seen": 1489633280
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004433781344032096,
+ "loss": 1.2233,
+ "theoretical_loss": 3.5466824744457366,
+ "tokens_seen": 1489698816
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044336208625877637,
+ "loss": 1.2936,
+ "theoretical_loss": 3.5466688102399995,
+ "tokens_seen": 1489764352
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000443346038114343,
+ "loss": 1.3799,
+ "theoretical_loss": 3.5466551468036487,
+ "tokens_seen": 1489829888
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044332998996990977,
+ "loss": 1.3461,
+ "theoretical_loss": 3.5466414841366074,
+ "tokens_seen": 1489895424
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004433139418254764,
+ "loss": 1.3389,
+ "theoretical_loss": 3.546627822238798,
+ "tokens_seen": 1489960960
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044329789368104317,
+ "loss": 1.3126,
+ "theoretical_loss": 3.5466141611101434,
+ "tokens_seen": 1490026496
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004432818455366098,
+ "loss": 1.2598,
+ "theoretical_loss": 3.546600500750567,
+ "tokens_seen": 1490092032
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044326579739217657,
+ "loss": 1.3039,
+ "theoretical_loss": 3.546586841159991,
+ "tokens_seen": 1490157568
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004432497492477432,
+ "loss": 1.2855,
+ "theoretical_loss": 3.546573182338338,
+ "tokens_seen": 1490223104
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044323370110331,
+ "loss": 1.3369,
+ "theoretical_loss": 3.546559524285532,
+ "tokens_seen": 1490288640
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004432176529588767,
+ "loss": 1.305,
+ "theoretical_loss": 3.5465458670014955,
+ "tokens_seen": 1490354176
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004432016048144434,
+ "loss": 1.3263,
+ "theoretical_loss": 3.546532210486151,
+ "tokens_seen": 1490419712
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431855566700101,
+ "loss": 1.3127,
+ "theoretical_loss": 3.5465185547394222,
+ "tokens_seen": 1490485248
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431695085255767,
+ "loss": 1.3767,
+ "theoretical_loss": 3.546504899761231,
+ "tokens_seen": 1490550784
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431534603811435,
+ "loss": 1.3478,
+ "theoretical_loss": 3.5464912455515014,
+ "tokens_seen": 1490616320
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431374122367101,
+ "loss": 1.3366,
+ "theoretical_loss": 3.546477592110156,
+ "tokens_seen": 1490681856
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431213640922769,
+ "loss": 1.254,
+ "theoretical_loss": 3.5464639394371167,
+ "tokens_seen": 1490747392
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004431053159478435,
+ "loss": 1.2013,
+ "theoretical_loss": 3.5464502875323083,
+ "tokens_seen": 1490812928
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004430892678034103,
+ "loss": 1.3132,
+ "theoretical_loss": 3.5464366363956534,
+ "tokens_seen": 1490878464
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 499903,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3060457706451416,
+ "objective/train/theoretical_loss": 3.546422986027074,
+ "objective/train/tokens_used": 1511404000,
+ "theoretical_loss": 3.546422986027074,
+ "tokens_seen": 1490944000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004430732196589769,
+ "loss": 1.272,
+ "theoretical_loss": 3.546422986027074,
+ "tokens_seen": 1490944000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004430571715145437,
+ "loss": 1.3209,
+ "theoretical_loss": 3.5464093364264935,
+ "tokens_seen": 1491009536
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004430411233701103,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5463956875938356,
+ "tokens_seen": 1491075072
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004430250752256771,
+ "loss": 1.3027,
+ "theoretical_loss": 3.546382039529023,
+ "tokens_seen": 1491140608
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044300902708124373,
+ "loss": 1.2638,
+ "theoretical_loss": 3.5463683922319786,
+ "tokens_seen": 1491206144
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004429929789368105,
+ "loss": 1.2887,
+ "theoretical_loss": 3.546354745702625,
+ "tokens_seen": 1491271680
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044297693079237713,
+ "loss": 1.3575,
+ "theoretical_loss": 3.5463410999408866,
+ "tokens_seen": 1491337216
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004429608826479439,
+ "loss": 1.2515,
+ "theoretical_loss": 3.5463274549466854,
+ "tokens_seen": 1491402752
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044294483450351053,
+ "loss": 1.3469,
+ "theoretical_loss": 3.546313810719945,
+ "tokens_seen": 1491468288
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004429287863590773,
+ "loss": 1.3389,
+ "theoretical_loss": 3.546300167260588,
+ "tokens_seen": 1491533824
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044291273821464393,
+ "loss": 1.3137,
+ "theoretical_loss": 3.546286524568538,
+ "tokens_seen": 1491599360
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004428966900702107,
+ "loss": 1.2875,
+ "theoretical_loss": 3.546272882643718,
+ "tokens_seen": 1491664896
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044288064192577733,
+ "loss": 1.258,
+ "theoretical_loss": 3.546259241486051,
+ "tokens_seen": 1491730432
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004428645937813441,
+ "loss": 1.2512,
+ "theoretical_loss": 3.5462456010954604,
+ "tokens_seen": 1491795968
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044284854563691073,
+ "loss": 1.2033,
+ "theoretical_loss": 3.54623196147187,
+ "tokens_seen": 1491861504
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004428324974924775,
+ "loss": 1.2309,
+ "theoretical_loss": 3.5462183226152013,
+ "tokens_seen": 1491927040
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044281644934804414,
+ "loss": 1.2305,
+ "theoretical_loss": 3.546204684525379,
+ "tokens_seen": 1491992576
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004428004012036109,
+ "loss": 1.2936,
+ "theoretical_loss": 3.546191047202325,
+ "tokens_seen": 1492058112
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044278435305917754,
+ "loss": 1.2525,
+ "theoretical_loss": 3.546177410645964,
+ "tokens_seen": 1492123648
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004427683049147443,
+ "loss": 1.2254,
+ "theoretical_loss": 3.5461637748562183,
+ "tokens_seen": 1492189184
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044275225677031094,
+ "loss": 1.1758,
+ "theoretical_loss": 3.546150139833011,
+ "tokens_seen": 1492254720
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004427362086258777,
+ "loss": 1.2772,
+ "theoretical_loss": 3.5461365055762664,
+ "tokens_seen": 1492320256
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044272016048144434,
+ "loss": 1.3466,
+ "theoretical_loss": 3.546122872085906,
+ "tokens_seen": 1492385792
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004427041123370111,
+ "loss": 1.221,
+ "theoretical_loss": 3.546109239361855,
+ "tokens_seen": 1492451328
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044268806419257774,
+ "loss": 1.3131,
+ "theoretical_loss": 3.5460956074040357,
+ "tokens_seen": 1492516864
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004426720160481445,
+ "loss": 1.1914,
+ "theoretical_loss": 3.5460819762123714,
+ "tokens_seen": 1492582400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044265596790371114,
+ "loss": 1.3059,
+ "theoretical_loss": 3.546068345786785,
+ "tokens_seen": 1492647936
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004426399197592779,
+ "loss": 1.2706,
+ "theoretical_loss": 3.546054716127201,
+ "tokens_seen": 1492713472
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044262387161484454,
+ "loss": 1.3403,
+ "theoretical_loss": 3.5460410872335415,
+ "tokens_seen": 1492779008
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004426078234704113,
+ "loss": 1.3431,
+ "theoretical_loss": 3.5460274591057304,
+ "tokens_seen": 1492844544
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044259177532597794,
+ "loss": 1.3342,
+ "theoretical_loss": 3.546013831743691,
+ "tokens_seen": 1492910080
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004425757271815447,
+ "loss": 1.3687,
+ "theoretical_loss": 3.546000205147347,
+ "tokens_seen": 1492975616
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044255967903711135,
+ "loss": 1.2215,
+ "theoretical_loss": 3.545986579316621,
+ "tokens_seen": 1493041152
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004425436308926781,
+ "loss": 1.3595,
+ "theoretical_loss": 3.545972954251437,
+ "tokens_seen": 1493106688
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044252758274824475,
+ "loss": 1.2245,
+ "theoretical_loss": 3.5459593299517183,
+ "tokens_seen": 1493172224
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044251153460381145,
+ "loss": 1.2358,
+ "theoretical_loss": 3.545945706417388,
+ "tokens_seen": 1493237760
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044249548645937815,
+ "loss": 1.2601,
+ "theoretical_loss": 3.5459320836483696,
+ "tokens_seen": 1493303296
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044247943831494485,
+ "loss": 1.2364,
+ "theoretical_loss": 3.545918461644587,
+ "tokens_seen": 1493368832
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044246339017051155,
+ "loss": 1.3508,
+ "theoretical_loss": 3.5459048404059628,
+ "tokens_seen": 1493434368
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044244734202607825,
+ "loss": 1.2605,
+ "theoretical_loss": 3.5458912199324217,
+ "tokens_seen": 1493499904
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044243129388164495,
+ "loss": 1.183,
+ "theoretical_loss": 3.5458776002238857,
+ "tokens_seen": 1493565440
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044241524573721165,
+ "loss": 1.295,
+ "theoretical_loss": 3.545863981280279,
+ "tokens_seen": 1493630976
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004423991975927783,
+ "loss": 1.3173,
+ "theoretical_loss": 3.5458503631015255,
+ "tokens_seen": 1493696512
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044238314944834505,
+ "loss": 1.2528,
+ "theoretical_loss": 3.5458367456875477,
+ "tokens_seen": 1493762048
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004423671013039117,
+ "loss": 1.3235,
+ "theoretical_loss": 3.5458231290382694,
+ "tokens_seen": 1493827584
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044235105315947845,
+ "loss": 1.3364,
+ "theoretical_loss": 3.545809513153615,
+ "tokens_seen": 1493893120
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004423350050150451,
+ "loss": 1.2103,
+ "theoretical_loss": 3.545795898033507,
+ "tokens_seen": 1493958656
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044231895687061186,
+ "loss": 1.3311,
+ "theoretical_loss": 3.545782283677869,
+ "tokens_seen": 1494024192
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004423029087261785,
+ "loss": 1.2977,
+ "theoretical_loss": 3.545768670086625,
+ "tokens_seen": 1494089728
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044228686058174526,
+ "loss": 1.3293,
+ "theoretical_loss": 3.545755057259699,
+ "tokens_seen": 1494155264
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 500805,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4166927337646484,
+ "objective/train/theoretical_loss": 3.545741445197013,
+ "objective/train/tokens_used": 1514680800,
+ "theoretical_loss": 3.545741445197013,
+ "tokens_seen": 1494220800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004422708124373119,
+ "loss": 1.3803,
+ "theoretical_loss": 3.545741445197013,
+ "tokens_seen": 1494220800
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044225476429287866,
+ "loss": 1.2705,
+ "theoretical_loss": 3.5457278338984923,
+ "tokens_seen": 1494286336
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004422387161484453,
+ "loss": 1.2304,
+ "theoretical_loss": 3.545714223364059,
+ "tokens_seen": 1494351872
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044222266800401206,
+ "loss": 1.1331,
+ "theoretical_loss": 3.545700613593638,
+ "tokens_seen": 1494417408
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004422066198595787,
+ "loss": 1.2417,
+ "theoretical_loss": 3.545687004587152,
+ "tokens_seen": 1494482944
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044219057171514546,
+ "loss": 1.2671,
+ "theoretical_loss": 3.5456733963445255,
+ "tokens_seen": 1494548480
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004421745235707121,
+ "loss": 1.2321,
+ "theoretical_loss": 3.545659788865681,
+ "tokens_seen": 1494614016
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044215847542627886,
+ "loss": 1.3777,
+ "theoretical_loss": 3.545646182150543,
+ "tokens_seen": 1494679552
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004421424272818455,
+ "loss": 1.228,
+ "theoretical_loss": 3.5456325761990346,
+ "tokens_seen": 1494745088
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044212637913741226,
+ "loss": 1.2912,
+ "theoretical_loss": 3.5456189710110797,
+ "tokens_seen": 1494810624
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004421103309929789,
+ "loss": 1.3349,
+ "theoretical_loss": 3.5456053665866025,
+ "tokens_seen": 1494876160
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044209428284854566,
+ "loss": 1.2647,
+ "theoretical_loss": 3.545591762925526,
+ "tokens_seen": 1494941696
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004420782347041123,
+ "loss": 1.3457,
+ "theoretical_loss": 3.545578160027774,
+ "tokens_seen": 1495007232
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044206218655967907,
+ "loss": 1.2867,
+ "theoretical_loss": 3.5455645578932704,
+ "tokens_seen": 1495072768
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004420461384152457,
+ "loss": 1.169,
+ "theoretical_loss": 3.545550956521939,
+ "tokens_seen": 1495138304
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044203009027081247,
+ "loss": 1.2631,
+ "theoretical_loss": 3.5455373559137033,
+ "tokens_seen": 1495203840
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004420140421263792,
+ "loss": 1.2381,
+ "theoretical_loss": 3.5455237560684867,
+ "tokens_seen": 1495269376
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044199799398194587,
+ "loss": 1.234,
+ "theoretical_loss": 3.5455101569862135,
+ "tokens_seen": 1495334912
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004419819458375126,
+ "loss": 1.2497,
+ "theoretical_loss": 3.545496558666808,
+ "tokens_seen": 1495400448
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044196589769307927,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5454829611101926,
+ "tokens_seen": 1495465984
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441949849548646,
+ "loss": 1.1738,
+ "theoretical_loss": 3.5454693643162916,
+ "tokens_seen": 1495531520
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044193380140421267,
+ "loss": 1.1725,
+ "theoretical_loss": 3.54545576828503,
+ "tokens_seen": 1495597056
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004419177532597794,
+ "loss": 1.3059,
+ "theoretical_loss": 3.5454421730163297,
+ "tokens_seen": 1495662592
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044190170511534607,
+ "loss": 1.2743,
+ "theoretical_loss": 3.5454285785101156,
+ "tokens_seen": 1495728128
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044188565697091277,
+ "loss": 1.2422,
+ "theoretical_loss": 3.5454149847663112,
+ "tokens_seen": 1495793664
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004418696088264795,
+ "loss": 1.2817,
+ "theoretical_loss": 3.545401391784841,
+ "tokens_seen": 1495859200
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004418535606820462,
+ "loss": 1.3426,
+ "theoretical_loss": 3.545387799565628,
+ "tokens_seen": 1495924736
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004418375125376129,
+ "loss": 1.2575,
+ "theoretical_loss": 3.545374208108597,
+ "tokens_seen": 1495990272
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004418214643931796,
+ "loss": 1.3284,
+ "theoretical_loss": 3.5453606174136705,
+ "tokens_seen": 1496055808
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004418054162487463,
+ "loss": 1.3364,
+ "theoretical_loss": 3.5453470274807737,
+ "tokens_seen": 1496121344
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441789368104313,
+ "loss": 1.282,
+ "theoretical_loss": 3.5453334383098296,
+ "tokens_seen": 1496186880
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004417733199598797,
+ "loss": 1.2636,
+ "theoretical_loss": 3.5453198499007623,
+ "tokens_seen": 1496252416
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004417572718154464,
+ "loss": 1.3119,
+ "theoretical_loss": 3.5453062622534963,
+ "tokens_seen": 1496317952
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441741223671013,
+ "loss": 1.2526,
+ "theoretical_loss": 3.5452926753679552,
+ "tokens_seen": 1496383488
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004417251755265798,
+ "loss": 1.3335,
+ "theoretical_loss": 3.5452790892440627,
+ "tokens_seen": 1496449024
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004417091273821464,
+ "loss": 1.3197,
+ "theoretical_loss": 3.545265503881742,
+ "tokens_seen": 1496514560
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004416930792377132,
+ "loss": 1.3034,
+ "theoretical_loss": 3.5452519192809193,
+ "tokens_seen": 1496580096
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004416770310932798,
+ "loss": 1.3558,
+ "theoretical_loss": 3.5452383354415167,
+ "tokens_seen": 1496645632
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004416609829488466,
+ "loss": 1.2489,
+ "theoretical_loss": 3.545224752363459,
+ "tokens_seen": 1496711168
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044164493480441323,
+ "loss": 1.3406,
+ "theoretical_loss": 3.5452111700466697,
+ "tokens_seen": 1496776704
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044162888665998,
+ "loss": 1.3659,
+ "theoretical_loss": 3.5451975884910727,
+ "tokens_seen": 1496842240
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044161283851554663,
+ "loss": 1.2632,
+ "theoretical_loss": 3.5451840076965926,
+ "tokens_seen": 1496907776
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004415967903711134,
+ "loss": 1.333,
+ "theoretical_loss": 3.5451704276631535,
+ "tokens_seen": 1496973312
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044158074222668003,
+ "loss": 1.2692,
+ "theoretical_loss": 3.545156848390679,
+ "tokens_seen": 1497038848
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004415646940822468,
+ "loss": 1.246,
+ "theoretical_loss": 3.545143269879093,
+ "tokens_seen": 1497104384
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044154864593781343,
+ "loss": 1.3659,
+ "theoretical_loss": 3.5451296921283197,
+ "tokens_seen": 1497169920
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004415325977933802,
+ "loss": 1.2116,
+ "theoretical_loss": 3.5451161151382835,
+ "tokens_seen": 1497235456
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044151654964894683,
+ "loss": 1.3613,
+ "theoretical_loss": 3.5451025389089086,
+ "tokens_seen": 1497300992
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004415005015045136,
+ "loss": 1.3407,
+ "theoretical_loss": 3.545088963440118,
+ "tokens_seen": 1497366528
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044148445336008023,
+ "loss": 1.416,
+ "theoretical_loss": 3.545075388731837,
+ "tokens_seen": 1497432064
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 501929,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1920337677001953,
+ "objective/train/theoretical_loss": 3.545061814783989,
+ "objective/train/tokens_used": 1517957600,
+ "theoretical_loss": 3.545061814783989,
+ "tokens_seen": 1497497600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441468405215647,
+ "loss": 1.2595,
+ "theoretical_loss": 3.545061814783989,
+ "tokens_seen": 1497497600
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044145235707121364,
+ "loss": 1.32,
+ "theoretical_loss": 3.5450482415964992,
+ "tokens_seen": 1497563136
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004414363089267804,
+ "loss": 1.2764,
+ "theoretical_loss": 3.54503466916929,
+ "tokens_seen": 1497628672
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044142026078234704,
+ "loss": 1.2148,
+ "theoretical_loss": 3.5450210975022873,
+ "tokens_seen": 1497694208
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004414042126379138,
+ "loss": 1.3527,
+ "theoretical_loss": 3.545007526595414,
+ "tokens_seen": 1497759744
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044138816449348044,
+ "loss": 1.3511,
+ "theoretical_loss": 3.544993956448595,
+ "tokens_seen": 1497825280
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004413721163490472,
+ "loss": 1.2421,
+ "theoretical_loss": 3.544980387061753,
+ "tokens_seen": 1497890816
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044135606820461384,
+ "loss": 1.2499,
+ "theoretical_loss": 3.5449668184348146,
+ "tokens_seen": 1497956352
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004413400200601806,
+ "loss": 1.2888,
+ "theoretical_loss": 3.5449532505677026,
+ "tokens_seen": 1498021888
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044132397191574724,
+ "loss": 1.2662,
+ "theoretical_loss": 3.544939683460341,
+ "tokens_seen": 1498087424
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441307923771314,
+ "loss": 1.2896,
+ "theoretical_loss": 3.544926117112655,
+ "tokens_seen": 1498152960
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044129187562688064,
+ "loss": 1.2332,
+ "theoretical_loss": 3.544912551524568,
+ "tokens_seen": 1498218496
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004412758274824474,
+ "loss": 1.2381,
+ "theoretical_loss": 3.5448989866960043,
+ "tokens_seen": 1498284032
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044125977933801404,
+ "loss": 1.2528,
+ "theoretical_loss": 3.544885422626889,
+ "tokens_seen": 1498349568
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004412437311935808,
+ "loss": 1.2944,
+ "theoretical_loss": 3.5448718593171447,
+ "tokens_seen": 1498415104
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044122768304914744,
+ "loss": 1.2595,
+ "theoretical_loss": 3.544858296766697,
+ "tokens_seen": 1498480640
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004412116349047142,
+ "loss": 1.3792,
+ "theoretical_loss": 3.54484473497547,
+ "tokens_seen": 1498546176
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044119558676028085,
+ "loss": 1.2765,
+ "theoretical_loss": 3.5448311739433884,
+ "tokens_seen": 1498611712
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004411795386158476,
+ "loss": 1.2502,
+ "theoretical_loss": 3.5448176136703755,
+ "tokens_seen": 1498677248
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044116349047141425,
+ "loss": 1.2886,
+ "theoretical_loss": 3.544804054156356,
+ "tokens_seen": 1498742784
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000441147442326981,
+ "loss": 1.3088,
+ "theoretical_loss": 3.544790495401255,
+ "tokens_seen": 1498808320
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044113139418254765,
+ "loss": 1.3935,
+ "theoretical_loss": 3.5447769374049956,
+ "tokens_seen": 1498873856
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044111534603811435,
+ "loss": 1.2399,
+ "theoretical_loss": 3.544763380167503,
+ "tokens_seen": 1498939392
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044109929789368105,
+ "loss": 1.4056,
+ "theoretical_loss": 3.5447498236887007,
+ "tokens_seen": 1499004928
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044108324974924775,
+ "loss": 1.2911,
+ "theoretical_loss": 3.5447362679685144,
+ "tokens_seen": 1499070464
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044106720160481445,
+ "loss": 1.3031,
+ "theoretical_loss": 3.544722713006868,
+ "tokens_seen": 1499136000
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044105115346038115,
+ "loss": 1.2811,
+ "theoretical_loss": 3.544709158803685,
+ "tokens_seen": 1499201536
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044103510531594785,
+ "loss": 1.2504,
+ "theoretical_loss": 3.5446956053588905,
+ "tokens_seen": 1499267072
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044101905717151455,
+ "loss": 1.3346,
+ "theoretical_loss": 3.544682052672409,
+ "tokens_seen": 1499332608
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044100300902708125,
+ "loss": 1.1265,
+ "theoretical_loss": 3.544668500744165,
+ "tokens_seen": 1499398144
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044098696088264795,
+ "loss": 1.3361,
+ "theoretical_loss": 3.5446549495740824,
+ "tokens_seen": 1499463680
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004409709127382146,
+ "loss": 1.2644,
+ "theoretical_loss": 3.5446413991620864,
+ "tokens_seen": 1499529216
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044095486459378136,
+ "loss": 1.3488,
+ "theoretical_loss": 3.544627849508101,
+ "tokens_seen": 1499594752
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.000440938816449348,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5446143006120505,
+ "tokens_seen": 1499660288
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044092276830491476,
+ "loss": 1.3208,
+ "theoretical_loss": 3.54460075247386,
+ "tokens_seen": 1499725824
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004409067201604814,
+ "loss": 1.3258,
+ "theoretical_loss": 3.5445872050934533,
+ "tokens_seen": 1499791360
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044089067201604816,
+ "loss": 1.1752,
+ "theoretical_loss": 3.5445736584707555,
+ "tokens_seen": 1499856896
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004408746238716148,
+ "loss": 1.2849,
+ "theoretical_loss": 3.5445601126056907,
+ "tokens_seen": 1499922432
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044085857572718156,
+ "loss": 1.3171,
+ "theoretical_loss": 3.544546567498183,
+ "tokens_seen": 1499987968
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004408425275827482,
+ "loss": 1.2543,
+ "theoretical_loss": 3.5445330231481584,
+ "tokens_seen": 1500053504
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044082647943831496,
+ "loss": 1.2205,
+ "theoretical_loss": 3.5445194795555404,
+ "tokens_seen": 1500119040
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004408104312938817,
+ "loss": 1.3444,
+ "theoretical_loss": 3.5445059367202534,
+ "tokens_seen": 1500184576
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044079438314944836,
+ "loss": 1.3537,
+ "theoretical_loss": 3.5444923946422224,
+ "tokens_seen": 1500250112
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004407783350050151,
+ "loss": 1.3226,
+ "theoretical_loss": 3.5444788533213725,
+ "tokens_seen": 1500315648
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044076228686058176,
+ "loss": 1.3159,
+ "theoretical_loss": 3.5444653127576267,
+ "tokens_seen": 1500381184
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004407462387161485,
+ "loss": 1.265,
+ "theoretical_loss": 3.544451772950911,
+ "tokens_seen": 1500446720
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044073019057171516,
+ "loss": 1.3354,
+ "theoretical_loss": 3.5444382339011495,
+ "tokens_seen": 1500512256
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004407141424272819,
+ "loss": 1.3545,
+ "theoretical_loss": 3.544424695608267,
+ "tokens_seen": 1500577792
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044069809428284857,
+ "loss": 1.2056,
+ "theoretical_loss": 3.5444111580721875,
+ "tokens_seen": 1500643328
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004406820461384153,
+ "loss": 1.2853,
+ "theoretical_loss": 3.5443976212928368,
+ "tokens_seen": 1500708864
+ },
+ {
+ "epoch": 0.45,
+ "objective/train/docs_used": 502894,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.272984266281128,
+ "objective/train/theoretical_loss": 3.544384085270139,
+ "objective/train/tokens_used": 1521234400,
+ "theoretical_loss": 3.544384085270139,
+ "tokens_seen": 1500774400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044066599799398197,
+ "loss": 1.3016,
+ "theoretical_loss": 3.544384085270139,
+ "tokens_seen": 1500774400
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004406499498495487,
+ "loss": 1.2469,
+ "theoretical_loss": 3.544370550004018,
+ "tokens_seen": 1500839936
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044063390170511537,
+ "loss": 1.3869,
+ "theoretical_loss": 3.5443570154944,
+ "tokens_seen": 1500905472
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004406178535606821,
+ "loss": 1.323,
+ "theoretical_loss": 3.5443434817412083,
+ "tokens_seen": 1500971008
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044060180541624877,
+ "loss": 1.2838,
+ "theoretical_loss": 3.5443299487443687,
+ "tokens_seen": 1501036544
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004405857572718155,
+ "loss": 1.256,
+ "theoretical_loss": 3.544316416503805,
+ "tokens_seen": 1501102080
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044056970912738217,
+ "loss": 1.3683,
+ "theoretical_loss": 3.5443028850194427,
+ "tokens_seen": 1501167616
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004405536609829489,
+ "loss": 1.334,
+ "theoretical_loss": 3.5442893542912053,
+ "tokens_seen": 1501233152
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044053761283851557,
+ "loss": 1.3719,
+ "theoretical_loss": 3.544275824319019,
+ "tokens_seen": 1501298688
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004405215646940823,
+ "loss": 1.2759,
+ "theoretical_loss": 3.5442622951028078,
+ "tokens_seen": 1501364224
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.00044050551654964897,
+ "loss": 1.2972,
+ "theoretical_loss": 3.5442487666424967,
+ "tokens_seen": 1501429760
+ },
+ {
+ "epoch": 0.45,
+ "learning_rate": 0.0004404894684052157,
+ "loss": 1.3218,
+ "theoretical_loss": 3.5442352389380103,
+ "tokens_seen": 1501495296
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004404734202607824,
+ "loss": 1.2625,
+ "theoretical_loss": 3.544221711989273,
+ "tokens_seen": 1501560832
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004404573721163491,
+ "loss": 1.3805,
+ "theoretical_loss": 3.544208185796211,
+ "tokens_seen": 1501626368
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004404413239719158,
+ "loss": 1.3547,
+ "theoretical_loss": 3.5441946603587478,
+ "tokens_seen": 1501691904
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004404252758274825,
+ "loss": 1.349,
+ "theoretical_loss": 3.5441811356768085,
+ "tokens_seen": 1501757440
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004404092276830492,
+ "loss": 1.3398,
+ "theoretical_loss": 3.544167611750318,
+ "tokens_seen": 1501822976
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403931795386159,
+ "loss": 1.2158,
+ "theoretical_loss": 3.5441540885792007,
+ "tokens_seen": 1501888512
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403771313941826,
+ "loss": 1.2974,
+ "theoretical_loss": 3.5441405661633825,
+ "tokens_seen": 1501954048
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403610832497493,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5441270445027873,
+ "tokens_seen": 1502019584
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403450351053159,
+ "loss": 1.2833,
+ "theoretical_loss": 3.5441135235973404,
+ "tokens_seen": 1502085120
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403289869608827,
+ "loss": 1.3349,
+ "theoretical_loss": 3.5441000034469665,
+ "tokens_seen": 1502150656
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004403129388164493,
+ "loss": 1.3148,
+ "theoretical_loss": 3.544086484051591,
+ "tokens_seen": 1502216192
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004402968906720161,
+ "loss": 1.3139,
+ "theoretical_loss": 3.5440729654111385,
+ "tokens_seen": 1502281728
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044028084252758273,
+ "loss": 1.3666,
+ "theoretical_loss": 3.544059447525533,
+ "tokens_seen": 1502347264
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004402647943831495,
+ "loss": 1.342,
+ "theoretical_loss": 3.544045930394701,
+ "tokens_seen": 1502412800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044024874623871613,
+ "loss": 1.2179,
+ "theoretical_loss": 3.5440324140185666,
+ "tokens_seen": 1502478336
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004402326980942829,
+ "loss": 1.3526,
+ "theoretical_loss": 3.5440188983970544,
+ "tokens_seen": 1502543872
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044021664994984953,
+ "loss": 1.3061,
+ "theoretical_loss": 3.54400538353009,
+ "tokens_seen": 1502609408
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004402006018054163,
+ "loss": 1.2093,
+ "theoretical_loss": 3.5439918694175985,
+ "tokens_seen": 1502674944
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044018455366098293,
+ "loss": 1.2408,
+ "theoretical_loss": 3.543978356059504,
+ "tokens_seen": 1502740480
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004401685055165497,
+ "loss": 1.3752,
+ "theoretical_loss": 3.5439648434557327,
+ "tokens_seen": 1502806016
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044015245737211633,
+ "loss": 1.1801,
+ "theoretical_loss": 3.543951331606208,
+ "tokens_seen": 1502871552
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004401364092276831,
+ "loss": 1.2551,
+ "theoretical_loss": 3.543937820510857,
+ "tokens_seen": 1502937088
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044012036108324973,
+ "loss": 1.1826,
+ "theoretical_loss": 3.5439243101696025,
+ "tokens_seen": 1503002624
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004401043129388165,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5439108005823714,
+ "tokens_seen": 1503068160
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044008826479438314,
+ "loss": 1.3107,
+ "theoretical_loss": 3.5438972917490874,
+ "tokens_seen": 1503133696
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004400722166499499,
+ "loss": 1.2832,
+ "theoretical_loss": 3.5438837836696764,
+ "tokens_seen": 1503199232
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044005616850551654,
+ "loss": 1.3399,
+ "theoretical_loss": 3.5438702763440633,
+ "tokens_seen": 1503264768
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004400401203610833,
+ "loss": 1.268,
+ "theoretical_loss": 3.5438567697721726,
+ "tokens_seen": 1503330304
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00044002407221664994,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5438432639539297,
+ "tokens_seen": 1503395840
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004400080240722167,
+ "loss": 1.3275,
+ "theoretical_loss": 3.5438297588892604,
+ "tokens_seen": 1503461376
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043999197592778334,
+ "loss": 1.3376,
+ "theoretical_loss": 3.543816254578089,
+ "tokens_seen": 1503526912
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004399759277833501,
+ "loss": 1.3758,
+ "theoretical_loss": 3.5438027510203405,
+ "tokens_seen": 1503592448
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043995987963891674,
+ "loss": 1.3116,
+ "theoretical_loss": 3.5437892482159405,
+ "tokens_seen": 1503657984
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004399438314944835,
+ "loss": 1.2753,
+ "theoretical_loss": 3.5437757461648145,
+ "tokens_seen": 1503723520
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043992778335005014,
+ "loss": 1.3406,
+ "theoretical_loss": 3.543762244866887,
+ "tokens_seen": 1503789056
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004399117352056169,
+ "loss": 1.3919,
+ "theoretical_loss": 3.543748744322083,
+ "tokens_seen": 1503854592
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043989568706118354,
+ "loss": 1.1844,
+ "theoretical_loss": 3.5437352445303283,
+ "tokens_seen": 1503920128
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004398796389167503,
+ "loss": 1.2561,
+ "theoretical_loss": 3.5437217454915473,
+ "tokens_seen": 1503985664
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 504139,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3649600744247437,
+ "objective/train/theoretical_loss": 3.5437082472056662,
+ "objective/train/tokens_used": 1524511200,
+ "theoretical_loss": 3.5437082472056662,
+ "tokens_seen": 1504051200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043986359077231694,
+ "loss": 1.2784,
+ "theoretical_loss": 3.5437082472056662,
+ "tokens_seen": 1504051200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004398475426278837,
+ "loss": 1.2817,
+ "theoretical_loss": 3.5436947496726097,
+ "tokens_seen": 1504116736
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043983149448345035,
+ "loss": 1.3325,
+ "theoretical_loss": 3.5436812528923025,
+ "tokens_seen": 1504182272
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004398154463390171,
+ "loss": 1.3115,
+ "theoretical_loss": 3.5436677568646706,
+ "tokens_seen": 1504247808
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043979939819458375,
+ "loss": 1.3038,
+ "theoretical_loss": 3.5436542615896394,
+ "tokens_seen": 1504313344
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004397833500501505,
+ "loss": 1.1666,
+ "theoretical_loss": 3.5436407670671333,
+ "tokens_seen": 1504378880
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043976730190571715,
+ "loss": 1.237,
+ "theoretical_loss": 3.5436272732970777,
+ "tokens_seen": 1504444416
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004397512537612839,
+ "loss": 1.2513,
+ "theoretical_loss": 3.543613780279398,
+ "tokens_seen": 1504509952
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043973520561685055,
+ "loss": 1.1959,
+ "theoretical_loss": 3.5436002880140203,
+ "tokens_seen": 1504575488
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043971915747241725,
+ "loss": 1.3554,
+ "theoretical_loss": 3.543586796500869,
+ "tokens_seen": 1504641024
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043970310932798395,
+ "loss": 1.2286,
+ "theoretical_loss": 3.5435733057398693,
+ "tokens_seen": 1504706560
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043968706118355065,
+ "loss": 1.3703,
+ "theoretical_loss": 3.5435598157309474,
+ "tokens_seen": 1504772096
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043967101303911735,
+ "loss": 1.2902,
+ "theoretical_loss": 3.5435463264740275,
+ "tokens_seen": 1504837632
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043965496489468405,
+ "loss": 1.3072,
+ "theoretical_loss": 3.5435328379690354,
+ "tokens_seen": 1504903168
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004396389167502508,
+ "loss": 1.2065,
+ "theoretical_loss": 3.543519350215897,
+ "tokens_seen": 1504968704
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043962286860581745,
+ "loss": 1.3759,
+ "theoretical_loss": 3.543505863214537,
+ "tokens_seen": 1505034240
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004396068204613842,
+ "loss": 1.2626,
+ "theoretical_loss": 3.5434923769648803,
+ "tokens_seen": 1505099776
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043959077231695085,
+ "loss": 1.2391,
+ "theoretical_loss": 3.5434788914668536,
+ "tokens_seen": 1505165312
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004395747241725176,
+ "loss": 1.2324,
+ "theoretical_loss": 3.5434654067203812,
+ "tokens_seen": 1505230848
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043955867602808426,
+ "loss": 1.2531,
+ "theoretical_loss": 3.543451922725389,
+ "tokens_seen": 1505296384
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000439542627883651,
+ "loss": 1.3445,
+ "theoretical_loss": 3.5434384394818026,
+ "tokens_seen": 1505361920
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043952657973921766,
+ "loss": 1.3479,
+ "theoretical_loss": 3.543424956989547,
+ "tokens_seen": 1505427456
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004395105315947844,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5434114752485475,
+ "tokens_seen": 1505492992
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043949448345035106,
+ "loss": 1.3928,
+ "theoretical_loss": 3.54339799425873,
+ "tokens_seen": 1505558528
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004394784353059178,
+ "loss": 1.235,
+ "theoretical_loss": 3.5433845140200195,
+ "tokens_seen": 1505624064
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043946238716148446,
+ "loss": 1.3208,
+ "theoretical_loss": 3.543371034532342,
+ "tokens_seen": 1505689600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004394463390170512,
+ "loss": 1.2994,
+ "theoretical_loss": 3.5433575557956223,
+ "tokens_seen": 1505755136
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043943029087261786,
+ "loss": 1.3554,
+ "theoretical_loss": 3.5433440778097864,
+ "tokens_seen": 1505820672
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004394142427281846,
+ "loss": 1.3126,
+ "theoretical_loss": 3.5433306005747593,
+ "tokens_seen": 1505886208
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043939819458375126,
+ "loss": 1.3214,
+ "theoretical_loss": 3.5433171240904673,
+ "tokens_seen": 1505951744
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000439382146439318,
+ "loss": 1.208,
+ "theoretical_loss": 3.543303648356835,
+ "tokens_seen": 1506017280
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043936609829488466,
+ "loss": 1.2476,
+ "theoretical_loss": 3.543290173373789,
+ "tokens_seen": 1506082816
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004393500501504514,
+ "loss": 1.3929,
+ "theoretical_loss": 3.543276699141254,
+ "tokens_seen": 1506148352
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043933400200601806,
+ "loss": 1.2842,
+ "theoretical_loss": 3.5432632256591554,
+ "tokens_seen": 1506213888
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004393179538615848,
+ "loss": 1.3316,
+ "theoretical_loss": 3.5432497529274194,
+ "tokens_seen": 1506279424
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043930190571715147,
+ "loss": 1.2433,
+ "theoretical_loss": 3.543236280945971,
+ "tokens_seen": 1506344960
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004392858575727182,
+ "loss": 1.2448,
+ "theoretical_loss": 3.5432228097147362,
+ "tokens_seen": 1506410496
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043926980942828487,
+ "loss": 1.3293,
+ "theoretical_loss": 3.5432093392336403,
+ "tokens_seen": 1506476032
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004392537612838516,
+ "loss": 1.3229,
+ "theoretical_loss": 3.5431958695026093,
+ "tokens_seen": 1506541568
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043923771313941827,
+ "loss": 1.3073,
+ "theoretical_loss": 3.5431824005215686,
+ "tokens_seen": 1506607104
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000439221664994985,
+ "loss": 1.2099,
+ "theoretical_loss": 3.5431689322904436,
+ "tokens_seen": 1506672640
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043920561685055167,
+ "loss": 1.2974,
+ "theoretical_loss": 3.54315546480916,
+ "tokens_seen": 1506738176
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004391895687061184,
+ "loss": 1.3451,
+ "theoretical_loss": 3.543141998077644,
+ "tokens_seen": 1506803712
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043917352056168507,
+ "loss": 1.171,
+ "theoretical_loss": 3.54312853209582,
+ "tokens_seen": 1506869248
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004391574724172518,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5431150668636153,
+ "tokens_seen": 1506934784
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043914142427281847,
+ "loss": 1.2949,
+ "theoretical_loss": 3.543101602380954,
+ "tokens_seen": 1507000320
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043912537612838523,
+ "loss": 1.2533,
+ "theoretical_loss": 3.5430881386477626,
+ "tokens_seen": 1507065856
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004391093279839519,
+ "loss": 1.2842,
+ "theoretical_loss": 3.5430746756639673,
+ "tokens_seen": 1507131392
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004390932798395186,
+ "loss": 1.4479,
+ "theoretical_loss": 3.5430612134294925,
+ "tokens_seen": 1507196928
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004390772316950853,
+ "loss": 1.2583,
+ "theoretical_loss": 3.5430477519442647,
+ "tokens_seen": 1507262464
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 505087,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2310185432434082,
+ "objective/train/theoretical_loss": 3.54303429120821,
+ "objective/train/tokens_used": 1527788000,
+ "theoretical_loss": 3.54303429120821,
+ "tokens_seen": 1507328000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000439061183550652,
+ "loss": 1.3043,
+ "theoretical_loss": 3.54303429120821,
+ "tokens_seen": 1507328000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004390451354062187,
+ "loss": 1.4286,
+ "theoretical_loss": 3.5430208312212534,
+ "tokens_seen": 1507393536
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004390290872617854,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5430073719833213,
+ "tokens_seen": 1507459072
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004390130391173521,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5429939134943385,
+ "tokens_seen": 1507524608
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004389969909729188,
+ "loss": 1.2838,
+ "theoretical_loss": 3.5429804557542317,
+ "tokens_seen": 1507590144
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004389809428284855,
+ "loss": 1.2117,
+ "theoretical_loss": 3.5429669987629264,
+ "tokens_seen": 1507655680
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004389648946840522,
+ "loss": 1.3165,
+ "theoretical_loss": 3.542953542520348,
+ "tokens_seen": 1507721216
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004389488465396188,
+ "loss": 1.2695,
+ "theoretical_loss": 3.5429400870264227,
+ "tokens_seen": 1507786752
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004389327983951856,
+ "loss": 1.3088,
+ "theoretical_loss": 3.542926632281076,
+ "tokens_seen": 1507852288
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043891675025075223,
+ "loss": 1.2226,
+ "theoretical_loss": 3.542913178284235,
+ "tokens_seen": 1507917824
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000438900702106319,
+ "loss": 1.2687,
+ "theoretical_loss": 3.5428997250358236,
+ "tokens_seen": 1507983360
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043888465396188563,
+ "loss": 1.3363,
+ "theoretical_loss": 3.5428862725357684,
+ "tokens_seen": 1508048896
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004388686058174524,
+ "loss": 1.308,
+ "theoretical_loss": 3.5428728207839955,
+ "tokens_seen": 1508114432
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043885255767301903,
+ "loss": 1.2982,
+ "theoretical_loss": 3.542859369780431,
+ "tokens_seen": 1508179968
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004388365095285858,
+ "loss": 1.2222,
+ "theoretical_loss": 3.542845919525,
+ "tokens_seen": 1508245504
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043882046138415243,
+ "loss": 1.2818,
+ "theoretical_loss": 3.542832470017629,
+ "tokens_seen": 1508311040
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004388044132397192,
+ "loss": 1.2244,
+ "theoretical_loss": 3.5428190212582438,
+ "tokens_seen": 1508376576
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043878836509528583,
+ "loss": 1.2179,
+ "theoretical_loss": 3.54280557324677,
+ "tokens_seen": 1508442112
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004387723169508526,
+ "loss": 1.2499,
+ "theoretical_loss": 3.5427921259831336,
+ "tokens_seen": 1508507648
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043875626880641923,
+ "loss": 1.2349,
+ "theoretical_loss": 3.5427786794672613,
+ "tokens_seen": 1508573184
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000438740220661986,
+ "loss": 1.3937,
+ "theoretical_loss": 3.542765233699078,
+ "tokens_seen": 1508638720
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043872417251755263,
+ "loss": 1.2186,
+ "theoretical_loss": 3.5427517886785096,
+ "tokens_seen": 1508704256
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004387081243731194,
+ "loss": 1.1481,
+ "theoretical_loss": 3.5427383444054827,
+ "tokens_seen": 1508769792
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043869207622868604,
+ "loss": 1.3161,
+ "theoretical_loss": 3.5427249008799233,
+ "tokens_seen": 1508835328
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004386760280842528,
+ "loss": 1.1781,
+ "theoretical_loss": 3.542711458101757,
+ "tokens_seen": 1508900864
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043865997993981944,
+ "loss": 1.2086,
+ "theoretical_loss": 3.54269801607091,
+ "tokens_seen": 1508966400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004386439317953862,
+ "loss": 1.241,
+ "theoretical_loss": 3.542684574787308,
+ "tokens_seen": 1509031936
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043862788365095284,
+ "loss": 1.206,
+ "theoretical_loss": 3.5426711342508774,
+ "tokens_seen": 1509097472
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004386118355065196,
+ "loss": 1.2558,
+ "theoretical_loss": 3.542657694461544,
+ "tokens_seen": 1509163008
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043859578736208624,
+ "loss": 1.2302,
+ "theoretical_loss": 3.5426442554192343,
+ "tokens_seen": 1509228544
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000438579739217653,
+ "loss": 1.2997,
+ "theoretical_loss": 3.5426308171238734,
+ "tokens_seen": 1509294080
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043856369107321964,
+ "loss": 1.2498,
+ "theoretical_loss": 3.542617379575388,
+ "tokens_seen": 1509359616
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004385476429287864,
+ "loss": 1.3063,
+ "theoretical_loss": 3.542603942773704,
+ "tokens_seen": 1509425152
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043853159478435304,
+ "loss": 1.2168,
+ "theoretical_loss": 3.5425905067187475,
+ "tokens_seen": 1509490688
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004385155466399198,
+ "loss": 1.21,
+ "theoretical_loss": 3.542577071410445,
+ "tokens_seen": 1509556224
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043849949849548644,
+ "loss": 1.2149,
+ "theoretical_loss": 3.5425636368487217,
+ "tokens_seen": 1509621760
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004384834503510532,
+ "loss": 1.3009,
+ "theoretical_loss": 3.542550203033504,
+ "tokens_seen": 1509687296
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043846740220661984,
+ "loss": 1.2862,
+ "theoretical_loss": 3.542536769964719,
+ "tokens_seen": 1509752832
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004384513540621866,
+ "loss": 1.3723,
+ "theoretical_loss": 3.542523337642292,
+ "tokens_seen": 1509818368
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004384353059177533,
+ "loss": 1.264,
+ "theoretical_loss": 3.5425099060661487,
+ "tokens_seen": 1509883904
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043841925777332,
+ "loss": 1.234,
+ "theoretical_loss": 3.5424964752362156,
+ "tokens_seen": 1509949440
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004384032096288867,
+ "loss": 1.2317,
+ "theoretical_loss": 3.5424830451524194,
+ "tokens_seen": 1510014976
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004383871614844534,
+ "loss": 1.3667,
+ "theoretical_loss": 3.542469615814686,
+ "tokens_seen": 1510080512
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004383711133400201,
+ "loss": 1.2616,
+ "theoretical_loss": 3.542456187222941,
+ "tokens_seen": 1510146048
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004383550651955868,
+ "loss": 1.3187,
+ "theoretical_loss": 3.5424427593771117,
+ "tokens_seen": 1510211584
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004383390170511535,
+ "loss": 1.3061,
+ "theoretical_loss": 3.542429332277123,
+ "tokens_seen": 1510277120
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043832296890672015,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5424159059229017,
+ "tokens_seen": 1510342656
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004383069207622869,
+ "loss": 1.2588,
+ "theoretical_loss": 3.542402480314374,
+ "tokens_seen": 1510408192
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043829087261785355,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5423890554514665,
+ "tokens_seen": 1510473728
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004382748244734203,
+ "loss": 1.2576,
+ "theoretical_loss": 3.5423756313341057,
+ "tokens_seen": 1510539264
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 506269,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1935040950775146,
+ "objective/train/theoretical_loss": 3.542362207962216,
+ "objective/train/tokens_used": 1531064800,
+ "theoretical_loss": 3.542362207962216,
+ "tokens_seen": 1510604800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043825877632898695,
+ "loss": 1.1419,
+ "theoretical_loss": 3.542362207962216,
+ "tokens_seen": 1510604800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004382427281845537,
+ "loss": 1.3459,
+ "theoretical_loss": 3.542348785335726,
+ "tokens_seen": 1510670336
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043822668004012035,
+ "loss": 1.27,
+ "theoretical_loss": 3.5423353634545607,
+ "tokens_seen": 1510735872
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004382106318956871,
+ "loss": 1.2022,
+ "theoretical_loss": 3.5423219423186465,
+ "tokens_seen": 1510801408
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043819458375125376,
+ "loss": 1.2414,
+ "theoretical_loss": 3.54230852192791,
+ "tokens_seen": 1510866944
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004381785356068205,
+ "loss": 1.3336,
+ "theoretical_loss": 3.542295102282277,
+ "tokens_seen": 1510932480
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043816248746238716,
+ "loss": 1.1944,
+ "theoretical_loss": 3.542281683381674,
+ "tokens_seen": 1510998016
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004381464393179539,
+ "loss": 1.0949,
+ "theoretical_loss": 3.542268265226028,
+ "tokens_seen": 1511063552
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043813039117352056,
+ "loss": 1.3464,
+ "theoretical_loss": 3.5422548478152645,
+ "tokens_seen": 1511129088
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004381143430290873,
+ "loss": 1.2643,
+ "theoretical_loss": 3.54224143114931,
+ "tokens_seen": 1511194624
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043809829488465396,
+ "loss": 1.2565,
+ "theoretical_loss": 3.542228015228091,
+ "tokens_seen": 1511260160
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004380822467402207,
+ "loss": 1.2449,
+ "theoretical_loss": 3.542214600051534,
+ "tokens_seen": 1511325696
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043806619859578736,
+ "loss": 1.1884,
+ "theoretical_loss": 3.5422011856195654,
+ "tokens_seen": 1511391232
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004380501504513541,
+ "loss": 1.1885,
+ "theoretical_loss": 3.5421877719321113,
+ "tokens_seen": 1511456768
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043803410230692076,
+ "loss": 1.3002,
+ "theoretical_loss": 3.542174358989098,
+ "tokens_seen": 1511522304
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004380180541624875,
+ "loss": 1.2442,
+ "theoretical_loss": 3.5421609467904522,
+ "tokens_seen": 1511587840
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043800200601805416,
+ "loss": 1.2248,
+ "theoretical_loss": 3.5421475353361007,
+ "tokens_seen": 1511653376
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004379859578736209,
+ "loss": 1.2079,
+ "theoretical_loss": 3.542134124625969,
+ "tokens_seen": 1511718912
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043796990972918756,
+ "loss": 1.179,
+ "theoretical_loss": 3.5421207146599842,
+ "tokens_seen": 1511784448
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004379538615847543,
+ "loss": 1.2244,
+ "theoretical_loss": 3.5421073054380727,
+ "tokens_seen": 1511849984
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043793781344032097,
+ "loss": 1.2915,
+ "theoretical_loss": 3.5420938969601607,
+ "tokens_seen": 1511915520
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004379217652958877,
+ "loss": 1.2193,
+ "theoretical_loss": 3.5420804892261746,
+ "tokens_seen": 1511981056
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043790571715145437,
+ "loss": 1.2582,
+ "theoretical_loss": 3.5420670822360414,
+ "tokens_seen": 1512046592
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004378896690070211,
+ "loss": 1.4046,
+ "theoretical_loss": 3.5420536759896875,
+ "tokens_seen": 1512112128
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043787362086258777,
+ "loss": 1.3198,
+ "theoretical_loss": 3.542040270487039,
+ "tokens_seen": 1512177664
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004378575727181545,
+ "loss": 1.2916,
+ "theoretical_loss": 3.5420268657280225,
+ "tokens_seen": 1512243200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043784152457372117,
+ "loss": 1.2808,
+ "theoretical_loss": 3.542013461712565,
+ "tokens_seen": 1512308736
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004378254764292879,
+ "loss": 1.2988,
+ "theoretical_loss": 3.542000058440592,
+ "tokens_seen": 1512374272
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043780942828485457,
+ "loss": 1.3036,
+ "theoretical_loss": 3.5419866559120314,
+ "tokens_seen": 1512439808
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004377933801404213,
+ "loss": 1.2818,
+ "theoretical_loss": 3.541973254126809,
+ "tokens_seen": 1512505344
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043777733199598797,
+ "loss": 1.3089,
+ "theoretical_loss": 3.541959853084851,
+ "tokens_seen": 1512570880
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004377612838515547,
+ "loss": 1.1615,
+ "theoretical_loss": 3.5419464527860844,
+ "tokens_seen": 1512636416
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004377452357071214,
+ "loss": 1.3224,
+ "theoretical_loss": 3.5419330532304363,
+ "tokens_seen": 1512701952
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043772918756268813,
+ "loss": 1.2271,
+ "theoretical_loss": 3.541919654417833,
+ "tokens_seen": 1512767488
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004377131394182548,
+ "loss": 1.2903,
+ "theoretical_loss": 3.5419062563482004,
+ "tokens_seen": 1512833024
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376970912738215,
+ "loss": 1.3538,
+ "theoretical_loss": 3.5418928590214653,
+ "tokens_seen": 1512898560
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376810431293882,
+ "loss": 1.304,
+ "theoretical_loss": 3.5418794624375556,
+ "tokens_seen": 1512964096
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376649949849549,
+ "loss": 1.2842,
+ "theoretical_loss": 3.5418660665963966,
+ "tokens_seen": 1513029632
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376489468405216,
+ "loss": 1.1832,
+ "theoretical_loss": 3.541852671497915,
+ "tokens_seen": 1513095168
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376328986960883,
+ "loss": 1.4902,
+ "theoretical_loss": 3.541839277142038,
+ "tokens_seen": 1513160704
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000437616850551655,
+ "loss": 1.2467,
+ "theoretical_loss": 3.541825883528692,
+ "tokens_seen": 1513226240
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004376008024072217,
+ "loss": 1.3917,
+ "theoretical_loss": 3.5418124906578043,
+ "tokens_seen": 1513291776
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004375847542627884,
+ "loss": 1.2329,
+ "theoretical_loss": 3.541799098529301,
+ "tokens_seen": 1513357312
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004375687061183551,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5417857071431085,
+ "tokens_seen": 1513422848
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004375526579739217,
+ "loss": 1.3836,
+ "theoretical_loss": 3.541772316499154,
+ "tokens_seen": 1513488384
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004375366098294885,
+ "loss": 1.1754,
+ "theoretical_loss": 3.541758926597364,
+ "tokens_seen": 1513553920
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043752056168505513,
+ "loss": 1.3074,
+ "theoretical_loss": 3.541745537437665,
+ "tokens_seen": 1513619456
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004375045135406219,
+ "loss": 1.266,
+ "theoretical_loss": 3.541732149019985,
+ "tokens_seen": 1513684992
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043748846539618853,
+ "loss": 1.3281,
+ "theoretical_loss": 3.5417187613442493,
+ "tokens_seen": 1513750528
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004374724172517553,
+ "loss": 1.2361,
+ "theoretical_loss": 3.541705374410385,
+ "tokens_seen": 1513816064
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 507531,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.190418004989624,
+ "objective/train/theoretical_loss": 3.5416919882183193,
+ "objective/train/tokens_used": 1534341600,
+ "theoretical_loss": 3.5416919882183193,
+ "tokens_seen": 1513881600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043745636910732193,
+ "loss": 1.2266,
+ "theoretical_loss": 3.5416919882183193,
+ "tokens_seen": 1513881600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004374403209628887,
+ "loss": 1.3258,
+ "theoretical_loss": 3.5416786027679787,
+ "tokens_seen": 1513947136
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043742427281845533,
+ "loss": 1.2911,
+ "theoretical_loss": 3.5416652180592902,
+ "tokens_seen": 1514012672
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004374082246740221,
+ "loss": 1.4003,
+ "theoretical_loss": 3.54165183409218,
+ "tokens_seen": 1514078208
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043739217652958873,
+ "loss": 1.3224,
+ "theoretical_loss": 3.541638450866576,
+ "tokens_seen": 1514143744
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004373761283851555,
+ "loss": 1.2397,
+ "theoretical_loss": 3.5416250683824044,
+ "tokens_seen": 1514209280
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043736008024072213,
+ "loss": 1.3816,
+ "theoretical_loss": 3.5416116866395915,
+ "tokens_seen": 1514274816
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004373440320962889,
+ "loss": 1.3999,
+ "theoretical_loss": 3.541598305638065,
+ "tokens_seen": 1514340352
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043732798395185554,
+ "loss": 1.3185,
+ "theoretical_loss": 3.5415849253777516,
+ "tokens_seen": 1514405888
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004373119358074223,
+ "loss": 1.1787,
+ "theoretical_loss": 3.541571545858578,
+ "tokens_seen": 1514471424
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043729588766298894,
+ "loss": 1.2119,
+ "theoretical_loss": 3.5415581670804706,
+ "tokens_seen": 1514536960
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004372798395185557,
+ "loss": 1.3185,
+ "theoretical_loss": 3.5415447890433573,
+ "tokens_seen": 1514602496
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043726379137412234,
+ "loss": 1.3128,
+ "theoretical_loss": 3.5415314117471643,
+ "tokens_seen": 1514668032
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004372477432296891,
+ "loss": 1.3142,
+ "theoretical_loss": 3.5415180351918187,
+ "tokens_seen": 1514733568
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043723169508525585,
+ "loss": 1.3487,
+ "theoretical_loss": 3.5415046593772477,
+ "tokens_seen": 1514799104
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004372156469408225,
+ "loss": 1.2566,
+ "theoretical_loss": 3.5414912843033775,
+ "tokens_seen": 1514864640
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043719959879638925,
+ "loss": 1.2866,
+ "theoretical_loss": 3.5414779099701352,
+ "tokens_seen": 1514930176
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004371835506519559,
+ "loss": 1.3375,
+ "theoretical_loss": 3.5414645363774486,
+ "tokens_seen": 1514995712
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043716750250752265,
+ "loss": 1.3559,
+ "theoretical_loss": 3.5414511635252444,
+ "tokens_seen": 1515061248
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004371514543630893,
+ "loss": 1.3581,
+ "theoretical_loss": 3.5414377914134487,
+ "tokens_seen": 1515126784
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043713540621865605,
+ "loss": 1.2881,
+ "theoretical_loss": 3.541424420041989,
+ "tokens_seen": 1515192320
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004371193580742227,
+ "loss": 1.2667,
+ "theoretical_loss": 3.541411049410793,
+ "tokens_seen": 1515257856
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043710330992978945,
+ "loss": 1.2893,
+ "theoretical_loss": 3.541397679519786,
+ "tokens_seen": 1515323392
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004370872617853561,
+ "loss": 1.302,
+ "theoretical_loss": 3.5413843103688967,
+ "tokens_seen": 1515388928
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043707121364092285,
+ "loss": 1.42,
+ "theoretical_loss": 3.5413709419580517,
+ "tokens_seen": 1515454464
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004370551654964895,
+ "loss": 1.2374,
+ "theoretical_loss": 3.5413575742871775,
+ "tokens_seen": 1515520000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004370391173520562,
+ "loss": 1.2864,
+ "theoretical_loss": 3.5413442073562016,
+ "tokens_seen": 1515585536
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004370230692076229,
+ "loss": 1.1992,
+ "theoretical_loss": 3.541330841165051,
+ "tokens_seen": 1515651072
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004370070210631896,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5413174757136527,
+ "tokens_seen": 1515716608
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004369909729187563,
+ "loss": 1.3657,
+ "theoretical_loss": 3.5413041110019337,
+ "tokens_seen": 1515782144
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000436974924774323,
+ "loss": 1.3434,
+ "theoretical_loss": 3.541290747029821,
+ "tokens_seen": 1515847680
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004369588766298897,
+ "loss": 1.347,
+ "theoretical_loss": 3.541277383797242,
+ "tokens_seen": 1515913216
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004369428284854564,
+ "loss": 1.4375,
+ "theoretical_loss": 3.5412640213041238,
+ "tokens_seen": 1515978752
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043692678034102305,
+ "loss": 1.1744,
+ "theoretical_loss": 3.5412506595503928,
+ "tokens_seen": 1516044288
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004369107321965898,
+ "loss": 1.2759,
+ "theoretical_loss": 3.541237298535977,
+ "tokens_seen": 1516109824
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043689468405215645,
+ "loss": 1.3161,
+ "theoretical_loss": 3.5412239382608037,
+ "tokens_seen": 1516175360
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004368786359077232,
+ "loss": 1.2691,
+ "theoretical_loss": 3.541210578724799,
+ "tokens_seen": 1516240896
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043686258776328985,
+ "loss": 1.3528,
+ "theoretical_loss": 3.541197219927891,
+ "tokens_seen": 1516306432
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004368465396188566,
+ "loss": 1.2354,
+ "theoretical_loss": 3.541183861870006,
+ "tokens_seen": 1516371968
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043683049147442326,
+ "loss": 1.1958,
+ "theoretical_loss": 3.541170504551072,
+ "tokens_seen": 1516437504
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043681444332999,
+ "loss": 1.3622,
+ "theoretical_loss": 3.541157147971016,
+ "tokens_seen": 1516503040
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043679839518555666,
+ "loss": 1.2206,
+ "theoretical_loss": 3.5411437921297653,
+ "tokens_seen": 1516568576
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004367823470411234,
+ "loss": 1.2189,
+ "theoretical_loss": 3.5411304370272467,
+ "tokens_seen": 1516634112
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043676629889669006,
+ "loss": 1.294,
+ "theoretical_loss": 3.5411170826633875,
+ "tokens_seen": 1516699648
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004367502507522568,
+ "loss": 1.3575,
+ "theoretical_loss": 3.541103729038115,
+ "tokens_seen": 1516765184
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043673420260782346,
+ "loss": 1.2574,
+ "theoretical_loss": 3.541090376151357,
+ "tokens_seen": 1516830720
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004367181544633902,
+ "loss": 1.2853,
+ "theoretical_loss": 3.5410770240030396,
+ "tokens_seen": 1516896256
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043670210631895686,
+ "loss": 1.2021,
+ "theoretical_loss": 3.541063672593091,
+ "tokens_seen": 1516961792
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004366860581745236,
+ "loss": 1.2567,
+ "theoretical_loss": 3.541050321921438,
+ "tokens_seen": 1517027328
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043667001003009026,
+ "loss": 1.3325,
+ "theoretical_loss": 3.5410369719880084,
+ "tokens_seen": 1517092864
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 508543,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.311399221420288,
+ "objective/train/theoretical_loss": 3.5410236227927285,
+ "objective/train/tokens_used": 1537618400,
+ "theoretical_loss": 3.5410236227927285,
+ "tokens_seen": 1517158400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000436653961885657,
+ "loss": 1.2733,
+ "theoretical_loss": 3.5410236227927285,
+ "tokens_seen": 1517158400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043663791374122366,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5410102743355267,
+ "tokens_seen": 1517223936
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004366218655967904,
+ "loss": 1.2955,
+ "theoretical_loss": 3.54099692661633,
+ "tokens_seen": 1517289472
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043660581745235706,
+ "loss": 1.3617,
+ "theoretical_loss": 3.540983579635065,
+ "tokens_seen": 1517355008
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004365897693079238,
+ "loss": 1.3975,
+ "theoretical_loss": 3.5409702333916604,
+ "tokens_seen": 1517420544
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043657372116349047,
+ "loss": 1.1151,
+ "theoretical_loss": 3.5409568878860425,
+ "tokens_seen": 1517486080
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004365576730190572,
+ "loss": 1.2129,
+ "theoretical_loss": 3.5409435431181384,
+ "tokens_seen": 1517551616
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043654162487462387,
+ "loss": 1.2832,
+ "theoretical_loss": 3.5409301990878763,
+ "tokens_seen": 1517617152
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004365255767301906,
+ "loss": 1.2076,
+ "theoretical_loss": 3.5409168557951833,
+ "tokens_seen": 1517682688
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043650952858575727,
+ "loss": 1.3613,
+ "theoretical_loss": 3.540903513239987,
+ "tokens_seen": 1517748224
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000436493480441324,
+ "loss": 1.2954,
+ "theoretical_loss": 3.5408901714222143,
+ "tokens_seen": 1517813760
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043647743229689067,
+ "loss": 1.3134,
+ "theoretical_loss": 3.540876830341793,
+ "tokens_seen": 1517879296
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004364613841524574,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5408634899986504,
+ "tokens_seen": 1517944832
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043644533600802407,
+ "loss": 1.323,
+ "theoretical_loss": 3.5408501503927132,
+ "tokens_seen": 1518010368
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004364292878635908,
+ "loss": 1.2203,
+ "theoretical_loss": 3.5408368115239104,
+ "tokens_seen": 1518075904
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043641323971915747,
+ "loss": 1.2025,
+ "theoretical_loss": 3.540823473392168,
+ "tokens_seen": 1518141440
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004363971915747242,
+ "loss": 1.31,
+ "theoretical_loss": 3.5408101359974147,
+ "tokens_seen": 1518206976
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004363811434302909,
+ "loss": 1.3091,
+ "theoretical_loss": 3.5407967993395766,
+ "tokens_seen": 1518272512
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043636509528585763,
+ "loss": 1.1995,
+ "theoretical_loss": 3.540783463418582,
+ "tokens_seen": 1518338048
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004363490471414243,
+ "loss": 1.3549,
+ "theoretical_loss": 3.5407701282343584,
+ "tokens_seen": 1518403584
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043633299899699103,
+ "loss": 1.2267,
+ "theoretical_loss": 3.540756793786833,
+ "tokens_seen": 1518469120
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004363169508525577,
+ "loss": 1.2063,
+ "theoretical_loss": 3.5407434600759338,
+ "tokens_seen": 1518534656
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043630090270812443,
+ "loss": 1.3824,
+ "theoretical_loss": 3.540730127101588,
+ "tokens_seen": 1518600192
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362848545636911,
+ "loss": 1.2785,
+ "theoretical_loss": 3.5407167948637226,
+ "tokens_seen": 1518665728
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362688064192578,
+ "loss": 1.4351,
+ "theoretical_loss": 3.540703463362266,
+ "tokens_seen": 1518731264
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362527582748245,
+ "loss": 1.3,
+ "theoretical_loss": 3.5406901325971454,
+ "tokens_seen": 1518796800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362367101303912,
+ "loss": 1.202,
+ "theoretical_loss": 3.5406768025682887,
+ "tokens_seen": 1518862336
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362206619859579,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5406634732756226,
+ "tokens_seen": 1518927872
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004362046138415246,
+ "loss": 1.2868,
+ "theoretical_loss": 3.5406501447190752,
+ "tokens_seen": 1518993408
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004361885656970913,
+ "loss": 1.294,
+ "theoretical_loss": 3.5406368168985742,
+ "tokens_seen": 1519058944
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000436172517552658,
+ "loss": 1.2987,
+ "theoretical_loss": 3.540623489814047,
+ "tokens_seen": 1519124480
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043615646940822463,
+ "loss": 1.3564,
+ "theoretical_loss": 3.5406101634654217,
+ "tokens_seen": 1519190016
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004361404212637914,
+ "loss": 1.2843,
+ "theoretical_loss": 3.540596837852626,
+ "tokens_seen": 1519255552
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043612437311935803,
+ "loss": 1.2792,
+ "theoretical_loss": 3.5405835129755863,
+ "tokens_seen": 1519321088
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004361083249749248,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5405701888342307,
+ "tokens_seen": 1519386624
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043609227683049143,
+ "loss": 1.2529,
+ "theoretical_loss": 3.540556865428488,
+ "tokens_seen": 1519452160
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004360762286860582,
+ "loss": 1.261,
+ "theoretical_loss": 3.5405435427582845,
+ "tokens_seen": 1519517696
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043606018054162483,
+ "loss": 1.4229,
+ "theoretical_loss": 3.5405302208235483,
+ "tokens_seen": 1519583232
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004360441323971916,
+ "loss": 1.3048,
+ "theoretical_loss": 3.540516899624208,
+ "tokens_seen": 1519648768
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043602808425275834,
+ "loss": 1.2527,
+ "theoretical_loss": 3.5405035791601898,
+ "tokens_seen": 1519714304
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000436012036108325,
+ "loss": 1.3167,
+ "theoretical_loss": 3.5404902594314223,
+ "tokens_seen": 1519779840
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043599598796389174,
+ "loss": 1.2432,
+ "theoretical_loss": 3.5404769404378325,
+ "tokens_seen": 1519845376
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004359799398194584,
+ "loss": 1.2814,
+ "theoretical_loss": 3.540463622179349,
+ "tokens_seen": 1519910912
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043596389167502514,
+ "loss": 1.2208,
+ "theoretical_loss": 3.5404503046558995,
+ "tokens_seen": 1519976448
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004359478435305918,
+ "loss": 1.2732,
+ "theoretical_loss": 3.5404369878674107,
+ "tokens_seen": 1520041984
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043593179538615855,
+ "loss": 1.2721,
+ "theoretical_loss": 3.5404236718138113,
+ "tokens_seen": 1520107520
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004359157472417252,
+ "loss": 1.3007,
+ "theoretical_loss": 3.540410356495029,
+ "tokens_seen": 1520173056
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043589969909729195,
+ "loss": 1.3334,
+ "theoretical_loss": 3.5403970419109916,
+ "tokens_seen": 1520238592
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004358836509528586,
+ "loss": 1.2909,
+ "theoretical_loss": 3.5403837280616264,
+ "tokens_seen": 1520304128
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043586760280842535,
+ "loss": 1.266,
+ "theoretical_loss": 3.540370414946861,
+ "tokens_seen": 1520369664
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 509751,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2503533363342285,
+ "objective/train/theoretical_loss": 3.5403571025666247,
+ "objective/train/tokens_used": 1540895200,
+ "theoretical_loss": 3.5403571025666247,
+ "tokens_seen": 1520435200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000435851554663992,
+ "loss": 1.2463,
+ "theoretical_loss": 3.5403571025666247,
+ "tokens_seen": 1520435200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043583550651955875,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5403437909208435,
+ "tokens_seen": 1520500736
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004358194583751254,
+ "loss": 1.2966,
+ "theoretical_loss": 3.5403304800094464,
+ "tokens_seen": 1520566272
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043580341023069215,
+ "loss": 1.2952,
+ "theoretical_loss": 3.5403171698323606,
+ "tokens_seen": 1520631808
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004357873620862588,
+ "loss": 1.3056,
+ "theoretical_loss": 3.540303860389514,
+ "tokens_seen": 1520697344
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043577131394182555,
+ "loss": 1.3061,
+ "theoretical_loss": 3.540290551680835,
+ "tokens_seen": 1520762880
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004357552657973922,
+ "loss": 1.3502,
+ "theoretical_loss": 3.5402772437062513,
+ "tokens_seen": 1520828416
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043573921765295895,
+ "loss": 1.2762,
+ "theoretical_loss": 3.5402639364656903,
+ "tokens_seen": 1520893952
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004357231695085256,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5402506299590804,
+ "tokens_seen": 1520959488
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043570712136409235,
+ "loss": 1.2866,
+ "theoretical_loss": 3.540237324186349,
+ "tokens_seen": 1521025024
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000435691073219659,
+ "loss": 1.3873,
+ "theoretical_loss": 3.5402240191474244,
+ "tokens_seen": 1521090560
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043567502507522576,
+ "loss": 1.2747,
+ "theoretical_loss": 3.5402107148422344,
+ "tokens_seen": 1521156096
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004356589769307924,
+ "loss": 1.2825,
+ "theoretical_loss": 3.5401974112707073,
+ "tokens_seen": 1521221632
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004356429287863591,
+ "loss": 1.4176,
+ "theoretical_loss": 3.5401841084327703,
+ "tokens_seen": 1521287168
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004356268806419258,
+ "loss": 1.3239,
+ "theoretical_loss": 3.5401708063283515,
+ "tokens_seen": 1521352704
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004356108324974925,
+ "loss": 1.3202,
+ "theoretical_loss": 3.5401575049573792,
+ "tokens_seen": 1521418240
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004355947843530592,
+ "loss": 1.2148,
+ "theoretical_loss": 3.5401442043197813,
+ "tokens_seen": 1521483776
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004355787362086259,
+ "loss": 1.326,
+ "theoretical_loss": 3.540130904415486,
+ "tokens_seen": 1521549312
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004355626880641926,
+ "loss": 1.3531,
+ "theoretical_loss": 3.5401176052444208,
+ "tokens_seen": 1521614848
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004355466399197593,
+ "loss": 1.3342,
+ "theoretical_loss": 3.540104306806514,
+ "tokens_seen": 1521680384
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000435530591775326,
+ "loss": 1.3025,
+ "theoretical_loss": 3.5400910091016935,
+ "tokens_seen": 1521745920
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004355145436308927,
+ "loss": 1.3313,
+ "theoretical_loss": 3.540077712129887,
+ "tokens_seen": 1521811456
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043549849548645935,
+ "loss": 1.2824,
+ "theoretical_loss": 3.540064415891023,
+ "tokens_seen": 1521876992
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004354824473420261,
+ "loss": 1.2822,
+ "theoretical_loss": 3.54005112038503,
+ "tokens_seen": 1521942528
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043546639919759276,
+ "loss": 1.3042,
+ "theoretical_loss": 3.5400378256118348,
+ "tokens_seen": 1522008064
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004354503510531595,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5400245315713668,
+ "tokens_seen": 1522073600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043543430290872616,
+ "loss": 1.2594,
+ "theoretical_loss": 3.5400112382635527,
+ "tokens_seen": 1522139136
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004354182547642929,
+ "loss": 1.2435,
+ "theoretical_loss": 3.5399979456883215,
+ "tokens_seen": 1522204672
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043540220661985956,
+ "loss": 1.2718,
+ "theoretical_loss": 3.5399846538456012,
+ "tokens_seen": 1522270208
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004353861584754263,
+ "loss": 1.3395,
+ "theoretical_loss": 3.5399713627353195,
+ "tokens_seen": 1522335744
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043537011033099296,
+ "loss": 1.274,
+ "theoretical_loss": 3.539958072357405,
+ "tokens_seen": 1522401280
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004353540621865597,
+ "loss": 1.3374,
+ "theoretical_loss": 3.5399447827117854,
+ "tokens_seen": 1522466816
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043533801404212636,
+ "loss": 1.4313,
+ "theoretical_loss": 3.5399314937983895,
+ "tokens_seen": 1522532352
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004353219658976931,
+ "loss": 1.3218,
+ "theoretical_loss": 3.5399182056171448,
+ "tokens_seen": 1522597888
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043530591775325976,
+ "loss": 1.3027,
+ "theoretical_loss": 3.539904918167979,
+ "tokens_seen": 1522663424
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004352898696088265,
+ "loss": 1.2387,
+ "theoretical_loss": 3.5398916314508213,
+ "tokens_seen": 1522728960
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043527382146439316,
+ "loss": 1.24,
+ "theoretical_loss": 3.5398783454656,
+ "tokens_seen": 1522794496
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004352577733199599,
+ "loss": 1.3221,
+ "theoretical_loss": 3.539865060212242,
+ "tokens_seen": 1522860032
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043524172517552656,
+ "loss": 1.3932,
+ "theoretical_loss": 3.5398517756906767,
+ "tokens_seen": 1522925568
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004352256770310933,
+ "loss": 1.223,
+ "theoretical_loss": 3.539838491900831,
+ "tokens_seen": 1522991104
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043520962888665997,
+ "loss": 1.2602,
+ "theoretical_loss": 3.5398252088426347,
+ "tokens_seen": 1523056640
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004351935807422267,
+ "loss": 1.3085,
+ "theoretical_loss": 3.5398119265160153,
+ "tokens_seen": 1523122176
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043517753259779337,
+ "loss": 1.3413,
+ "theoretical_loss": 3.539798644920901,
+ "tokens_seen": 1523187712
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004351614844533601,
+ "loss": 1.3216,
+ "theoretical_loss": 3.539785364057219,
+ "tokens_seen": 1523253248
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043514543630892677,
+ "loss": 1.4126,
+ "theoretical_loss": 3.5397720839249,
+ "tokens_seen": 1523318784
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004351293881644935,
+ "loss": 1.3832,
+ "theoretical_loss": 3.5397588045238697,
+ "tokens_seen": 1523384320
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043511334002006017,
+ "loss": 1.3739,
+ "theoretical_loss": 3.5397455258540584,
+ "tokens_seen": 1523449856
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004350972918756269,
+ "loss": 1.3289,
+ "theoretical_loss": 3.539732247915393,
+ "tokens_seen": 1523515392
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043508124373119357,
+ "loss": 1.3113,
+ "theoretical_loss": 3.5397189707078027,
+ "tokens_seen": 1523580928
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004350651955867603,
+ "loss": 1.2262,
+ "theoretical_loss": 3.5397056942312153,
+ "tokens_seen": 1523646464
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 510713,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2261080741882324,
+ "objective/train/theoretical_loss": 3.5396924184855587,
+ "objective/train/tokens_used": 1544172000,
+ "theoretical_loss": 3.5396924184855587,
+ "tokens_seen": 1523712000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043504914744232697,
+ "loss": 1.2512,
+ "theoretical_loss": 3.5396924184855587,
+ "tokens_seen": 1523712000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004350330992978937,
+ "loss": 1.2323,
+ "theoretical_loss": 3.5396791434707624,
+ "tokens_seen": 1523777536
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004350170511534604,
+ "loss": 1.4104,
+ "theoretical_loss": 3.539665869186754,
+ "tokens_seen": 1523843072
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043500100300902713,
+ "loss": 1.3584,
+ "theoretical_loss": 3.5396525956334615,
+ "tokens_seen": 1523908608
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004349849548645938,
+ "loss": 1.3525,
+ "theoretical_loss": 3.539639322810814,
+ "tokens_seen": 1523974144
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043496890672016053,
+ "loss": 1.3044,
+ "theoretical_loss": 3.5396260507187396,
+ "tokens_seen": 1524039680
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004349528585757272,
+ "loss": 1.3124,
+ "theoretical_loss": 3.5396127793571663,
+ "tokens_seen": 1524105216
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043493681043129393,
+ "loss": 1.2217,
+ "theoretical_loss": 3.539599508726023,
+ "tokens_seen": 1524170752
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004349207622868606,
+ "loss": 1.2942,
+ "theoretical_loss": 3.5395862388252377,
+ "tokens_seen": 1524236288
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043490471414242733,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5395729696547393,
+ "tokens_seen": 1524301824
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000434888665997994,
+ "loss": 1.3595,
+ "theoretical_loss": 3.539559701214456,
+ "tokens_seen": 1524367360
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004348726178535607,
+ "loss": 1.2068,
+ "theoretical_loss": 3.5395464335043156,
+ "tokens_seen": 1524432896
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004348565697091274,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5395331665242473,
+ "tokens_seen": 1524498432
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004348405215646941,
+ "loss": 1.3361,
+ "theoretical_loss": 3.53951990027418,
+ "tokens_seen": 1524563968
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043482447342026083,
+ "loss": 1.3117,
+ "theoretical_loss": 3.5395066347540407,
+ "tokens_seen": 1524629504
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004348084252758275,
+ "loss": 1.3429,
+ "theoretical_loss": 3.5394933699637585,
+ "tokens_seen": 1524695040
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043479237713139424,
+ "loss": 1.4326,
+ "theoretical_loss": 3.5394801059032623,
+ "tokens_seen": 1524760576
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004347763289869609,
+ "loss": 1.2817,
+ "theoretical_loss": 3.5394668425724802,
+ "tokens_seen": 1524826112
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043476028084252764,
+ "loss": 1.3371,
+ "theoretical_loss": 3.539453579971341,
+ "tokens_seen": 1524891648
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004347442326980943,
+ "loss": 1.3175,
+ "theoretical_loss": 3.539440318099773,
+ "tokens_seen": 1524957184
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043472818455366104,
+ "loss": 1.2191,
+ "theoretical_loss": 3.539427056957704,
+ "tokens_seen": 1525022720
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004347121364092277,
+ "loss": 1.275,
+ "theoretical_loss": 3.5394137965450643,
+ "tokens_seen": 1525088256
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043469608826479444,
+ "loss": 1.3256,
+ "theoretical_loss": 3.539400536861781,
+ "tokens_seen": 1525153792
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004346800401203611,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5393872779077826,
+ "tokens_seen": 1525219328
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043466399197592784,
+ "loss": 1.2451,
+ "theoretical_loss": 3.539374019682999,
+ "tokens_seen": 1525284864
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004346479438314945,
+ "loss": 1.3357,
+ "theoretical_loss": 3.5393607621873566,
+ "tokens_seen": 1525350400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043463189568706124,
+ "loss": 1.3227,
+ "theoretical_loss": 3.539347505420786,
+ "tokens_seen": 1525415936
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004346158475426279,
+ "loss": 1.2107,
+ "theoretical_loss": 3.5393342493832147,
+ "tokens_seen": 1525481472
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043459979939819464,
+ "loss": 1.3845,
+ "theoretical_loss": 3.539320994074572,
+ "tokens_seen": 1525547008
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004345837512537613,
+ "loss": 1.3037,
+ "theoretical_loss": 3.5393077394947854,
+ "tokens_seen": 1525612544
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043456770310932804,
+ "loss": 1.3719,
+ "theoretical_loss": 3.5392944856437847,
+ "tokens_seen": 1525678080
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004345516549648947,
+ "loss": 1.2568,
+ "theoretical_loss": 3.539281232521498,
+ "tokens_seen": 1525743616
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043453560682046145,
+ "loss": 1.3319,
+ "theoretical_loss": 3.539267980127854,
+ "tokens_seen": 1525809152
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004345195586760281,
+ "loss": 1.268,
+ "theoretical_loss": 3.5392547284627813,
+ "tokens_seen": 1525874688
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043450351053159485,
+ "loss": 1.3637,
+ "theoretical_loss": 3.5392414775262084,
+ "tokens_seen": 1525940224
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004344874623871615,
+ "loss": 1.3438,
+ "theoretical_loss": 3.5392282273180644,
+ "tokens_seen": 1526005760
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043447141424272825,
+ "loss": 1.3017,
+ "theoretical_loss": 3.5392149778382773,
+ "tokens_seen": 1526071296
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004344553660982949,
+ "loss": 1.3636,
+ "theoretical_loss": 3.539201729086777,
+ "tokens_seen": 1526136832
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043443931795386165,
+ "loss": 1.2316,
+ "theoretical_loss": 3.5391884810634906,
+ "tokens_seen": 1526202368
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004344232698094283,
+ "loss": 1.2293,
+ "theoretical_loss": 3.539175233768348,
+ "tokens_seen": 1526267904
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043440722166499505,
+ "loss": 1.228,
+ "theoretical_loss": 3.5391619872012776,
+ "tokens_seen": 1526333440
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004343911735205617,
+ "loss": 1.2361,
+ "theoretical_loss": 3.539148741362208,
+ "tokens_seen": 1526398976
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043437512537612845,
+ "loss": 1.3815,
+ "theoretical_loss": 3.5391354962510677,
+ "tokens_seen": 1526464512
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004343590772316951,
+ "loss": 1.2754,
+ "theoretical_loss": 3.539122251867786,
+ "tokens_seen": 1526530048
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043434302908726185,
+ "loss": 1.3377,
+ "theoretical_loss": 3.5391090082122916,
+ "tokens_seen": 1526595584
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004343269809428285,
+ "loss": 1.3253,
+ "theoretical_loss": 3.5390957652845128,
+ "tokens_seen": 1526661120
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043431093279839526,
+ "loss": 1.3318,
+ "theoretical_loss": 3.539082523084379,
+ "tokens_seen": 1526726656
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004342948846539619,
+ "loss": 1.297,
+ "theoretical_loss": 3.539069281611818,
+ "tokens_seen": 1526792192
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043427883650952866,
+ "loss": 1.3165,
+ "theoretical_loss": 3.5390560408667593,
+ "tokens_seen": 1526857728
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004342627883650953,
+ "loss": 1.2439,
+ "theoretical_loss": 3.539042800849132,
+ "tokens_seen": 1526923264
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 511958,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.314308524131775,
+ "objective/train/theoretical_loss": 3.5390295615588645,
+ "objective/train/tokens_used": 1547448800,
+ "theoretical_loss": 3.5390295615588645,
+ "tokens_seen": 1526988800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000434246740220662,
+ "loss": 1.2534,
+ "theoretical_loss": 3.5390295615588645,
+ "tokens_seen": 1526988800
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004342306920762287,
+ "loss": 1.2488,
+ "theoretical_loss": 3.5390163229958853,
+ "tokens_seen": 1527054336
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004342146439317954,
+ "loss": 1.3445,
+ "theoretical_loss": 3.5390030851601244,
+ "tokens_seen": 1527119872
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341985957873621,
+ "loss": 1.3672,
+ "theoretical_loss": 3.538989848051509,
+ "tokens_seen": 1527185408
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341825476429288,
+ "loss": 1.4039,
+ "theoretical_loss": 3.5389766116699692,
+ "tokens_seen": 1527250944
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341664994984955,
+ "loss": 1.3695,
+ "theoretical_loss": 3.5389633760154338,
+ "tokens_seen": 1527316480
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341504513540622,
+ "loss": 1.3595,
+ "theoretical_loss": 3.5389501410878306,
+ "tokens_seen": 1527382016
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341344032096289,
+ "loss": 1.3764,
+ "theoretical_loss": 3.5389369068870895,
+ "tokens_seen": 1527447552
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004341183550651956,
+ "loss": 1.3329,
+ "theoretical_loss": 3.538923673413139,
+ "tokens_seen": 1527513088
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043410230692076225,
+ "loss": 1.2879,
+ "theoretical_loss": 3.5389104406659087,
+ "tokens_seen": 1527578624
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000434086258776329,
+ "loss": 1.3723,
+ "theoretical_loss": 3.538897208645327,
+ "tokens_seen": 1527644160
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043407021063189566,
+ "loss": 1.258,
+ "theoretical_loss": 3.5388839773513223,
+ "tokens_seen": 1527709696
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004340541624874624,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5388707467838243,
+ "tokens_seen": 1527775232
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043403811434302906,
+ "loss": 1.3699,
+ "theoretical_loss": 3.5388575169427616,
+ "tokens_seen": 1527840768
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004340220661985958,
+ "loss": 1.3514,
+ "theoretical_loss": 3.538844287828063,
+ "tokens_seen": 1527906304
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043400601805416246,
+ "loss": 1.3933,
+ "theoretical_loss": 3.5388310594396577,
+ "tokens_seen": 1527971840
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004339899699097292,
+ "loss": 1.2657,
+ "theoretical_loss": 3.538817831777475,
+ "tokens_seen": 1528037376
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043397392176529586,
+ "loss": 1.3165,
+ "theoretical_loss": 3.5388046048414434,
+ "tokens_seen": 1528102912
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004339578736208626,
+ "loss": 1.3014,
+ "theoretical_loss": 3.538791378631492,
+ "tokens_seen": 1528168448
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043394182547642926,
+ "loss": 1.2957,
+ "theoretical_loss": 3.5387781531475504,
+ "tokens_seen": 1528233984
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000433925777331996,
+ "loss": 1.2912,
+ "theoretical_loss": 3.5387649283895466,
+ "tokens_seen": 1528299520
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043390972918756266,
+ "loss": 1.2831,
+ "theoretical_loss": 3.53875170435741,
+ "tokens_seen": 1528365056
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004338936810431294,
+ "loss": 1.2042,
+ "theoretical_loss": 3.53873848105107,
+ "tokens_seen": 1528430592
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043387763289869606,
+ "loss": 1.3423,
+ "theoretical_loss": 3.538725258470455,
+ "tokens_seen": 1528496128
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004338615847542628,
+ "loss": 1.3012,
+ "theoretical_loss": 3.538712036615495,
+ "tokens_seen": 1528561664
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043384553660982947,
+ "loss": 1.2643,
+ "theoretical_loss": 3.5386988154861188,
+ "tokens_seen": 1528627200
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004338294884653962,
+ "loss": 1.3501,
+ "theoretical_loss": 3.538685595082254,
+ "tokens_seen": 1528692736
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043381344032096287,
+ "loss": 1.2993,
+ "theoretical_loss": 3.5386723754038316,
+ "tokens_seen": 1528758272
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004337973921765296,
+ "loss": 1.3957,
+ "theoretical_loss": 3.5386591564507803,
+ "tokens_seen": 1528823808
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043378134403209627,
+ "loss": 1.3315,
+ "theoretical_loss": 3.5386459382230284,
+ "tokens_seen": 1528889344
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000433765295887663,
+ "loss": 1.3887,
+ "theoretical_loss": 3.5386327207205057,
+ "tokens_seen": 1528954880
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043374924774322967,
+ "loss": 1.2869,
+ "theoretical_loss": 3.538619503943141,
+ "tokens_seen": 1529020416
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004337331995987964,
+ "loss": 1.3352,
+ "theoretical_loss": 3.5386062878908637,
+ "tokens_seen": 1529085952
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043371715145436307,
+ "loss": 1.3301,
+ "theoretical_loss": 3.538593072563603,
+ "tokens_seen": 1529151488
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004337011033099298,
+ "loss": 1.2331,
+ "theoretical_loss": 3.5385798579612873,
+ "tokens_seen": 1529217024
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043368505516549647,
+ "loss": 1.3779,
+ "theoretical_loss": 3.5385666440838466,
+ "tokens_seen": 1529282560
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004336690070210632,
+ "loss": 1.3058,
+ "theoretical_loss": 3.53855343093121,
+ "tokens_seen": 1529348096
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043365295887662987,
+ "loss": 1.2652,
+ "theoretical_loss": 3.5385402185033064,
+ "tokens_seen": 1529413632
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043363691073219663,
+ "loss": 1.3195,
+ "theoretical_loss": 3.5385270068000647,
+ "tokens_seen": 1529479168
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043362086258776333,
+ "loss": 1.2086,
+ "theoretical_loss": 3.538513795821415,
+ "tokens_seen": 1529544704
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043360481444333003,
+ "loss": 1.1886,
+ "theoretical_loss": 3.538500585567286,
+ "tokens_seen": 1529610240
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043358876629889673,
+ "loss": 1.2729,
+ "theoretical_loss": 3.5384873760376068,
+ "tokens_seen": 1529675776
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043357271815446343,
+ "loss": 1.2027,
+ "theoretical_loss": 3.538474167232307,
+ "tokens_seen": 1529741312
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043355667001003013,
+ "loss": 1.2775,
+ "theoretical_loss": 3.5384609591513154,
+ "tokens_seen": 1529806848
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043354062186559683,
+ "loss": 1.3396,
+ "theoretical_loss": 3.5384477517945614,
+ "tokens_seen": 1529872384
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043352457372116353,
+ "loss": 1.2684,
+ "theoretical_loss": 3.538434545161974,
+ "tokens_seen": 1529937920
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043350852557673023,
+ "loss": 1.2376,
+ "theoretical_loss": 3.5384213392534836,
+ "tokens_seen": 1530003456
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043349247743229693,
+ "loss": 1.3755,
+ "theoretical_loss": 3.538408134069018,
+ "tokens_seen": 1530068992
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004334764292878636,
+ "loss": 1.3668,
+ "theoretical_loss": 3.5383949296085078,
+ "tokens_seen": 1530134528
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043346038114343033,
+ "loss": 1.2863,
+ "theoretical_loss": 3.538381725871881,
+ "tokens_seen": 1530200064
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 512859,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3156123161315918,
+ "objective/train/theoretical_loss": 3.5383685228590687,
+ "objective/train/tokens_used": 1550725600,
+ "theoretical_loss": 3.5383685228590687,
+ "tokens_seen": 1530265600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000433444332998997,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5383685228590687,
+ "tokens_seen": 1530265600
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043342828485456374,
+ "loss": 1.4053,
+ "theoretical_loss": 3.5383553205699982,
+ "tokens_seen": 1530331136
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004334122367101304,
+ "loss": 1.255,
+ "theoretical_loss": 3.5383421190046,
+ "tokens_seen": 1530396672
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043339618856569714,
+ "loss": 1.354,
+ "theoretical_loss": 3.5383289181628035,
+ "tokens_seen": 1530462208
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004333801404212638,
+ "loss": 1.2306,
+ "theoretical_loss": 3.5383157180445375,
+ "tokens_seen": 1530527744
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043336409227683054,
+ "loss": 1.2193,
+ "theoretical_loss": 3.538302518649732,
+ "tokens_seen": 1530593280
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004333480441323972,
+ "loss": 1.2127,
+ "theoretical_loss": 3.5382893199783156,
+ "tokens_seen": 1530658816
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043333199598796394,
+ "loss": 1.3155,
+ "theoretical_loss": 3.5382761220302186,
+ "tokens_seen": 1530724352
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004333159478435306,
+ "loss": 1.3606,
+ "theoretical_loss": 3.5382629248053696,
+ "tokens_seen": 1530789888
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043329989969909734,
+ "loss": 1.3639,
+ "theoretical_loss": 3.538249728303698,
+ "tokens_seen": 1530855424
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000433283851554664,
+ "loss": 1.3253,
+ "theoretical_loss": 3.538236532525134,
+ "tokens_seen": 1530920960
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043326780341023074,
+ "loss": 1.3735,
+ "theoretical_loss": 3.538223337469607,
+ "tokens_seen": 1530986496
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004332517552657974,
+ "loss": 1.3221,
+ "theoretical_loss": 3.538210143137045,
+ "tokens_seen": 1531052032
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043323570712136414,
+ "loss": 1.3907,
+ "theoretical_loss": 3.5381969495273786,
+ "tokens_seen": 1531117568
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004332196589769308,
+ "loss": 1.2666,
+ "theoretical_loss": 3.538183756640538,
+ "tokens_seen": 1531183104
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043320361083249754,
+ "loss": 1.3686,
+ "theoretical_loss": 3.5381705644764505,
+ "tokens_seen": 1531248640
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004331875626880642,
+ "loss": 1.3164,
+ "theoretical_loss": 3.5381573730350473,
+ "tokens_seen": 1531314176
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043317151454363095,
+ "loss": 1.3652,
+ "theoretical_loss": 3.538144182316258,
+ "tokens_seen": 1531379712
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004331554663991976,
+ "loss": 1.2924,
+ "theoretical_loss": 3.538130992320011,
+ "tokens_seen": 1531445248
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043313941825476435,
+ "loss": 1.2772,
+ "theoretical_loss": 3.5381178030462364,
+ "tokens_seen": 1531510784
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000433123370110331,
+ "loss": 1.2965,
+ "theoretical_loss": 3.538104614494863,
+ "tokens_seen": 1531576320
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043310732196589775,
+ "loss": 1.2767,
+ "theoretical_loss": 3.5380914266658214,
+ "tokens_seen": 1531641856
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004330912738214644,
+ "loss": 1.3514,
+ "theoretical_loss": 3.538078239559041,
+ "tokens_seen": 1531707392
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043307522567703115,
+ "loss": 1.254,
+ "theoretical_loss": 3.5380650531744506,
+ "tokens_seen": 1531772928
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004330591775325978,
+ "loss": 1.3651,
+ "theoretical_loss": 3.5380518675119808,
+ "tokens_seen": 1531838464
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043304312938816455,
+ "loss": 1.2714,
+ "theoretical_loss": 3.5380386825715595,
+ "tokens_seen": 1531904000
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004330270812437312,
+ "loss": 1.2929,
+ "theoretical_loss": 3.538025498353118,
+ "tokens_seen": 1531969536
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043301103309929795,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5380123148565854,
+ "tokens_seen": 1532035072
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004329949849548646,
+ "loss": 1.2462,
+ "theoretical_loss": 3.5379991320818904,
+ "tokens_seen": 1532100608
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043297893681043135,
+ "loss": 1.2493,
+ "theoretical_loss": 3.537985950028964,
+ "tokens_seen": 1532166144
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000432962888665998,
+ "loss": 1.3372,
+ "theoretical_loss": 3.5379727686977347,
+ "tokens_seen": 1532231680
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043294684052156475,
+ "loss": 1.3136,
+ "theoretical_loss": 3.5379595880881327,
+ "tokens_seen": 1532297216
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004329307923771314,
+ "loss": 1.2955,
+ "theoretical_loss": 3.537946408200087,
+ "tokens_seen": 1532362752
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043291474423269816,
+ "loss": 1.4018,
+ "theoretical_loss": 3.537933229033528,
+ "tokens_seen": 1532428288
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004328986960882648,
+ "loss": 1.3813,
+ "theoretical_loss": 3.537920050588385,
+ "tokens_seen": 1532493824
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043288264794383156,
+ "loss": 1.2688,
+ "theoretical_loss": 3.537906872864588,
+ "tokens_seen": 1532559360
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004328665997993982,
+ "loss": 1.2696,
+ "theoretical_loss": 3.537893695862066,
+ "tokens_seen": 1532624896
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004328505516549649,
+ "loss": 1.3571,
+ "theoretical_loss": 3.5378805195807494,
+ "tokens_seen": 1532690432
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004328345035105316,
+ "loss": 1.2047,
+ "theoretical_loss": 3.537867344020567,
+ "tokens_seen": 1532755968
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004328184553660983,
+ "loss": 1.3288,
+ "theoretical_loss": 3.5378541691814496,
+ "tokens_seen": 1532821504
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.000432802407221665,
+ "loss": 1.2466,
+ "theoretical_loss": 3.537840995063326,
+ "tokens_seen": 1532887040
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004327863590772317,
+ "loss": 1.3258,
+ "theoretical_loss": 3.5378278216661267,
+ "tokens_seen": 1532952576
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004327703109327984,
+ "loss": 1.3445,
+ "theoretical_loss": 3.537814648989781,
+ "tokens_seen": 1533018112
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004327542627883651,
+ "loss": 1.3186,
+ "theoretical_loss": 3.5378014770342183,
+ "tokens_seen": 1533083648
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004327382146439318,
+ "loss": 1.3301,
+ "theoretical_loss": 3.537788305799369,
+ "tokens_seen": 1533149184
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004327221664994985,
+ "loss": 1.3567,
+ "theoretical_loss": 3.5377751352851625,
+ "tokens_seen": 1533214720
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043270611835506516,
+ "loss": 1.245,
+ "theoretical_loss": 3.537761965491529,
+ "tokens_seen": 1533280256
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004326900702106319,
+ "loss": 1.3002,
+ "theoretical_loss": 3.5377487964183976,
+ "tokens_seen": 1533345792
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043267402206619856,
+ "loss": 1.3425,
+ "theoretical_loss": 3.5377356280656977,
+ "tokens_seen": 1533411328
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004326579739217653,
+ "loss": 1.3162,
+ "theoretical_loss": 3.537722460433361,
+ "tokens_seen": 1533476864
+ },
+ {
+ "epoch": 0.46,
+ "objective/train/docs_used": 514171,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.121462345123291,
+ "objective/train/theoretical_loss": 3.5377092935213152,
+ "objective/train/tokens_used": 1554002400,
+ "theoretical_loss": 3.5377092935213152,
+ "tokens_seen": 1533542400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043264192577733196,
+ "loss": 1.1911,
+ "theoretical_loss": 3.5377092935213152,
+ "tokens_seen": 1533542400
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004326258776328987,
+ "loss": 1.3494,
+ "theoretical_loss": 3.537696127329492,
+ "tokens_seen": 1533607936
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043260982948846536,
+ "loss": 1.2347,
+ "theoretical_loss": 3.5376829618578194,
+ "tokens_seen": 1533673472
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004325937813440321,
+ "loss": 1.2744,
+ "theoretical_loss": 3.5376697971062283,
+ "tokens_seen": 1533739008
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043257773319959876,
+ "loss": 1.3188,
+ "theoretical_loss": 3.537656633074649,
+ "tokens_seen": 1533804544
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004325616850551655,
+ "loss": 1.2974,
+ "theoretical_loss": 3.53764346976301,
+ "tokens_seen": 1533870080
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043254563691073216,
+ "loss": 1.3428,
+ "theoretical_loss": 3.5376303071712423,
+ "tokens_seen": 1533935616
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004325295887662989,
+ "loss": 1.3152,
+ "theoretical_loss": 3.537617145299275,
+ "tokens_seen": 1534001152
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043251354062186556,
+ "loss": 1.3238,
+ "theoretical_loss": 3.5376039841470384,
+ "tokens_seen": 1534066688
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004324974924774323,
+ "loss": 1.3654,
+ "theoretical_loss": 3.5375908237144627,
+ "tokens_seen": 1534132224
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043248144433299896,
+ "loss": 1.3386,
+ "theoretical_loss": 3.5375776640014767,
+ "tokens_seen": 1534197760
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004324653961885657,
+ "loss": 1.346,
+ "theoretical_loss": 3.537564505008012,
+ "tokens_seen": 1534263296
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.00043244934804413237,
+ "loss": 1.2496,
+ "theoretical_loss": 3.537551346733997,
+ "tokens_seen": 1534328832
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004324332998996991,
+ "loss": 1.2911,
+ "theoretical_loss": 3.5375381891793625,
+ "tokens_seen": 1534394368
+ },
+ {
+ "epoch": 0.46,
+ "learning_rate": 0.0004324172517552659,
+ "loss": 1.2738,
+ "theoretical_loss": 3.537525032344038,
+ "tokens_seen": 1534459904
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004324012036108325,
+ "loss": 1.1993,
+ "theoretical_loss": 3.5375118762279545,
+ "tokens_seen": 1534525440
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323851554663993,
+ "loss": 1.3667,
+ "theoretical_loss": 3.5374987208310404,
+ "tokens_seen": 1534590976
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323691073219659,
+ "loss": 1.3241,
+ "theoretical_loss": 3.5374855661532263,
+ "tokens_seen": 1534656512
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323530591775327,
+ "loss": 1.3171,
+ "theoretical_loss": 3.5374724121944423,
+ "tokens_seen": 1534722048
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323370110330993,
+ "loss": 1.3691,
+ "theoretical_loss": 3.5374592589546188,
+ "tokens_seen": 1534787584
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323209628886661,
+ "loss": 1.3629,
+ "theoretical_loss": 3.537446106433685,
+ "tokens_seen": 1534853120
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004323049147442327,
+ "loss": 1.3614,
+ "theoretical_loss": 3.537432954631571,
+ "tokens_seen": 1534918656
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004322888665997995,
+ "loss": 1.2834,
+ "theoretical_loss": 3.537419803548208,
+ "tokens_seen": 1534984192
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043227281845536613,
+ "loss": 1.3042,
+ "theoretical_loss": 3.537406653183525,
+ "tokens_seen": 1535049728
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004322567703109329,
+ "loss": 1.3077,
+ "theoretical_loss": 3.5373935035374515,
+ "tokens_seen": 1535115264
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043224072216649953,
+ "loss": 1.2718,
+ "theoretical_loss": 3.5373803546099185,
+ "tokens_seen": 1535180800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043222467402206623,
+ "loss": 1.3983,
+ "theoretical_loss": 3.537367206400856,
+ "tokens_seen": 1535246336
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043220862587763293,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5373540589101937,
+ "tokens_seen": 1535311872
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043219257773319963,
+ "loss": 1.325,
+ "theoretical_loss": 3.5373409121378625,
+ "tokens_seen": 1535377408
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043217652958876633,
+ "loss": 1.3545,
+ "theoretical_loss": 3.5373277660837914,
+ "tokens_seen": 1535442944
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043216048144433303,
+ "loss": 1.2573,
+ "theoretical_loss": 3.537314620747911,
+ "tokens_seen": 1535508480
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043214443329989973,
+ "loss": 1.3863,
+ "theoretical_loss": 3.5373014761301516,
+ "tokens_seen": 1535574016
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043212838515546643,
+ "loss": 1.2602,
+ "theoretical_loss": 3.5372883322304425,
+ "tokens_seen": 1535639552
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043211233701103313,
+ "loss": 1.3346,
+ "theoretical_loss": 3.537275189048715,
+ "tokens_seen": 1535705088
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043209628886659983,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5372620465848987,
+ "tokens_seen": 1535770624
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004320802407221665,
+ "loss": 1.36,
+ "theoretical_loss": 3.5372489048389237,
+ "tokens_seen": 1535836160
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043206419257773324,
+ "loss": 1.3981,
+ "theoretical_loss": 3.5372357638107204,
+ "tokens_seen": 1535901696
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004320481444332999,
+ "loss": 1.2923,
+ "theoretical_loss": 3.5372226235002184,
+ "tokens_seen": 1535967232
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043203209628886664,
+ "loss": 1.3062,
+ "theoretical_loss": 3.5372094839073487,
+ "tokens_seen": 1536032768
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004320160481444333,
+ "loss": 1.3202,
+ "theoretical_loss": 3.537196345032041,
+ "tokens_seen": 1536098304
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043200000000000004,
+ "loss": 1.4005,
+ "theoretical_loss": 3.537183206874225,
+ "tokens_seen": 1536163840
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004319839518555667,
+ "loss": 1.2083,
+ "theoretical_loss": 3.5371700694338317,
+ "tokens_seen": 1536229376
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043196790371113344,
+ "loss": 1.3219,
+ "theoretical_loss": 3.5371569327107917,
+ "tokens_seen": 1536294912
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004319518555667001,
+ "loss": 1.288,
+ "theoretical_loss": 3.537143796705034,
+ "tokens_seen": 1536360448
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043193580742226684,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5371306614164895,
+ "tokens_seen": 1536425984
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004319197592778335,
+ "loss": 1.2217,
+ "theoretical_loss": 3.5371175268450887,
+ "tokens_seen": 1536491520
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043190371113340024,
+ "loss": 1.2825,
+ "theoretical_loss": 3.537104392990761,
+ "tokens_seen": 1536557056
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004318876629889669,
+ "loss": 1.2399,
+ "theoretical_loss": 3.537091259853438,
+ "tokens_seen": 1536622592
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043187161484453364,
+ "loss": 1.3843,
+ "theoretical_loss": 3.537078127433049,
+ "tokens_seen": 1536688128
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004318555667001003,
+ "loss": 1.3808,
+ "theoretical_loss": 3.5370649957295246,
+ "tokens_seen": 1536753664
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 515162,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2471009492874146,
+ "objective/train/theoretical_loss": 3.5370518647427946,
+ "objective/train/tokens_used": 1557279200,
+ "theoretical_loss": 3.5370518647427946,
+ "tokens_seen": 1536819200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043183951855566704,
+ "loss": 1.2905,
+ "theoretical_loss": 3.5370518647427946,
+ "tokens_seen": 1536819200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004318234704112337,
+ "loss": 1.2587,
+ "theoretical_loss": 3.53703873447279,
+ "tokens_seen": 1536884736
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043180742226680045,
+ "loss": 1.231,
+ "theoretical_loss": 3.5370256049194406,
+ "tokens_seen": 1536950272
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004317913741223671,
+ "loss": 1.2213,
+ "theoretical_loss": 3.537012476082678,
+ "tokens_seen": 1537015808
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043177532597793385,
+ "loss": 1.3391,
+ "theoretical_loss": 3.53699934796243,
+ "tokens_seen": 1537081344
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004317592778335005,
+ "loss": 1.4384,
+ "theoretical_loss": 3.5369862205586298,
+ "tokens_seen": 1537146880
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043174322968906725,
+ "loss": 1.2812,
+ "theoretical_loss": 3.5369730938712056,
+ "tokens_seen": 1537212416
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004317271815446339,
+ "loss": 1.2214,
+ "theoretical_loss": 3.536959967900089,
+ "tokens_seen": 1537277952
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043171113340020065,
+ "loss": 1.1356,
+ "theoretical_loss": 3.53694684264521,
+ "tokens_seen": 1537343488
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004316950852557673,
+ "loss": 1.315,
+ "theoretical_loss": 3.5369337181064986,
+ "tokens_seen": 1537409024
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043167903711133405,
+ "loss": 1.2999,
+ "theoretical_loss": 3.5369205942838855,
+ "tokens_seen": 1537474560
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004316629889669007,
+ "loss": 1.3237,
+ "theoretical_loss": 3.536907471177302,
+ "tokens_seen": 1537540096
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043164694082246745,
+ "loss": 1.2985,
+ "theoretical_loss": 3.536894348786676,
+ "tokens_seen": 1537605632
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004316308926780341,
+ "loss": 1.2835,
+ "theoretical_loss": 3.536881227111941,
+ "tokens_seen": 1537671168
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043161484453360085,
+ "loss": 1.3468,
+ "theoretical_loss": 3.536868106153026,
+ "tokens_seen": 1537736704
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004315987963891675,
+ "loss": 1.2031,
+ "theoretical_loss": 3.5368549859098612,
+ "tokens_seen": 1537802240
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043158274824473425,
+ "loss": 1.3878,
+ "theoretical_loss": 3.5368418663823773,
+ "tokens_seen": 1537867776
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004315667001003009,
+ "loss": 1.2901,
+ "theoretical_loss": 3.536828747570505,
+ "tokens_seen": 1537933312
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043155065195586766,
+ "loss": 1.4773,
+ "theoretical_loss": 3.536815629474174,
+ "tokens_seen": 1537998848
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004315346038114343,
+ "loss": 1.2473,
+ "theoretical_loss": 3.536802512093316,
+ "tokens_seen": 1538064384
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043151855566700106,
+ "loss": 1.3166,
+ "theoretical_loss": 3.5367893954278604,
+ "tokens_seen": 1538129920
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004315025075225677,
+ "loss": 1.2257,
+ "theoretical_loss": 3.5367762794777384,
+ "tokens_seen": 1538195456
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043148645937813446,
+ "loss": 1.2275,
+ "theoretical_loss": 3.5367631642428803,
+ "tokens_seen": 1538260992
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004314704112337011,
+ "loss": 1.2817,
+ "theoretical_loss": 3.5367500497232163,
+ "tokens_seen": 1538326528
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004314543630892678,
+ "loss": 1.2398,
+ "theoretical_loss": 3.5367369359186775,
+ "tokens_seen": 1538392064
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004314383149448345,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5367238228291944,
+ "tokens_seen": 1538457600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004314222668004012,
+ "loss": 1.3107,
+ "theoretical_loss": 3.536710710454697,
+ "tokens_seen": 1538523136
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004314062186559679,
+ "loss": 1.3775,
+ "theoretical_loss": 3.536697598795116,
+ "tokens_seen": 1538588672
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004313901705115346,
+ "loss": 1.1865,
+ "theoretical_loss": 3.5366844878503825,
+ "tokens_seen": 1538654208
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004313741223671013,
+ "loss": 1.3973,
+ "theoretical_loss": 3.5366713776204266,
+ "tokens_seen": 1538719744
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000431358074222668,
+ "loss": 1.4222,
+ "theoretical_loss": 3.5366582681051795,
+ "tokens_seen": 1538785280
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004313420260782347,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5366451593045705,
+ "tokens_seen": 1538850816
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004313259779338014,
+ "loss": 1.3019,
+ "theoretical_loss": 3.5366320512185316,
+ "tokens_seen": 1538916352
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043130992978936806,
+ "loss": 1.4575,
+ "theoretical_loss": 3.536618943846993,
+ "tokens_seen": 1538981888
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004312938816449348,
+ "loss": 1.2935,
+ "theoretical_loss": 3.5366058371898847,
+ "tokens_seen": 1539047424
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043127783350050146,
+ "loss": 1.2728,
+ "theoretical_loss": 3.536592731247138,
+ "tokens_seen": 1539112960
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004312617853560682,
+ "loss": 1.3154,
+ "theoretical_loss": 3.5365796260186837,
+ "tokens_seen": 1539178496
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043124573721163486,
+ "loss": 1.3419,
+ "theoretical_loss": 3.5365665215044517,
+ "tokens_seen": 1539244032
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004312296890672016,
+ "loss": 1.2948,
+ "theoretical_loss": 3.536553417704374,
+ "tokens_seen": 1539309568
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043121364092276837,
+ "loss": 1.3514,
+ "theoretical_loss": 3.536540314618379,
+ "tokens_seen": 1539375104
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000431197592778335,
+ "loss": 1.2423,
+ "theoretical_loss": 3.5365272122463995,
+ "tokens_seen": 1539440640
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043118154463390177,
+ "loss": 1.3184,
+ "theoretical_loss": 3.5365141105883655,
+ "tokens_seen": 1539506176
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004311654964894684,
+ "loss": 1.2177,
+ "theoretical_loss": 3.5365010096442075,
+ "tokens_seen": 1539571712
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043114944834503517,
+ "loss": 1.3086,
+ "theoretical_loss": 3.5364879094138564,
+ "tokens_seen": 1539637248
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004311334002006018,
+ "loss": 1.3674,
+ "theoretical_loss": 3.5364748098972436,
+ "tokens_seen": 1539702784
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004311173520561686,
+ "loss": 1.3462,
+ "theoretical_loss": 3.536461711094298,
+ "tokens_seen": 1539768320
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004311013039117352,
+ "loss": 1.2895,
+ "theoretical_loss": 3.5364486130049526,
+ "tokens_seen": 1539833856
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000431085255767302,
+ "loss": 1.3519,
+ "theoretical_loss": 3.5364355156291367,
+ "tokens_seen": 1539899392
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004310692076228686,
+ "loss": 1.3387,
+ "theoretical_loss": 3.5364224189667812,
+ "tokens_seen": 1539964928
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004310531594784354,
+ "loss": 1.2045,
+ "theoretical_loss": 3.536409323017817,
+ "tokens_seen": 1540030464
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 516450,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1523512601852417,
+ "objective/train/theoretical_loss": 3.536396227782175,
+ "objective/train/tokens_used": 1560556000,
+ "theoretical_loss": 3.536396227782175,
+ "tokens_seen": 1540096000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000431037111334002,
+ "loss": 1.2728,
+ "theoretical_loss": 3.536396227782175,
+ "tokens_seen": 1540096000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004310210631895688,
+ "loss": 1.2614,
+ "theoretical_loss": 3.5363831332597866,
+ "tokens_seen": 1540161536
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004310050150451354,
+ "loss": 1.3329,
+ "theoretical_loss": 3.5363700394505813,
+ "tokens_seen": 1540227072
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004309889669007022,
+ "loss": 1.3524,
+ "theoretical_loss": 3.5363569463544913,
+ "tokens_seen": 1540292608
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004309729187562688,
+ "loss": 1.398,
+ "theoretical_loss": 3.536343853971446,
+ "tokens_seen": 1540358144
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004309568706118356,
+ "loss": 1.345,
+ "theoretical_loss": 3.536330762301377,
+ "tokens_seen": 1540423680
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004309408224674022,
+ "loss": 1.3059,
+ "theoretical_loss": 3.536317671344216,
+ "tokens_seen": 1540489216
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000430924774322969,
+ "loss": 1.3272,
+ "theoretical_loss": 3.536304581099892,
+ "tokens_seen": 1540554752
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004309087261785356,
+ "loss": 1.2714,
+ "theoretical_loss": 3.5362914915683374,
+ "tokens_seen": 1540620288
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004308926780341024,
+ "loss": 1.2463,
+ "theoretical_loss": 3.536278402749482,
+ "tokens_seen": 1540685824
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043087662988966903,
+ "loss": 1.2279,
+ "theoretical_loss": 3.5362653146432574,
+ "tokens_seen": 1540751360
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004308605817452358,
+ "loss": 1.3438,
+ "theoretical_loss": 3.5362522272495944,
+ "tokens_seen": 1540816896
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043084453360080243,
+ "loss": 1.4957,
+ "theoretical_loss": 3.5362391405684237,
+ "tokens_seen": 1540882432
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004308284854563692,
+ "loss": 1.2327,
+ "theoretical_loss": 3.536226054599676,
+ "tokens_seen": 1540947968
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043081243731193583,
+ "loss": 1.5373,
+ "theoretical_loss": 3.536212969343283,
+ "tokens_seen": 1541013504
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043079638916750253,
+ "loss": 1.3178,
+ "theoretical_loss": 3.5361998847991747,
+ "tokens_seen": 1541079040
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043078034102306923,
+ "loss": 1.2739,
+ "theoretical_loss": 3.5361868009672826,
+ "tokens_seen": 1541144576
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043076429287863593,
+ "loss": 1.3036,
+ "theoretical_loss": 3.5361737178475376,
+ "tokens_seen": 1541210112
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043074824473420263,
+ "loss": 1.3397,
+ "theoretical_loss": 3.5361606354398702,
+ "tokens_seen": 1541275648
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043073219658976933,
+ "loss": 1.2485,
+ "theoretical_loss": 3.536147553744212,
+ "tokens_seen": 1541341184
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043071614844533603,
+ "loss": 1.3287,
+ "theoretical_loss": 3.5361344727604935,
+ "tokens_seen": 1541406720
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043070010030090274,
+ "loss": 1.4,
+ "theoretical_loss": 3.5361213924886465,
+ "tokens_seen": 1541472256
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004306840521564694,
+ "loss": 1.296,
+ "theoretical_loss": 3.536108312928601,
+ "tokens_seen": 1541537792
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043066800401203614,
+ "loss": 1.2847,
+ "theoretical_loss": 3.536095234080288,
+ "tokens_seen": 1541603328
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004306519558676028,
+ "loss": 1.2352,
+ "theoretical_loss": 3.5360821559436397,
+ "tokens_seen": 1541668864
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043063590772316954,
+ "loss": 1.3323,
+ "theoretical_loss": 3.5360690785185858,
+ "tokens_seen": 1541734400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004306198595787362,
+ "loss": 1.2805,
+ "theoretical_loss": 3.5360560018050577,
+ "tokens_seen": 1541799936
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043060381143430294,
+ "loss": 1.3551,
+ "theoretical_loss": 3.5360429258029873,
+ "tokens_seen": 1541865472
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004305877632898696,
+ "loss": 1.3619,
+ "theoretical_loss": 3.5360298505123047,
+ "tokens_seen": 1541931008
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043057171514543634,
+ "loss": 1.3813,
+ "theoretical_loss": 3.536016775932941,
+ "tokens_seen": 1541996544
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000430555667001003,
+ "loss": 1.2507,
+ "theoretical_loss": 3.5360037020648276,
+ "tokens_seen": 1542062080
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043053961885656974,
+ "loss": 1.3182,
+ "theoretical_loss": 3.5359906289078955,
+ "tokens_seen": 1542127616
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004305235707121364,
+ "loss": 1.2877,
+ "theoretical_loss": 3.5359775564620763,
+ "tokens_seen": 1542193152
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043050752256770314,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5359644847273,
+ "tokens_seen": 1542258688
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004304914744232698,
+ "loss": 1.2747,
+ "theoretical_loss": 3.535951413703499,
+ "tokens_seen": 1542324224
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043047542627883654,
+ "loss": 1.3439,
+ "theoretical_loss": 3.535938343390603,
+ "tokens_seen": 1542389760
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004304593781344032,
+ "loss": 1.2986,
+ "theoretical_loss": 3.535925273788544,
+ "tokens_seen": 1542455296
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043044332998996995,
+ "loss": 1.3168,
+ "theoretical_loss": 3.5359122048972527,
+ "tokens_seen": 1542520832
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004304272818455366,
+ "loss": 1.3188,
+ "theoretical_loss": 3.5358991367166617,
+ "tokens_seen": 1542586368
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043041123370110335,
+ "loss": 1.292,
+ "theoretical_loss": 3.5358860692466996,
+ "tokens_seen": 1542651904
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043039518555667,
+ "loss": 1.3397,
+ "theoretical_loss": 3.5358730024872997,
+ "tokens_seen": 1542717440
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043037913741223675,
+ "loss": 1.2441,
+ "theoretical_loss": 3.535859936438393,
+ "tokens_seen": 1542782976
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004303630892678034,
+ "loss": 1.294,
+ "theoretical_loss": 3.5358468710999094,
+ "tokens_seen": 1542848512
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043034704112337015,
+ "loss": 1.3804,
+ "theoretical_loss": 3.535833806471781,
+ "tokens_seen": 1542914048
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004303309929789368,
+ "loss": 1.3572,
+ "theoretical_loss": 3.5358207425539385,
+ "tokens_seen": 1542979584
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043031494483450355,
+ "loss": 1.2731,
+ "theoretical_loss": 3.5358076793463145,
+ "tokens_seen": 1543045120
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004302988966900702,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5357946168488383,
+ "tokens_seen": 1543110656
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043028284854563695,
+ "loss": 1.3671,
+ "theoretical_loss": 3.535781555061442,
+ "tokens_seen": 1543176192
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004302668004012036,
+ "loss": 1.4963,
+ "theoretical_loss": 3.5357684939840572,
+ "tokens_seen": 1543241728
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043025075225677035,
+ "loss": 1.3042,
+ "theoretical_loss": 3.535755433616615,
+ "tokens_seen": 1543307264
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 517633,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2595796585083008,
+ "objective/train/theoretical_loss": 3.535742373959047,
+ "objective/train/tokens_used": 1563832800,
+ "theoretical_loss": 3.535742373959047,
+ "tokens_seen": 1543372800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000430234704112337,
+ "loss": 1.2567,
+ "theoretical_loss": 3.535742373959047,
+ "tokens_seen": 1543372800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043021865596790375,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5357293150112827,
+ "tokens_seen": 1543438336
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004302026078234704,
+ "loss": 1.3431,
+ "theoretical_loss": 3.535716256773256,
+ "tokens_seen": 1543503872
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043018655967903716,
+ "loss": 1.2666,
+ "theoretical_loss": 3.535703199244896,
+ "tokens_seen": 1543569408
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004301705115346038,
+ "loss": 1.2314,
+ "theoretical_loss": 3.5356901424261347,
+ "tokens_seen": 1543634944
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043015446339017056,
+ "loss": 1.2401,
+ "theoretical_loss": 3.5356770863169036,
+ "tokens_seen": 1543700480
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004301384152457372,
+ "loss": 1.299,
+ "theoretical_loss": 3.5356640309171348,
+ "tokens_seen": 1543766016
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043012236710130396,
+ "loss": 1.2254,
+ "theoretical_loss": 3.5356509762267585,
+ "tokens_seen": 1543831552
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004301063189568706,
+ "loss": 1.2598,
+ "theoretical_loss": 3.535637922245706,
+ "tokens_seen": 1543897088
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043009027081243736,
+ "loss": 1.3127,
+ "theoretical_loss": 3.535624868973909,
+ "tokens_seen": 1543962624
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000430074222668004,
+ "loss": 1.411,
+ "theoretical_loss": 3.5356118164112997,
+ "tokens_seen": 1544028160
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043005817452357076,
+ "loss": 1.282,
+ "theoretical_loss": 3.5355987645578084,
+ "tokens_seen": 1544093696
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004300421263791374,
+ "loss": 1.3043,
+ "theoretical_loss": 3.535585713413366,
+ "tokens_seen": 1544159232
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004300260782347041,
+ "loss": 1.3206,
+ "theoretical_loss": 3.5355726629779056,
+ "tokens_seen": 1544224768
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00043001003009027086,
+ "loss": 1.2718,
+ "theoretical_loss": 3.5355596132513565,
+ "tokens_seen": 1544290304
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004299939819458375,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5355465642336523,
+ "tokens_seen": 1544355840
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042997793380140426,
+ "loss": 1.245,
+ "theoretical_loss": 3.535533515924723,
+ "tokens_seen": 1544421376
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004299618856569709,
+ "loss": 1.3345,
+ "theoretical_loss": 3.5355204683245,
+ "tokens_seen": 1544486912
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042994583751253767,
+ "loss": 1.3975,
+ "theoretical_loss": 3.535507421432916,
+ "tokens_seen": 1544552448
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004299297893681043,
+ "loss": 1.3733,
+ "theoretical_loss": 3.535494375249901,
+ "tokens_seen": 1544617984
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042991374122367107,
+ "loss": 1.2574,
+ "theoretical_loss": 3.535481329775387,
+ "tokens_seen": 1544683520
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004298976930792377,
+ "loss": 1.3012,
+ "theoretical_loss": 3.5354682850093058,
+ "tokens_seen": 1544749056
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042988164493480447,
+ "loss": 1.2129,
+ "theoretical_loss": 3.5354552409515883,
+ "tokens_seen": 1544814592
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004298655967903711,
+ "loss": 1.2319,
+ "theoretical_loss": 3.5354421976021664,
+ "tokens_seen": 1544880128
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042984954864593787,
+ "loss": 1.2392,
+ "theoretical_loss": 3.535429154960971,
+ "tokens_seen": 1544945664
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004298335005015045,
+ "loss": 1.273,
+ "theoretical_loss": 3.5354161130279347,
+ "tokens_seen": 1545011200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042981745235707127,
+ "loss": 1.3392,
+ "theoretical_loss": 3.535403071802988,
+ "tokens_seen": 1545076736
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004298014042126379,
+ "loss": 1.2347,
+ "theoretical_loss": 3.535390031286063,
+ "tokens_seen": 1545142272
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042978535606820467,
+ "loss": 1.446,
+ "theoretical_loss": 3.535376991477091,
+ "tokens_seen": 1545207808
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004297693079237713,
+ "loss": 1.2679,
+ "theoretical_loss": 3.535363952376003,
+ "tokens_seen": 1545273344
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042975325977933807,
+ "loss": 1.3099,
+ "theoretical_loss": 3.5353509139827324,
+ "tokens_seen": 1545338880
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004297372116349047,
+ "loss": 1.3434,
+ "theoretical_loss": 3.5353378762972083,
+ "tokens_seen": 1545404416
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004297211634904715,
+ "loss": 1.2683,
+ "theoretical_loss": 3.535324839319364,
+ "tokens_seen": 1545469952
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004297051153460381,
+ "loss": 1.3098,
+ "theoretical_loss": 3.5353118030491304,
+ "tokens_seen": 1545535488
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296890672016049,
+ "loss": 1.3316,
+ "theoretical_loss": 3.535298767486439,
+ "tokens_seen": 1545601024
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296730190571715,
+ "loss": 1.2115,
+ "theoretical_loss": 3.5352857326312224,
+ "tokens_seen": 1545666560
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296569709127383,
+ "loss": 1.2262,
+ "theoretical_loss": 3.5352726984834106,
+ "tokens_seen": 1545732096
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296409227683049,
+ "loss": 1.2519,
+ "theoretical_loss": 3.535259665042936,
+ "tokens_seen": 1545797632
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296248746238717,
+ "loss": 1.4258,
+ "theoretical_loss": 3.5352466323097307,
+ "tokens_seen": 1545863168
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004296088264794383,
+ "loss": 1.3336,
+ "theoretical_loss": 3.535233600283726,
+ "tokens_seen": 1545928704
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004295927783350051,
+ "loss": 1.2113,
+ "theoretical_loss": 3.5352205689648537,
+ "tokens_seen": 1545994240
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004295767301905717,
+ "loss": 1.2761,
+ "theoretical_loss": 3.5352075383530446,
+ "tokens_seen": 1546059776
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004295606820461385,
+ "loss": 1.3123,
+ "theoretical_loss": 3.5351945084482312,
+ "tokens_seen": 1546125312
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004295446339017051,
+ "loss": 1.3042,
+ "theoretical_loss": 3.535181479250345,
+ "tokens_seen": 1546190848
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004295285857572719,
+ "loss": 1.3349,
+ "theoretical_loss": 3.5351684507593175,
+ "tokens_seen": 1546256384
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042951253761283853,
+ "loss": 1.2305,
+ "theoretical_loss": 3.5351554229750812,
+ "tokens_seen": 1546321920
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004294964894684053,
+ "loss": 1.2464,
+ "theoretical_loss": 3.5351423958975667,
+ "tokens_seen": 1546387456
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042948044132397193,
+ "loss": 1.3007,
+ "theoretical_loss": 3.5351293695267056,
+ "tokens_seen": 1546452992
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004294643931795387,
+ "loss": 1.2223,
+ "theoretical_loss": 3.5351163438624313,
+ "tokens_seen": 1546518528
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042944834503510533,
+ "loss": 1.2771,
+ "theoretical_loss": 3.5351033189046737,
+ "tokens_seen": 1546584064
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 518472,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.236567497253418,
+ "objective/train/theoretical_loss": 3.535090294653365,
+ "objective/train/tokens_used": 1567109600,
+ "theoretical_loss": 3.535090294653365,
+ "tokens_seen": 1546649600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004294322968906721,
+ "loss": 1.2046,
+ "theoretical_loss": 3.535090294653365,
+ "tokens_seen": 1546649600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042941624874623873,
+ "loss": 1.3078,
+ "theoretical_loss": 3.535077271108438,
+ "tokens_seen": 1546715136
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042940020060180543,
+ "loss": 1.2607,
+ "theoretical_loss": 3.5350642482698236,
+ "tokens_seen": 1546780672
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042938415245737213,
+ "loss": 1.2726,
+ "theoretical_loss": 3.535051226137453,
+ "tokens_seen": 1546846208
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042936810431293883,
+ "loss": 1.3819,
+ "theoretical_loss": 3.5350382047112587,
+ "tokens_seen": 1546911744
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042935205616850553,
+ "loss": 1.316,
+ "theoretical_loss": 3.535025183991173,
+ "tokens_seen": 1546977280
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042933600802407223,
+ "loss": 1.2514,
+ "theoretical_loss": 3.535012163977127,
+ "tokens_seen": 1547042816
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042931995987963894,
+ "loss": 1.212,
+ "theoretical_loss": 3.5349991446690523,
+ "tokens_seen": 1547108352
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042930391173520564,
+ "loss": 1.2569,
+ "theoretical_loss": 3.5349861260668813,
+ "tokens_seen": 1547173888
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042928786359077234,
+ "loss": 1.2488,
+ "theoretical_loss": 3.5349731081705453,
+ "tokens_seen": 1547239424
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042927181544633904,
+ "loss": 1.3093,
+ "theoretical_loss": 3.5349600909799763,
+ "tokens_seen": 1547304960
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004292557673019057,
+ "loss": 1.3115,
+ "theoretical_loss": 3.534947074495107,
+ "tokens_seen": 1547370496
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042923971915747244,
+ "loss": 1.2775,
+ "theoretical_loss": 3.5349340587158675,
+ "tokens_seen": 1547436032
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004292236710130391,
+ "loss": 1.3298,
+ "theoretical_loss": 3.5349210436421914,
+ "tokens_seen": 1547501568
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042920762286860584,
+ "loss": 1.2702,
+ "theoretical_loss": 3.5349080292740096,
+ "tokens_seen": 1547567104
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004291915747241725,
+ "loss": 1.3547,
+ "theoretical_loss": 3.534895015611254,
+ "tokens_seen": 1547632640
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042917552657973924,
+ "loss": 1.3815,
+ "theoretical_loss": 3.534882002653857,
+ "tokens_seen": 1547698176
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004291594784353059,
+ "loss": 1.2962,
+ "theoretical_loss": 3.5348689904017507,
+ "tokens_seen": 1547763712
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042914343029087264,
+ "loss": 1.2315,
+ "theoretical_loss": 3.534855978854866,
+ "tokens_seen": 1547829248
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004291273821464393,
+ "loss": 1.236,
+ "theoretical_loss": 3.5348429680131357,
+ "tokens_seen": 1547894784
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042911133400200604,
+ "loss": 1.3738,
+ "theoretical_loss": 3.5348299578764912,
+ "tokens_seen": 1547960320
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004290952858575727,
+ "loss": 1.3437,
+ "theoretical_loss": 3.534816948444864,
+ "tokens_seen": 1548025856
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042907923771313945,
+ "loss": 1.2632,
+ "theoretical_loss": 3.534803939718188,
+ "tokens_seen": 1548091392
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004290631895687061,
+ "loss": 1.3386,
+ "theoretical_loss": 3.534790931696393,
+ "tokens_seen": 1548156928
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042904714142427285,
+ "loss": 1.2929,
+ "theoretical_loss": 3.5347779243794113,
+ "tokens_seen": 1548222464
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004290310932798395,
+ "loss": 1.2576,
+ "theoretical_loss": 3.5347649177671765,
+ "tokens_seen": 1548288000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042901504513540625,
+ "loss": 1.2951,
+ "theoretical_loss": 3.534751911859619,
+ "tokens_seen": 1548353536
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004289989969909729,
+ "loss": 1.3275,
+ "theoretical_loss": 3.534738906656671,
+ "tokens_seen": 1548419072
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042898294884653965,
+ "loss": 1.4164,
+ "theoretical_loss": 3.5347259021582653,
+ "tokens_seen": 1548484608
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004289669007021063,
+ "loss": 1.2645,
+ "theoretical_loss": 3.534712898364333,
+ "tokens_seen": 1548550144
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042895085255767305,
+ "loss": 1.3163,
+ "theoretical_loss": 3.534699895274807,
+ "tokens_seen": 1548615680
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004289348044132397,
+ "loss": 1.1835,
+ "theoretical_loss": 3.5346868928896185,
+ "tokens_seen": 1548681216
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042891875626880645,
+ "loss": 1.3344,
+ "theoretical_loss": 3.5346738912087003,
+ "tokens_seen": 1548746752
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004289027081243731,
+ "loss": 1.2556,
+ "theoretical_loss": 3.5346608902319834,
+ "tokens_seen": 1548812288
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042888665997993985,
+ "loss": 1.2936,
+ "theoretical_loss": 3.534647889959401,
+ "tokens_seen": 1548877824
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004288706118355065,
+ "loss": 1.2922,
+ "theoretical_loss": 3.5346348903908846,
+ "tokens_seen": 1548943360
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042885456369107325,
+ "loss": 1.2536,
+ "theoretical_loss": 3.534621891526366,
+ "tokens_seen": 1549008896
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004288385155466399,
+ "loss": 1.3227,
+ "theoretical_loss": 3.534608893365778,
+ "tokens_seen": 1549074432
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042882246740220666,
+ "loss": 1.3009,
+ "theoretical_loss": 3.534595895909053,
+ "tokens_seen": 1549139968
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004288064192577734,
+ "loss": 1.2954,
+ "theoretical_loss": 3.534582899156122,
+ "tokens_seen": 1549205504
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042879037111334006,
+ "loss": 1.2277,
+ "theoretical_loss": 3.5345699031069175,
+ "tokens_seen": 1549271040
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042877432296890676,
+ "loss": 1.1816,
+ "theoretical_loss": 3.534556907761372,
+ "tokens_seen": 1549336576
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042875827482447346,
+ "loss": 1.3137,
+ "theoretical_loss": 3.5345439131194167,
+ "tokens_seen": 1549402112
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042874222668004016,
+ "loss": 1.2871,
+ "theoretical_loss": 3.534530919180985,
+ "tokens_seen": 1549467648
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042872617853560686,
+ "loss": 1.3037,
+ "theoretical_loss": 3.5345179259460084,
+ "tokens_seen": 1549533184
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042871013039117356,
+ "loss": 1.2883,
+ "theoretical_loss": 3.534504933414419,
+ "tokens_seen": 1549598720
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042869408224674026,
+ "loss": 1.3328,
+ "theoretical_loss": 3.5344919415861495,
+ "tokens_seen": 1549664256
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042867803410230696,
+ "loss": 1.2862,
+ "theoretical_loss": 3.5344789504611316,
+ "tokens_seen": 1549729792
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042866198595787366,
+ "loss": 1.2269,
+ "theoretical_loss": 3.5344659600392974,
+ "tokens_seen": 1549795328
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042864593781344036,
+ "loss": 1.3232,
+ "theoretical_loss": 3.5344529703205794,
+ "tokens_seen": 1549860864
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 519652,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1471920013427734,
+ "objective/train/theoretical_loss": 3.53443998130491,
+ "objective/train/tokens_used": 1570386400,
+ "theoretical_loss": 3.53443998130491,
+ "tokens_seen": 1549926400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428629889669007,
+ "loss": 1.2064,
+ "theoretical_loss": 3.53443998130491,
+ "tokens_seen": 1549926400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042861384152457376,
+ "loss": 1.1769,
+ "theoretical_loss": 3.5344269929922207,
+ "tokens_seen": 1549991936
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004285977933801404,
+ "loss": 1.4177,
+ "theoretical_loss": 3.5344140053824447,
+ "tokens_seen": 1550057472
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042858174523570716,
+ "loss": 1.261,
+ "theoretical_loss": 3.534401018475513,
+ "tokens_seen": 1550123008
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004285656970912738,
+ "loss": 1.2789,
+ "theoretical_loss": 3.5343880322713592,
+ "tokens_seen": 1550188544
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042854964894684057,
+ "loss": 1.3154,
+ "theoretical_loss": 3.534375046769915,
+ "tokens_seen": 1550254080
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004285336008024072,
+ "loss": 1.3079,
+ "theoretical_loss": 3.5343620619711125,
+ "tokens_seen": 1550319616
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042851755265797397,
+ "loss": 1.2498,
+ "theoretical_loss": 3.5343490778748836,
+ "tokens_seen": 1550385152
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004285015045135406,
+ "loss": 1.2783,
+ "theoretical_loss": 3.5343360944811613,
+ "tokens_seen": 1550450688
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042848545636910737,
+ "loss": 1.3021,
+ "theoretical_loss": 3.5343231117898783,
+ "tokens_seen": 1550516224
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428469408224674,
+ "loss": 1.2649,
+ "theoretical_loss": 3.5343101298009656,
+ "tokens_seen": 1550581760
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042845336008024077,
+ "loss": 1.2425,
+ "theoretical_loss": 3.534297148514357,
+ "tokens_seen": 1550647296
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004284373119358074,
+ "loss": 1.2251,
+ "theoretical_loss": 3.5342841679299832,
+ "tokens_seen": 1550712832
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042842126379137417,
+ "loss": 1.3239,
+ "theoretical_loss": 3.5342711880477777,
+ "tokens_seen": 1550778368
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004284052156469408,
+ "loss": 1.3285,
+ "theoretical_loss": 3.534258208867672,
+ "tokens_seen": 1550843904
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042838916750250757,
+ "loss": 1.3055,
+ "theoretical_loss": 3.5342452303895997,
+ "tokens_seen": 1550909440
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004283731193580742,
+ "loss": 1.2488,
+ "theoretical_loss": 3.534232252613492,
+ "tokens_seen": 1550974976
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428357071213641,
+ "loss": 1.3286,
+ "theoretical_loss": 3.5342192755392823,
+ "tokens_seen": 1551040512
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004283410230692076,
+ "loss": 1.3346,
+ "theoretical_loss": 3.534206299166902,
+ "tokens_seen": 1551106048
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004283249749247744,
+ "loss": 1.2439,
+ "theoretical_loss": 3.5341933234962837,
+ "tokens_seen": 1551171584
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428308926780341,
+ "loss": 1.3698,
+ "theoretical_loss": 3.53418034852736,
+ "tokens_seen": 1551237120
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282928786359078,
+ "loss": 1.2807,
+ "theoretical_loss": 3.534167374260064,
+ "tokens_seen": 1551302656
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282768304914744,
+ "loss": 1.2501,
+ "theoretical_loss": 3.5341544006943266,
+ "tokens_seen": 1551368192
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282607823470412,
+ "loss": 1.4101,
+ "theoretical_loss": 3.5341414278300816,
+ "tokens_seen": 1551433728
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282447342026078,
+ "loss": 1.3603,
+ "theoretical_loss": 3.5341284556672603,
+ "tokens_seen": 1551499264
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282286860581746,
+ "loss": 1.2832,
+ "theoretical_loss": 3.5341154842057962,
+ "tokens_seen": 1551564800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004282126379137412,
+ "loss": 1.3274,
+ "theoretical_loss": 3.5341025134456214,
+ "tokens_seen": 1551630336
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428196589769308,
+ "loss": 1.2631,
+ "theoretical_loss": 3.534089543386668,
+ "tokens_seen": 1551695872
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004281805416248746,
+ "loss": 1.3186,
+ "theoretical_loss": 3.5340765740288687,
+ "tokens_seen": 1551761408
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004281644934804414,
+ "loss": 1.3629,
+ "theoretical_loss": 3.5340636053721566,
+ "tokens_seen": 1551826944
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042814844533600803,
+ "loss": 1.1716,
+ "theoretical_loss": 3.534050637416463,
+ "tokens_seen": 1551892480
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004281323971915748,
+ "loss": 1.2977,
+ "theoretical_loss": 3.5340376701617213,
+ "tokens_seen": 1551958016
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042811634904714143,
+ "loss": 1.3734,
+ "theoretical_loss": 3.534024703607864,
+ "tokens_seen": 1552023552
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004281003009027082,
+ "loss": 1.2986,
+ "theoretical_loss": 3.5340117377548226,
+ "tokens_seen": 1552089088
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042808425275827483,
+ "loss": 1.2099,
+ "theoretical_loss": 3.533998772602531,
+ "tokens_seen": 1552154624
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004280682046138416,
+ "loss": 1.3073,
+ "theoretical_loss": 3.533985808150921,
+ "tokens_seen": 1552220160
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042805215646940823,
+ "loss": 1.2613,
+ "theoretical_loss": 3.5339728443999254,
+ "tokens_seen": 1552285696
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000428036108324975,
+ "loss": 1.3253,
+ "theoretical_loss": 3.5339598813494764,
+ "tokens_seen": 1552351232
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042802006018054163,
+ "loss": 1.3133,
+ "theoretical_loss": 3.533946918999507,
+ "tokens_seen": 1552416768
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042800401203610833,
+ "loss": 1.2276,
+ "theoretical_loss": 3.5339339573499498,
+ "tokens_seen": 1552482304
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042798796389167503,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5339209964007368,
+ "tokens_seen": 1552547840
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042797191574724173,
+ "loss": 1.3577,
+ "theoretical_loss": 3.533908036151801,
+ "tokens_seen": 1552613376
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042795586760280844,
+ "loss": 1.2769,
+ "theoretical_loss": 3.5338950766030752,
+ "tokens_seen": 1552678912
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042793981945837514,
+ "loss": 1.3473,
+ "theoretical_loss": 3.533882117754492,
+ "tokens_seen": 1552744448
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042792377131394184,
+ "loss": 1.2914,
+ "theoretical_loss": 3.5338691596059837,
+ "tokens_seen": 1552809984
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042790772316950854,
+ "loss": 1.3106,
+ "theoretical_loss": 3.533856202157483,
+ "tokens_seen": 1552875520
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042789167502507524,
+ "loss": 1.254,
+ "theoretical_loss": 3.5338432454089226,
+ "tokens_seen": 1552941056
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042787562688064194,
+ "loss": 1.2376,
+ "theoretical_loss": 3.5338302893602354,
+ "tokens_seen": 1553006592
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004278595787362086,
+ "loss": 1.2185,
+ "theoretical_loss": 3.5338173340113537,
+ "tokens_seen": 1553072128
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042784353059177534,
+ "loss": 1.3752,
+ "theoretical_loss": 3.533804379362211,
+ "tokens_seen": 1553137664
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 520524,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.32770836353302,
+ "objective/train/theoretical_loss": 3.5337914254127383,
+ "objective/train/tokens_used": 1573663200,
+ "theoretical_loss": 3.5337914254127383,
+ "tokens_seen": 1553203200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000427827482447342,
+ "loss": 1.3689,
+ "theoretical_loss": 3.5337914254127383,
+ "tokens_seen": 1553203200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042781143430290874,
+ "loss": 1.2887,
+ "theoretical_loss": 3.5337784721628696,
+ "tokens_seen": 1553268736
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004277953861584754,
+ "loss": 1.3047,
+ "theoretical_loss": 3.5337655196125373,
+ "tokens_seen": 1553334272
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042777933801404214,
+ "loss": 1.3148,
+ "theoretical_loss": 3.533752567761674,
+ "tokens_seen": 1553399808
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004277632898696088,
+ "loss": 1.3365,
+ "theoretical_loss": 3.533739616610213,
+ "tokens_seen": 1553465344
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042774724172517554,
+ "loss": 1.3883,
+ "theoretical_loss": 3.5337266661580857,
+ "tokens_seen": 1553530880
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004277311935807422,
+ "loss": 1.3837,
+ "theoretical_loss": 3.5337137164052264,
+ "tokens_seen": 1553596416
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042771514543630894,
+ "loss": 1.2435,
+ "theoretical_loss": 3.5337007673515672,
+ "tokens_seen": 1553661952
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004276990972918756,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5336878189970404,
+ "tokens_seen": 1553727488
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042768304914744235,
+ "loss": 1.4372,
+ "theoretical_loss": 3.5336748713415793,
+ "tokens_seen": 1553793024
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000427667001003009,
+ "loss": 1.3402,
+ "theoretical_loss": 3.533661924385117,
+ "tokens_seen": 1553858560
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042765095285857575,
+ "loss": 1.3643,
+ "theoretical_loss": 3.5336489781275855,
+ "tokens_seen": 1553924096
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004276349047141424,
+ "loss": 1.2836,
+ "theoretical_loss": 3.5336360325689173,
+ "tokens_seen": 1553989632
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042761885656970915,
+ "loss": 1.2715,
+ "theoretical_loss": 3.5336230877090467,
+ "tokens_seen": 1554055168
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004276028084252759,
+ "loss": 1.2843,
+ "theoretical_loss": 3.533610143547905,
+ "tokens_seen": 1554120704
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042758676028084255,
+ "loss": 1.2929,
+ "theoretical_loss": 3.533597200085426,
+ "tokens_seen": 1554186240
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004275707121364093,
+ "loss": 1.2736,
+ "theoretical_loss": 3.533584257321542,
+ "tokens_seen": 1554251776
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042755466399197595,
+ "loss": 1.3559,
+ "theoretical_loss": 3.5335713152561863,
+ "tokens_seen": 1554317312
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004275386158475427,
+ "loss": 1.2667,
+ "theoretical_loss": 3.5335583738892913,
+ "tokens_seen": 1554382848
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042752256770310935,
+ "loss": 1.2962,
+ "theoretical_loss": 3.53354543322079,
+ "tokens_seen": 1554448384
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004275065195586761,
+ "loss": 1.2324,
+ "theoretical_loss": 3.5335324932506156,
+ "tokens_seen": 1554513920
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042749047141424275,
+ "loss": 1.1982,
+ "theoretical_loss": 3.5335195539787003,
+ "tokens_seen": 1554579456
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004274744232698095,
+ "loss": 1.3705,
+ "theoretical_loss": 3.5335066154049777,
+ "tokens_seen": 1554644992
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042745837512537616,
+ "loss": 1.3359,
+ "theoretical_loss": 3.53349367752938,
+ "tokens_seen": 1554710528
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004274423269809429,
+ "loss": 1.3287,
+ "theoretical_loss": 3.5334807403518407,
+ "tokens_seen": 1554776064
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042742627883650956,
+ "loss": 1.3146,
+ "theoretical_loss": 3.533467803872292,
+ "tokens_seen": 1554841600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004274102306920763,
+ "loss": 1.2455,
+ "theoretical_loss": 3.533454868090668,
+ "tokens_seen": 1554907136
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042739418254764296,
+ "loss": 1.2268,
+ "theoretical_loss": 3.533441933006901,
+ "tokens_seen": 1554972672
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042737813440320966,
+ "loss": 1.3357,
+ "theoretical_loss": 3.5334289986209235,
+ "tokens_seen": 1555038208
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042736208625877636,
+ "loss": 1.2692,
+ "theoretical_loss": 3.533416064932669,
+ "tokens_seen": 1555103744
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042734603811434306,
+ "loss": 1.3651,
+ "theoretical_loss": 3.53340313194207,
+ "tokens_seen": 1555169280
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042732998996990976,
+ "loss": 1.3358,
+ "theoretical_loss": 3.5333901996490598,
+ "tokens_seen": 1555234816
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042731394182547646,
+ "loss": 1.3439,
+ "theoretical_loss": 3.533377268053572,
+ "tokens_seen": 1555300352
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042729789368104316,
+ "loss": 1.3057,
+ "theoretical_loss": 3.533364337155538,
+ "tokens_seen": 1555365888
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042728184553660986,
+ "loss": 1.2499,
+ "theoretical_loss": 3.5333514069548926,
+ "tokens_seen": 1555431424
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042726579739217656,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5333384774515677,
+ "tokens_seen": 1555496960
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042724974924774326,
+ "loss": 1.3315,
+ "theoretical_loss": 3.533325548645496,
+ "tokens_seen": 1555562496
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004272337011033099,
+ "loss": 1.3259,
+ "theoretical_loss": 3.533312620536612,
+ "tokens_seen": 1555628032
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042721765295887666,
+ "loss": 1.2772,
+ "theoretical_loss": 3.533299693124847,
+ "tokens_seen": 1555693568
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004272016048144433,
+ "loss": 1.3266,
+ "theoretical_loss": 3.5332867664101353,
+ "tokens_seen": 1555759104
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042718555667001007,
+ "loss": 1.3005,
+ "theoretical_loss": 3.5332738403924093,
+ "tokens_seen": 1555824640
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004271695085255767,
+ "loss": 1.3301,
+ "theoretical_loss": 3.533260915071603,
+ "tokens_seen": 1555890176
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042715346038114347,
+ "loss": 1.2411,
+ "theoretical_loss": 3.5332479904476477,
+ "tokens_seen": 1555955712
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004271374122367101,
+ "loss": 1.3077,
+ "theoretical_loss": 3.533235066520478,
+ "tokens_seen": 1556021248
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042712136409227687,
+ "loss": 1.2867,
+ "theoretical_loss": 3.5332221432900264,
+ "tokens_seen": 1556086784
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004271053159478435,
+ "loss": 1.4103,
+ "theoretical_loss": 3.5332092207562265,
+ "tokens_seen": 1556152320
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042708926780341027,
+ "loss": 1.3604,
+ "theoretical_loss": 3.5331962989190107,
+ "tokens_seen": 1556217856
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004270732196589769,
+ "loss": 1.3854,
+ "theoretical_loss": 3.5331833777783124,
+ "tokens_seen": 1556283392
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042705717151454367,
+ "loss": 1.2996,
+ "theoretical_loss": 3.533170457334065,
+ "tokens_seen": 1556348928
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004270411233701103,
+ "loss": 1.2552,
+ "theoretical_loss": 3.5331575375862014,
+ "tokens_seen": 1556414464
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 521491,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3240457773208618,
+ "objective/train/theoretical_loss": 3.5331446185346547,
+ "objective/train/tokens_used": 1576940000,
+ "theoretical_loss": 3.5331446185346547,
+ "tokens_seen": 1556480000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042702507522567707,
+ "loss": 1.2918,
+ "theoretical_loss": 3.5331446185346547,
+ "tokens_seen": 1556480000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004270090270812437,
+ "loss": 1.2906,
+ "theoretical_loss": 3.533131700179358,
+ "tokens_seen": 1556545536
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269929789368105,
+ "loss": 1.2264,
+ "theoretical_loss": 3.533118782520245,
+ "tokens_seen": 1556611072
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269769307923771,
+ "loss": 1.3078,
+ "theoretical_loss": 3.533105865557248,
+ "tokens_seen": 1556676608
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269608826479439,
+ "loss": 1.2517,
+ "theoretical_loss": 3.5330929492903005,
+ "tokens_seen": 1556742144
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269448345035105,
+ "loss": 1.4029,
+ "theoretical_loss": 3.5330800337193367,
+ "tokens_seen": 1556807680
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269287863590773,
+ "loss": 1.3347,
+ "theoretical_loss": 3.533067118844288,
+ "tokens_seen": 1556873216
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004269127382146439,
+ "loss": 1.4732,
+ "theoretical_loss": 3.5330542046650892,
+ "tokens_seen": 1556938752
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268966900702107,
+ "loss": 1.3391,
+ "theoretical_loss": 3.5330412911816724,
+ "tokens_seen": 1557004288
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268806419257773,
+ "loss": 1.3001,
+ "theoretical_loss": 3.5330283783939715,
+ "tokens_seen": 1557069824
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268645937813441,
+ "loss": 1.3208,
+ "theoretical_loss": 3.5330154663019195,
+ "tokens_seen": 1557135360
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268485456369107,
+ "loss": 1.3041,
+ "theoretical_loss": 3.53300255490545,
+ "tokens_seen": 1557200896
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268324974924775,
+ "loss": 1.2906,
+ "theoretical_loss": 3.5329896442044957,
+ "tokens_seen": 1557266432
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268164493480441,
+ "loss": 1.3582,
+ "theoretical_loss": 3.5329767341989897,
+ "tokens_seen": 1557331968
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004268004012036109,
+ "loss": 1.3053,
+ "theoretical_loss": 3.532963824888866,
+ "tokens_seen": 1557397504
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042678435305917753,
+ "loss": 1.2449,
+ "theoretical_loss": 3.532950916274058,
+ "tokens_seen": 1557463040
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004267683049147443,
+ "loss": 1.3824,
+ "theoretical_loss": 3.532938008354498,
+ "tokens_seen": 1557528576
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042675225677031093,
+ "loss": 1.3398,
+ "theoretical_loss": 3.53292510113012,
+ "tokens_seen": 1557594112
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004267362086258777,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5329121946008573,
+ "tokens_seen": 1557659648
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042672016048144433,
+ "loss": 1.2383,
+ "theoretical_loss": 3.532899288766643,
+ "tokens_seen": 1557725184
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004267041123370111,
+ "loss": 1.2848,
+ "theoretical_loss": 3.5328863836274103,
+ "tokens_seen": 1557790720
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042668806419257773,
+ "loss": 1.3185,
+ "theoretical_loss": 3.532873479183093,
+ "tokens_seen": 1557856256
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004266720160481445,
+ "loss": 1.203,
+ "theoretical_loss": 3.5328605754336246,
+ "tokens_seen": 1557921792
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042665596790371113,
+ "loss": 1.3224,
+ "theoretical_loss": 3.5328476723789377,
+ "tokens_seen": 1557987328
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004266399197592779,
+ "loss": 1.2763,
+ "theoretical_loss": 3.532834770018966,
+ "tokens_seen": 1558052864
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042662387161484453,
+ "loss": 1.4266,
+ "theoretical_loss": 3.5328218683536425,
+ "tokens_seen": 1558118400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042660782347041123,
+ "loss": 1.3181,
+ "theoretical_loss": 3.532808967382902,
+ "tokens_seen": 1558183936
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042659177532597794,
+ "loss": 1.2314,
+ "theoretical_loss": 3.532796067106676,
+ "tokens_seen": 1558249472
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042657572718154464,
+ "loss": 1.3438,
+ "theoretical_loss": 3.532783167524899,
+ "tokens_seen": 1558315008
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042655967903711134,
+ "loss": 1.3015,
+ "theoretical_loss": 3.5327702686375044,
+ "tokens_seen": 1558380544
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042654363089267804,
+ "loss": 1.3237,
+ "theoretical_loss": 3.5327573704444255,
+ "tokens_seen": 1558446080
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042652758274824474,
+ "loss": 1.3257,
+ "theoretical_loss": 3.5327444729455957,
+ "tokens_seen": 1558511616
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042651153460381144,
+ "loss": 1.2575,
+ "theoretical_loss": 3.5327315761409483,
+ "tokens_seen": 1558577152
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042649548645937814,
+ "loss": 1.3749,
+ "theoretical_loss": 3.532718680030417,
+ "tokens_seen": 1558642688
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042647943831494484,
+ "loss": 1.3282,
+ "theoretical_loss": 3.5327057846139347,
+ "tokens_seen": 1558708224
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004264633901705115,
+ "loss": 1.2417,
+ "theoretical_loss": 3.532692889891435,
+ "tokens_seen": 1558773760
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042644734202607824,
+ "loss": 1.3793,
+ "theoretical_loss": 3.5326799958628525,
+ "tokens_seen": 1558839296
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004264312938816449,
+ "loss": 1.2934,
+ "theoretical_loss": 3.5326671025281193,
+ "tokens_seen": 1558904832
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042641524573721164,
+ "loss": 1.2813,
+ "theoretical_loss": 3.53265420988717,
+ "tokens_seen": 1558970368
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004263991975927784,
+ "loss": 1.333,
+ "theoretical_loss": 3.5326413179399374,
+ "tokens_seen": 1559035904
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042638314944834504,
+ "loss": 1.1715,
+ "theoretical_loss": 3.5326284266863546,
+ "tokens_seen": 1559101440
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004263671013039118,
+ "loss": 1.2605,
+ "theoretical_loss": 3.5326155361263565,
+ "tokens_seen": 1559166976
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042635105315947844,
+ "loss": 1.3299,
+ "theoretical_loss": 3.532602646259875,
+ "tokens_seen": 1559232512
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004263350050150452,
+ "loss": 1.3748,
+ "theoretical_loss": 3.532589757086845,
+ "tokens_seen": 1559298048
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042631895687061185,
+ "loss": 1.373,
+ "theoretical_loss": 3.532576868607199,
+ "tokens_seen": 1559363584
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004263029087261786,
+ "loss": 1.1886,
+ "theoretical_loss": 3.5325639808208718,
+ "tokens_seen": 1559429120
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042628686058174525,
+ "loss": 1.2684,
+ "theoretical_loss": 3.532551093727796,
+ "tokens_seen": 1559494656
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000426270812437312,
+ "loss": 1.2345,
+ "theoretical_loss": 3.5325382073279052,
+ "tokens_seen": 1559560192
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042625476429287865,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5325253216211334,
+ "tokens_seen": 1559625728
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004262387161484454,
+ "loss": 1.3175,
+ "theoretical_loss": 3.532512436607414,
+ "tokens_seen": 1559691264
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 522672,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3013179302215576,
+ "objective/train/theoretical_loss": 3.5324995522866804,
+ "objective/train/tokens_used": 1580216800,
+ "theoretical_loss": 3.5324995522866804,
+ "tokens_seen": 1559756800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042622266800401205,
+ "loss": 1.2442,
+ "theoretical_loss": 3.5324995522866804,
+ "tokens_seen": 1559756800
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004262066198595788,
+ "loss": 1.3117,
+ "theoretical_loss": 3.5324866686588665,
+ "tokens_seen": 1559822336
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042619057171514545,
+ "loss": 1.3347,
+ "theoretical_loss": 3.532473785723906,
+ "tokens_seen": 1559887872
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004261745235707122,
+ "loss": 1.2703,
+ "theoretical_loss": 3.5324609034817325,
+ "tokens_seen": 1559953408
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042615847542627885,
+ "loss": 1.2673,
+ "theoretical_loss": 3.532448021932279,
+ "tokens_seen": 1560018944
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004261424272818456,
+ "loss": 1.1718,
+ "theoretical_loss": 3.53243514107548,
+ "tokens_seen": 1560084480
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042612637913741225,
+ "loss": 1.2249,
+ "theoretical_loss": 3.532422260911269,
+ "tokens_seen": 1560150016
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000426110330992979,
+ "loss": 1.3452,
+ "theoretical_loss": 3.532409381439579,
+ "tokens_seen": 1560215552
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042609428284854565,
+ "loss": 1.4643,
+ "theoretical_loss": 3.5323965026603448,
+ "tokens_seen": 1560281088
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004260782347041124,
+ "loss": 1.3409,
+ "theoretical_loss": 3.532383624573499,
+ "tokens_seen": 1560346624
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042606218655967906,
+ "loss": 1.2447,
+ "theoretical_loss": 3.532370747178976,
+ "tokens_seen": 1560412160
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004260461384152458,
+ "loss": 1.2747,
+ "theoretical_loss": 3.5323578704767096,
+ "tokens_seen": 1560477696
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042603009027081246,
+ "loss": 1.2598,
+ "theoretical_loss": 3.5323449944666327,
+ "tokens_seen": 1560543232
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004260140421263792,
+ "loss": 1.2654,
+ "theoretical_loss": 3.5323321191486796,
+ "tokens_seen": 1560608768
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042599799398194586,
+ "loss": 1.2982,
+ "theoretical_loss": 3.532319244522784,
+ "tokens_seen": 1560674304
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042598194583751256,
+ "loss": 1.3188,
+ "theoretical_loss": 3.5323063705888798,
+ "tokens_seen": 1560739840
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042596589769307926,
+ "loss": 1.4243,
+ "theoretical_loss": 3.5322934973469002,
+ "tokens_seen": 1560805376
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042594984954864596,
+ "loss": 1.4556,
+ "theoretical_loss": 3.5322806247967793,
+ "tokens_seen": 1560870912
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042593380140421266,
+ "loss": 1.3688,
+ "theoretical_loss": 3.532267752938451,
+ "tokens_seen": 1560936448
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042591775325977936,
+ "loss": 1.2601,
+ "theoretical_loss": 3.5322548817718493,
+ "tokens_seen": 1561001984
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042590170511534606,
+ "loss": 1.3325,
+ "theoretical_loss": 3.532242011296907,
+ "tokens_seen": 1561067520
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042588565697091276,
+ "loss": 1.2179,
+ "theoretical_loss": 3.532229141513559,
+ "tokens_seen": 1561133056
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042586960882647946,
+ "loss": 1.3719,
+ "theoretical_loss": 3.5322162724217385,
+ "tokens_seen": 1561198592
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042585356068204616,
+ "loss": 1.3862,
+ "theoretical_loss": 3.5322034040213794,
+ "tokens_seen": 1561264128
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004258375125376128,
+ "loss": 1.216,
+ "theoretical_loss": 3.5321905363124158,
+ "tokens_seen": 1561329664
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042582146439317957,
+ "loss": 1.3263,
+ "theoretical_loss": 3.5321776692947813,
+ "tokens_seen": 1561395200
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004258054162487462,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5321648029684094,
+ "tokens_seen": 1561460736
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042578936810431297,
+ "loss": 1.3676,
+ "theoretical_loss": 3.532151937333235,
+ "tokens_seen": 1561526272
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004257733199598796,
+ "loss": 1.4322,
+ "theoretical_loss": 3.5321390723891906,
+ "tokens_seen": 1561591808
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042575727181544637,
+ "loss": 1.4022,
+ "theoretical_loss": 3.532126208136211,
+ "tokens_seen": 1561657344
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000425741223671013,
+ "loss": 1.3105,
+ "theoretical_loss": 3.5321133445742294,
+ "tokens_seen": 1561722880
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042572517552657977,
+ "loss": 1.2388,
+ "theoretical_loss": 3.532100481703181,
+ "tokens_seen": 1561788416
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004257091273821464,
+ "loss": 1.364,
+ "theoretical_loss": 3.532087619522998,
+ "tokens_seen": 1561853952
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042569307923771317,
+ "loss": 1.3245,
+ "theoretical_loss": 3.5320747580336156,
+ "tokens_seen": 1561919488
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004256770310932798,
+ "loss": 1.2795,
+ "theoretical_loss": 3.532061897234967,
+ "tokens_seen": 1561985024
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042566098294884657,
+ "loss": 1.3224,
+ "theoretical_loss": 3.5320490371269866,
+ "tokens_seen": 1562050560
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004256449348044132,
+ "loss": 1.3358,
+ "theoretical_loss": 3.532036177709608,
+ "tokens_seen": 1562116096
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042562888665998,
+ "loss": 1.2707,
+ "theoretical_loss": 3.5320233189827652,
+ "tokens_seen": 1562181632
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004256128385155466,
+ "loss": 1.4499,
+ "theoretical_loss": 3.5320104609463923,
+ "tokens_seen": 1562247168
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255967903711134,
+ "loss": 1.3217,
+ "theoretical_loss": 3.5319976036004235,
+ "tokens_seen": 1562312704
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042558074222668,
+ "loss": 1.2959,
+ "theoretical_loss": 3.531984746944792,
+ "tokens_seen": 1562378240
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255646940822468,
+ "loss": 1.3239,
+ "theoretical_loss": 3.531971890979432,
+ "tokens_seen": 1562443776
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255486459378134,
+ "loss": 1.2033,
+ "theoretical_loss": 3.531959035704278,
+ "tokens_seen": 1562509312
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255325977933802,
+ "loss": 1.2849,
+ "theoretical_loss": 3.531946181119263,
+ "tokens_seen": 1562574848
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255165496489468,
+ "loss": 1.3024,
+ "theoretical_loss": 3.5319333272243227,
+ "tokens_seen": 1562640384
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004255005015045136,
+ "loss": 1.3079,
+ "theoretical_loss": 3.5319204740193895,
+ "tokens_seen": 1562705920
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004254844533600802,
+ "loss": 1.3268,
+ "theoretical_loss": 3.5319076215043985,
+ "tokens_seen": 1562771456
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000425468405215647,
+ "loss": 1.2136,
+ "theoretical_loss": 3.531894769679283,
+ "tokens_seen": 1562836992
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004254523570712136,
+ "loss": 1.1638,
+ "theoretical_loss": 3.531881918543977,
+ "tokens_seen": 1562902528
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004254363089267804,
+ "loss": 1.3084,
+ "theoretical_loss": 3.5318690680984153,
+ "tokens_seen": 1562968064
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 523864,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2828794717788696,
+ "objective/train/theoretical_loss": 3.5318562183425315,
+ "objective/train/tokens_used": 1583493600,
+ "theoretical_loss": 3.5318562183425315,
+ "tokens_seen": 1563033600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042542026078234703,
+ "loss": 1.3545,
+ "theoretical_loss": 3.5318562183425315,
+ "tokens_seen": 1563033600
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004254042126379138,
+ "loss": 1.2865,
+ "theoretical_loss": 3.5318433692762596,
+ "tokens_seen": 1563099136
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042538816449348043,
+ "loss": 1.2645,
+ "theoretical_loss": 3.5318305208995335,
+ "tokens_seen": 1563164672
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004253721163490472,
+ "loss": 1.2767,
+ "theoretical_loss": 3.5318176732122883,
+ "tokens_seen": 1563230208
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042535606820461383,
+ "loss": 1.3011,
+ "theoretical_loss": 3.531804826214457,
+ "tokens_seen": 1563295744
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004253400200601806,
+ "loss": 1.2318,
+ "theoretical_loss": 3.531791979905974,
+ "tokens_seen": 1563361280
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042532397191574723,
+ "loss": 1.2611,
+ "theoretical_loss": 3.531779134286773,
+ "tokens_seen": 1563426816
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.000425307923771314,
+ "loss": 1.2714,
+ "theoretical_loss": 3.5317662893567894,
+ "tokens_seen": 1563492352
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042529187562688063,
+ "loss": 1.3322,
+ "theoretical_loss": 3.5317534451159567,
+ "tokens_seen": 1563557888
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004252758274824474,
+ "loss": 1.3916,
+ "theoretical_loss": 3.5317406015642083,
+ "tokens_seen": 1563623424
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042525977933801403,
+ "loss": 1.2943,
+ "theoretical_loss": 3.5317277587014786,
+ "tokens_seen": 1563688960
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004252437311935808,
+ "loss": 1.4274,
+ "theoretical_loss": 3.5317149165277026,
+ "tokens_seen": 1563754496
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042522768304914743,
+ "loss": 1.3747,
+ "theoretical_loss": 3.5317020750428143,
+ "tokens_seen": 1563820032
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042521163490471414,
+ "loss": 1.2321,
+ "theoretical_loss": 3.531689234246747,
+ "tokens_seen": 1563885568
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004251955867602809,
+ "loss": 1.2014,
+ "theoretical_loss": 3.531676394139436,
+ "tokens_seen": 1563951104
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042517953861584754,
+ "loss": 1.265,
+ "theoretical_loss": 3.531663554720814,
+ "tokens_seen": 1564016640
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004251634904714143,
+ "loss": 1.2123,
+ "theoretical_loss": 3.531650715990817,
+ "tokens_seen": 1564082176
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042514744232698094,
+ "loss": 1.35,
+ "theoretical_loss": 3.531637877949378,
+ "tokens_seen": 1564147712
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004251313941825477,
+ "loss": 1.2883,
+ "theoretical_loss": 3.5316250405964316,
+ "tokens_seen": 1564213248
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042511534603811434,
+ "loss": 1.2547,
+ "theoretical_loss": 3.531612203931912,
+ "tokens_seen": 1564278784
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004250992978936811,
+ "loss": 1.2528,
+ "theoretical_loss": 3.531599367955754,
+ "tokens_seen": 1564344320
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042508324974924774,
+ "loss": 1.4118,
+ "theoretical_loss": 3.5315865326678906,
+ "tokens_seen": 1564409856
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004250672016048145,
+ "loss": 1.4012,
+ "theoretical_loss": 3.531573698068257,
+ "tokens_seen": 1564475392
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042505115346038114,
+ "loss": 1.2908,
+ "theoretical_loss": 3.531560864156787,
+ "tokens_seen": 1564540928
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004250351053159479,
+ "loss": 1.3058,
+ "theoretical_loss": 3.5315480309334157,
+ "tokens_seen": 1564606464
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042501905717151454,
+ "loss": 1.313,
+ "theoretical_loss": 3.5315351983980765,
+ "tokens_seen": 1564672000
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004250030090270813,
+ "loss": 1.3286,
+ "theoretical_loss": 3.5315223665507034,
+ "tokens_seen": 1564737536
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042498696088264794,
+ "loss": 1.2794,
+ "theoretical_loss": 3.531509535391232,
+ "tokens_seen": 1564803072
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004249709127382147,
+ "loss": 1.342,
+ "theoretical_loss": 3.531496704919596,
+ "tokens_seen": 1564868608
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042495486459378135,
+ "loss": 1.301,
+ "theoretical_loss": 3.5314838751357294,
+ "tokens_seen": 1564934144
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004249388164493481,
+ "loss": 1.2521,
+ "theoretical_loss": 3.5314710460395666,
+ "tokens_seen": 1564999680
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042492276830491475,
+ "loss": 1.2948,
+ "theoretical_loss": 3.5314582176310427,
+ "tokens_seen": 1565065216
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004249067201604815,
+ "loss": 1.3802,
+ "theoretical_loss": 3.5314453899100906,
+ "tokens_seen": 1565130752
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042489067201604815,
+ "loss": 1.2886,
+ "theoretical_loss": 3.5314325628766463,
+ "tokens_seen": 1565196288
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004248746238716149,
+ "loss": 1.3553,
+ "theoretical_loss": 3.5314197365306432,
+ "tokens_seen": 1565261824
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042485857572718155,
+ "loss": 1.3395,
+ "theoretical_loss": 3.5314069108720156,
+ "tokens_seen": 1565327360
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004248425275827483,
+ "loss": 1.2585,
+ "theoretical_loss": 3.5313940859006983,
+ "tokens_seen": 1565392896
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042482647943831495,
+ "loss": 1.309,
+ "theoretical_loss": 3.531381261616626,
+ "tokens_seen": 1565458432
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004248104312938817,
+ "loss": 1.282,
+ "theoretical_loss": 3.5313684380197317,
+ "tokens_seen": 1565523968
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042479438314944835,
+ "loss": 1.3635,
+ "theoretical_loss": 3.5313556151099514,
+ "tokens_seen": 1565589504
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004247783350050151,
+ "loss": 1.3329,
+ "theoretical_loss": 3.5313427928872185,
+ "tokens_seen": 1565655040
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042476228686058175,
+ "loss": 1.3536,
+ "theoretical_loss": 3.5313299713514685,
+ "tokens_seen": 1565720576
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004247462387161485,
+ "loss": 1.2668,
+ "theoretical_loss": 3.531317150502635,
+ "tokens_seen": 1565786112
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042473019057171515,
+ "loss": 1.2595,
+ "theoretical_loss": 3.531304330340652,
+ "tokens_seen": 1565851648
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004247141424272819,
+ "loss": 1.3059,
+ "theoretical_loss": 3.5312915108654552,
+ "tokens_seen": 1565917184
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042469809428284856,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5312786920769783,
+ "tokens_seen": 1565982720
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004246820461384153,
+ "loss": 1.3542,
+ "theoretical_loss": 3.5312658739751557,
+ "tokens_seen": 1566048256
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042466599799398196,
+ "loss": 1.3757,
+ "theoretical_loss": 3.5312530565599225,
+ "tokens_seen": 1566113792
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004246499498495487,
+ "loss": 1.2794,
+ "theoretical_loss": 3.531240239831212,
+ "tokens_seen": 1566179328
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042463390170511536,
+ "loss": 1.2843,
+ "theoretical_loss": 3.5312274237889607,
+ "tokens_seen": 1566244864
+ },
+ {
+ "epoch": 0.47,
+ "objective/train/docs_used": 524888,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4155240058898926,
+ "objective/train/theoretical_loss": 3.531214608433101,
+ "objective/train/tokens_used": 1586770400,
+ "theoretical_loss": 3.531214608433101,
+ "tokens_seen": 1566310400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004246178535606821,
+ "loss": 1.3172,
+ "theoretical_loss": 3.531214608433101,
+ "tokens_seen": 1566310400
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042460180541624876,
+ "loss": 1.2831,
+ "theoretical_loss": 3.531201793763569,
+ "tokens_seen": 1566375936
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004245857572718155,
+ "loss": 1.3451,
+ "theoretical_loss": 3.531188979780298,
+ "tokens_seen": 1566441472
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042456970912738216,
+ "loss": 1.2661,
+ "theoretical_loss": 3.531176166483223,
+ "tokens_seen": 1566507008
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042455366098294886,
+ "loss": 1.3355,
+ "theoretical_loss": 3.5311633538722793,
+ "tokens_seen": 1566572544
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042453761283851556,
+ "loss": 1.3159,
+ "theoretical_loss": 3.5311505419474005,
+ "tokens_seen": 1566638080
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042452156469408226,
+ "loss": 1.2903,
+ "theoretical_loss": 3.5311377307085214,
+ "tokens_seen": 1566703616
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042450551654964896,
+ "loss": 1.1181,
+ "theoretical_loss": 3.5311249201555768,
+ "tokens_seen": 1566769152
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042448946840521566,
+ "loss": 1.2878,
+ "theoretical_loss": 3.5311121102885012,
+ "tokens_seen": 1566834688
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042447342026078236,
+ "loss": 1.37,
+ "theoretical_loss": 3.531099301107229,
+ "tokens_seen": 1566900224
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042445737211634907,
+ "loss": 1.3192,
+ "theoretical_loss": 3.5310864926116947,
+ "tokens_seen": 1566965760
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004244413239719157,
+ "loss": 1.2813,
+ "theoretical_loss": 3.5310736848018336,
+ "tokens_seen": 1567031296
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042442527582748247,
+ "loss": 1.2733,
+ "theoretical_loss": 3.5310608776775796,
+ "tokens_seen": 1567096832
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004244092276830491,
+ "loss": 1.2068,
+ "theoretical_loss": 3.531048071238868,
+ "tokens_seen": 1567162368
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042439317953861587,
+ "loss": 1.3221,
+ "theoretical_loss": 3.5310352654856327,
+ "tokens_seen": 1567227904
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004243771313941825,
+ "loss": 1.2494,
+ "theoretical_loss": 3.5310224604178084,
+ "tokens_seen": 1567293440
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042436108324974927,
+ "loss": 1.4405,
+ "theoretical_loss": 3.5310096560353306,
+ "tokens_seen": 1567358976
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.0004243450351053159,
+ "loss": 1.2756,
+ "theoretical_loss": 3.530996852338133,
+ "tokens_seen": 1567424512
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 0.00042432898696088267,
+ "loss": 1.299,
+ "theoretical_loss": 3.530984049326151,
+ "tokens_seen": 1567490048
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004243129388164493,
+ "loss": 1.2588,
+ "theoretical_loss": 3.530971246999319,
+ "tokens_seen": 1567555584
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042429689067201607,
+ "loss": 1.2105,
+ "theoretical_loss": 3.5309584453575713,
+ "tokens_seen": 1567621120
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004242808425275827,
+ "loss": 1.2394,
+ "theoretical_loss": 3.530945644400843,
+ "tokens_seen": 1567686656
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042426479438314947,
+ "loss": 1.3263,
+ "theoretical_loss": 3.530932844129069,
+ "tokens_seen": 1567752192
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004242487462387161,
+ "loss": 1.3468,
+ "theoretical_loss": 3.5309200445421833,
+ "tokens_seen": 1567817728
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004242326980942829,
+ "loss": 1.3322,
+ "theoretical_loss": 3.5309072456401216,
+ "tokens_seen": 1567883264
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004242166499498495,
+ "loss": 1.1928,
+ "theoretical_loss": 3.530894447422818,
+ "tokens_seen": 1567948800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004242006018054163,
+ "loss": 1.3663,
+ "theoretical_loss": 3.5308816498902074,
+ "tokens_seen": 1568014336
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241845536609829,
+ "loss": 1.2905,
+ "theoretical_loss": 3.5308688530422243,
+ "tokens_seen": 1568079872
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241685055165497,
+ "loss": 1.3351,
+ "theoretical_loss": 3.5308560568788048,
+ "tokens_seen": 1568145408
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241524573721163,
+ "loss": 1.2627,
+ "theoretical_loss": 3.5308432613998812,
+ "tokens_seen": 1568210944
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241364092276831,
+ "loss": 1.4045,
+ "theoretical_loss": 3.5308304666053907,
+ "tokens_seen": 1568276480
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241203610832497,
+ "loss": 1.3299,
+ "theoretical_loss": 3.5308176724952665,
+ "tokens_seen": 1568342016
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004241043129388165,
+ "loss": 1.3265,
+ "theoretical_loss": 3.5308048790694437,
+ "tokens_seen": 1568407552
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004240882647943831,
+ "loss": 1.1841,
+ "theoretical_loss": 3.5307920863278577,
+ "tokens_seen": 1568473088
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004240722166499499,
+ "loss": 1.2533,
+ "theoretical_loss": 3.5307792942704435,
+ "tokens_seen": 1568538624
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004240561685055165,
+ "loss": 1.4414,
+ "theoretical_loss": 3.5307665028971345,
+ "tokens_seen": 1568604160
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004240401203610833,
+ "loss": 1.2273,
+ "theoretical_loss": 3.530753712207867,
+ "tokens_seen": 1568669696
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042402407221664993,
+ "loss": 1.2805,
+ "theoretical_loss": 3.5307409222025754,
+ "tokens_seen": 1568735232
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004240080240722167,
+ "loss": 1.2144,
+ "theoretical_loss": 3.530728132881194,
+ "tokens_seen": 1568800768
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042399197592778344,
+ "loss": 1.3788,
+ "theoretical_loss": 3.5307153442436583,
+ "tokens_seen": 1568866304
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004239759277833501,
+ "loss": 1.254,
+ "theoretical_loss": 3.530702556289903,
+ "tokens_seen": 1568931840
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042395987963891684,
+ "loss": 1.3153,
+ "theoretical_loss": 3.530689769019863,
+ "tokens_seen": 1568997376
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004239438314944835,
+ "loss": 1.4061,
+ "theoretical_loss": 3.530676982433473,
+ "tokens_seen": 1569062912
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004239277833500502,
+ "loss": 1.3393,
+ "theoretical_loss": 3.530664196530668,
+ "tokens_seen": 1569128448
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004239117352056169,
+ "loss": 1.3325,
+ "theoretical_loss": 3.5306514113113834,
+ "tokens_seen": 1569193984
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004238956870611836,
+ "loss": 1.3402,
+ "theoretical_loss": 3.5306386267755534,
+ "tokens_seen": 1569259520
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004238796389167503,
+ "loss": 1.2606,
+ "theoretical_loss": 3.530625842923113,
+ "tokens_seen": 1569325056
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423863590772317,
+ "loss": 1.1971,
+ "theoretical_loss": 3.5306130597539975,
+ "tokens_seen": 1569390592
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004238475426278837,
+ "loss": 1.435,
+ "theoretical_loss": 3.5306002772681415,
+ "tokens_seen": 1569456128
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004238314944834504,
+ "loss": 1.3341,
+ "theoretical_loss": 3.530587495465481,
+ "tokens_seen": 1569521664
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 526097,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2334527969360352,
+ "objective/train/theoretical_loss": 3.5305747143459487,
+ "objective/train/tokens_used": 1590047200,
+ "theoretical_loss": 3.5305747143459487,
+ "tokens_seen": 1569587200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004238154463390171,
+ "loss": 1.2915,
+ "theoretical_loss": 3.5305747143459487,
+ "tokens_seen": 1569587200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004237993981945838,
+ "loss": 1.262,
+ "theoretical_loss": 3.530561933909482,
+ "tokens_seen": 1569652736
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042378335005015044,
+ "loss": 1.3578,
+ "theoretical_loss": 3.530549154156014,
+ "tokens_seen": 1569718272
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004237673019057172,
+ "loss": 1.2499,
+ "theoretical_loss": 3.5305363750854806,
+ "tokens_seen": 1569783808
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042375125376128384,
+ "loss": 1.3204,
+ "theoretical_loss": 3.5305235966978175,
+ "tokens_seen": 1569849344
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004237352056168506,
+ "loss": 1.2857,
+ "theoretical_loss": 3.5305108189929584,
+ "tokens_seen": 1569914880
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042371915747241724,
+ "loss": 1.2165,
+ "theoretical_loss": 3.530498041970839,
+ "tokens_seen": 1569980416
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423703109327984,
+ "loss": 1.2779,
+ "theoretical_loss": 3.530485265631394,
+ "tokens_seen": 1570045952
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042368706118355064,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5304724899745583,
+ "tokens_seen": 1570111488
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004236710130391174,
+ "loss": 1.272,
+ "theoretical_loss": 3.5304597150002675,
+ "tokens_seen": 1570177024
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042365496489468404,
+ "loss": 1.2711,
+ "theoretical_loss": 3.5304469407084564,
+ "tokens_seen": 1570242560
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004236389167502508,
+ "loss": 1.3257,
+ "theoretical_loss": 3.5304341670990596,
+ "tokens_seen": 1570308096
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042362286860581744,
+ "loss": 1.274,
+ "theoretical_loss": 3.5304213941720133,
+ "tokens_seen": 1570373632
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004236068204613842,
+ "loss": 1.361,
+ "theoretical_loss": 3.5304086219272515,
+ "tokens_seen": 1570439168
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042359077231695085,
+ "loss": 1.2634,
+ "theoretical_loss": 3.5303958503647097,
+ "tokens_seen": 1570504704
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004235747241725176,
+ "loss": 1.3646,
+ "theoretical_loss": 3.5303830794843227,
+ "tokens_seen": 1570570240
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042355867602808425,
+ "loss": 1.2974,
+ "theoretical_loss": 3.530370309286026,
+ "tokens_seen": 1570635776
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423542627883651,
+ "loss": 1.2879,
+ "theoretical_loss": 3.5303575397697546,
+ "tokens_seen": 1570701312
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042352657973921765,
+ "loss": 1.3977,
+ "theoretical_loss": 3.530344770935444,
+ "tokens_seen": 1570766848
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004235105315947844,
+ "loss": 1.2272,
+ "theoretical_loss": 3.5303320027830285,
+ "tokens_seen": 1570832384
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042349448345035105,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5303192353124437,
+ "tokens_seen": 1570897920
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004234784353059178,
+ "loss": 1.3116,
+ "theoretical_loss": 3.5303064685236243,
+ "tokens_seen": 1570963456
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042346238716148445,
+ "loss": 1.3329,
+ "theoretical_loss": 3.5302937024165058,
+ "tokens_seen": 1571028992
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004234463390170512,
+ "loss": 1.426,
+ "theoretical_loss": 3.530280936991024,
+ "tokens_seen": 1571094528
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042343029087261785,
+ "loss": 1.4062,
+ "theoretical_loss": 3.5302681722471134,
+ "tokens_seen": 1571160064
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004234142427281846,
+ "loss": 1.2104,
+ "theoretical_loss": 3.530255408184709,
+ "tokens_seen": 1571225600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042339819458375125,
+ "loss": 1.3025,
+ "theoretical_loss": 3.5302426448037467,
+ "tokens_seen": 1571291136
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423382146439318,
+ "loss": 1.3241,
+ "theoretical_loss": 3.5302298821041607,
+ "tokens_seen": 1571356672
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042336609829488465,
+ "loss": 1.3077,
+ "theoretical_loss": 3.5302171200858874,
+ "tokens_seen": 1571422208
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004233500501504514,
+ "loss": 1.3636,
+ "theoretical_loss": 3.5302043587488607,
+ "tokens_seen": 1571487744
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042333400200601806,
+ "loss": 1.2708,
+ "theoretical_loss": 3.530191598093017,
+ "tokens_seen": 1571553280
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004233179538615848,
+ "loss": 1.4087,
+ "theoretical_loss": 3.5301788381182906,
+ "tokens_seen": 1571618816
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042330190571715146,
+ "loss": 1.2877,
+ "theoretical_loss": 3.5301660788246174,
+ "tokens_seen": 1571684352
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004232858575727182,
+ "loss": 1.419,
+ "theoretical_loss": 3.530153320211933,
+ "tokens_seen": 1571749888
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042326980942828486,
+ "loss": 1.2931,
+ "theoretical_loss": 3.530140562280171,
+ "tokens_seen": 1571815424
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004232537612838516,
+ "loss": 1.2951,
+ "theoretical_loss": 3.530127805029269,
+ "tokens_seen": 1571880960
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042323771313941826,
+ "loss": 1.2114,
+ "theoretical_loss": 3.53011504845916,
+ "tokens_seen": 1571946496
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423221664994985,
+ "loss": 1.2458,
+ "theoretical_loss": 3.530102292569781,
+ "tokens_seen": 1572012032
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042320561685055166,
+ "loss": 1.2399,
+ "theoretical_loss": 3.5300895373610666,
+ "tokens_seen": 1572077568
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004231895687061184,
+ "loss": 1.3002,
+ "theoretical_loss": 3.5300767828329516,
+ "tokens_seen": 1572143104
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042317352056168506,
+ "loss": 1.268,
+ "theoretical_loss": 3.5300640289853726,
+ "tokens_seen": 1572208640
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042315747241725176,
+ "loss": 1.4284,
+ "theoretical_loss": 3.5300512758182636,
+ "tokens_seen": 1572274176
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042314142427281846,
+ "loss": 1.3799,
+ "theoretical_loss": 3.530038523331561,
+ "tokens_seen": 1572339712
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042312537612838516,
+ "loss": 1.2414,
+ "theoretical_loss": 3.5300257715251995,
+ "tokens_seen": 1572405248
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042310932798395186,
+ "loss": 1.3376,
+ "theoretical_loss": 3.5300130203991147,
+ "tokens_seen": 1572470784
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042309327983951856,
+ "loss": 1.4151,
+ "theoretical_loss": 3.5300002699532413,
+ "tokens_seen": 1572536320
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042307723169508527,
+ "loss": 1.3123,
+ "theoretical_loss": 3.5299875201875155,
+ "tokens_seen": 1572601856
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042306118355065197,
+ "loss": 1.3207,
+ "theoretical_loss": 3.529974771101873,
+ "tokens_seen": 1572667392
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042304513540621867,
+ "loss": 1.3893,
+ "theoretical_loss": 3.529962022696248,
+ "tokens_seen": 1572732928
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042302908726178537,
+ "loss": 1.2399,
+ "theoretical_loss": 3.5299492749705768,
+ "tokens_seen": 1572798464
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 527076,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3733115196228027,
+ "objective/train/theoretical_loss": 3.5299365279247943,
+ "objective/train/tokens_used": 1593324000,
+ "theoretical_loss": 3.5299365279247943,
+ "tokens_seen": 1572864000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000423013039117352,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5299365279247943,
+ "tokens_seen": 1572864000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042299699097291877,
+ "loss": 1.2936,
+ "theoretical_loss": 3.529923781558836,
+ "tokens_seen": 1572929536
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004229809428284854,
+ "loss": 1.3017,
+ "theoretical_loss": 3.5299110358726375,
+ "tokens_seen": 1572995072
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042296489468405217,
+ "loss": 1.4578,
+ "theoretical_loss": 3.5298982908661345,
+ "tokens_seen": 1573060608
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004229488465396188,
+ "loss": 1.2904,
+ "theoretical_loss": 3.529885546539262,
+ "tokens_seen": 1573126144
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042293279839518557,
+ "loss": 1.2763,
+ "theoretical_loss": 3.5298728028919557,
+ "tokens_seen": 1573191680
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004229167502507522,
+ "loss": 1.3212,
+ "theoretical_loss": 3.5298600599241503,
+ "tokens_seen": 1573257216
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042290070210631897,
+ "loss": 1.2949,
+ "theoretical_loss": 3.5298473176357823,
+ "tokens_seen": 1573322752
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004228846539618856,
+ "loss": 1.3182,
+ "theoretical_loss": 3.5298345760267864,
+ "tokens_seen": 1573388288
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004228686058174524,
+ "loss": 1.2748,
+ "theoretical_loss": 3.529821835097099,
+ "tokens_seen": 1573453824
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000422852557673019,
+ "loss": 1.2204,
+ "theoretical_loss": 3.5298090948466543,
+ "tokens_seen": 1573519360
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004228365095285858,
+ "loss": 1.347,
+ "theoretical_loss": 3.5297963552753893,
+ "tokens_seen": 1573584896
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042282046138415253,
+ "loss": 1.2267,
+ "theoretical_loss": 3.5297836163832383,
+ "tokens_seen": 1573650432
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004228044132397192,
+ "loss": 1.275,
+ "theoretical_loss": 3.529770878170137,
+ "tokens_seen": 1573715968
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042278836509528593,
+ "loss": 1.2845,
+ "theoretical_loss": 3.529758140636022,
+ "tokens_seen": 1573781504
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004227723169508526,
+ "loss": 1.2745,
+ "theoretical_loss": 3.5297454037808276,
+ "tokens_seen": 1573847040
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042275626880641933,
+ "loss": 1.2914,
+ "theoretical_loss": 3.529732667604489,
+ "tokens_seen": 1573912576
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000422740220661986,
+ "loss": 1.2849,
+ "theoretical_loss": 3.5297199321069437,
+ "tokens_seen": 1573978112
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042272417251755273,
+ "loss": 1.2802,
+ "theoretical_loss": 3.5297071972881255,
+ "tokens_seen": 1574043648
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004227081243731194,
+ "loss": 1.2531,
+ "theoretical_loss": 3.5296944631479703,
+ "tokens_seen": 1574109184
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042269207622868614,
+ "loss": 1.2373,
+ "theoretical_loss": 3.5296817296864145,
+ "tokens_seen": 1574174720
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004226760280842528,
+ "loss": 1.3647,
+ "theoretical_loss": 3.5296689969033928,
+ "tokens_seen": 1574240256
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042265997993981954,
+ "loss": 1.2165,
+ "theoretical_loss": 3.5296562647988408,
+ "tokens_seen": 1574305792
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004226439317953862,
+ "loss": 1.3965,
+ "theoretical_loss": 3.5296435333726945,
+ "tokens_seen": 1574371328
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042262788365095294,
+ "loss": 1.2745,
+ "theoretical_loss": 3.52963080262489,
+ "tokens_seen": 1574436864
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004226118355065196,
+ "loss": 1.3204,
+ "theoretical_loss": 3.529618072555362,
+ "tokens_seen": 1574502400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042259578736208634,
+ "loss": 1.3651,
+ "theoretical_loss": 3.5296053431640466,
+ "tokens_seen": 1574567936
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000422579739217653,
+ "loss": 1.2688,
+ "theoretical_loss": 3.529592614450879,
+ "tokens_seen": 1574633472
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042256369107321974,
+ "loss": 1.302,
+ "theoretical_loss": 3.5295798864157955,
+ "tokens_seen": 1574699008
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004225476429287864,
+ "loss": 1.2396,
+ "theoretical_loss": 3.529567159058731,
+ "tokens_seen": 1574764544
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004225315947843531,
+ "loss": 1.2741,
+ "theoretical_loss": 3.529554432379622,
+ "tokens_seen": 1574830080
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004225155466399198,
+ "loss": 1.3089,
+ "theoretical_loss": 3.529541706378404,
+ "tokens_seen": 1574895616
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224994984954865,
+ "loss": 1.2228,
+ "theoretical_loss": 3.5295289810550123,
+ "tokens_seen": 1574961152
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224834503510532,
+ "loss": 1.2698,
+ "theoretical_loss": 3.5295162564093827,
+ "tokens_seen": 1575026688
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224674022066199,
+ "loss": 1.2509,
+ "theoretical_loss": 3.5295035324414505,
+ "tokens_seen": 1575092224
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224513540621866,
+ "loss": 1.2494,
+ "theoretical_loss": 3.5294908091511523,
+ "tokens_seen": 1575157760
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224353059177533,
+ "loss": 1.3338,
+ "theoretical_loss": 3.5294780865384237,
+ "tokens_seen": 1575223296
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042241925777332,
+ "loss": 1.3868,
+ "theoretical_loss": 3.5294653646031993,
+ "tokens_seen": 1575288832
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004224032096288867,
+ "loss": 1.305,
+ "theoretical_loss": 3.529452643345416,
+ "tokens_seen": 1575354368
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042238716148445334,
+ "loss": 1.2398,
+ "theoretical_loss": 3.5294399227650097,
+ "tokens_seen": 1575419904
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004223711133400201,
+ "loss": 1.3808,
+ "theoretical_loss": 3.5294272028619154,
+ "tokens_seen": 1575485440
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042235506519558674,
+ "loss": 1.3114,
+ "theoretical_loss": 3.529414483636069,
+ "tokens_seen": 1575550976
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004223390170511535,
+ "loss": 1.2883,
+ "theoretical_loss": 3.5294017650874063,
+ "tokens_seen": 1575616512
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042232296890672014,
+ "loss": 1.3171,
+ "theoretical_loss": 3.5293890472158633,
+ "tokens_seen": 1575682048
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004223069207622869,
+ "loss": 1.3048,
+ "theoretical_loss": 3.5293763300213756,
+ "tokens_seen": 1575747584
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042229087261785354,
+ "loss": 1.2641,
+ "theoretical_loss": 3.5293636135038793,
+ "tokens_seen": 1575813120
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004222748244734203,
+ "loss": 1.2291,
+ "theoretical_loss": 3.52935089766331,
+ "tokens_seen": 1575878656
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042225877632898694,
+ "loss": 1.3643,
+ "theoretical_loss": 3.529338182499603,
+ "tokens_seen": 1575944192
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004222427281845537,
+ "loss": 1.2591,
+ "theoretical_loss": 3.5293254680126953,
+ "tokens_seen": 1576009728
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042222668004012035,
+ "loss": 1.3474,
+ "theoretical_loss": 3.5293127542025213,
+ "tokens_seen": 1576075264
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 528401,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.289647102355957,
+ "objective/train/theoretical_loss": 3.529300041069018,
+ "objective/train/tokens_used": 1596600800,
+ "theoretical_loss": 3.529300041069018,
+ "tokens_seen": 1576140800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004222106318956871,
+ "loss": 1.3653,
+ "theoretical_loss": 3.529300041069018,
+ "tokens_seen": 1576140800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042219458375125375,
+ "loss": 1.2696,
+ "theoretical_loss": 3.5292873286121207,
+ "tokens_seen": 1576206336
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004221785356068205,
+ "loss": 1.3086,
+ "theoretical_loss": 3.5292746168317657,
+ "tokens_seen": 1576271872
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042216248746238715,
+ "loss": 1.2939,
+ "theoretical_loss": 3.5292619057278887,
+ "tokens_seen": 1576337408
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004221464393179539,
+ "loss": 1.3272,
+ "theoretical_loss": 3.529249195300425,
+ "tokens_seen": 1576402944
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042213039117352055,
+ "loss": 1.3479,
+ "theoretical_loss": 3.529236485549311,
+ "tokens_seen": 1576468480
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004221143430290873,
+ "loss": 1.3035,
+ "theoretical_loss": 3.529223776474483,
+ "tokens_seen": 1576534016
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042209829488465395,
+ "loss": 1.2684,
+ "theoretical_loss": 3.5292110680758757,
+ "tokens_seen": 1576599552
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004220822467402207,
+ "loss": 1.2601,
+ "theoretical_loss": 3.5291983603534263,
+ "tokens_seen": 1576665088
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042206619859578735,
+ "loss": 1.274,
+ "theoretical_loss": 3.52918565330707,
+ "tokens_seen": 1576730624
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004220501504513541,
+ "loss": 1.2333,
+ "theoretical_loss": 3.529172946936743,
+ "tokens_seen": 1576796160
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042203410230692075,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5291602412423813,
+ "tokens_seen": 1576861696
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004220180541624875,
+ "loss": 1.3126,
+ "theoretical_loss": 3.5291475362239204,
+ "tokens_seen": 1576927232
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042200200601805415,
+ "loss": 1.3129,
+ "theoretical_loss": 3.5291348318812963,
+ "tokens_seen": 1576992768
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004219859578736209,
+ "loss": 1.1857,
+ "theoretical_loss": 3.5291221282144454,
+ "tokens_seen": 1577058304
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042196990972918756,
+ "loss": 1.3501,
+ "theoretical_loss": 3.5291094252233037,
+ "tokens_seen": 1577123840
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004219538615847543,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5290967229078074,
+ "tokens_seen": 1577189376
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042193781344032096,
+ "loss": 1.274,
+ "theoretical_loss": 3.5290840212678916,
+ "tokens_seen": 1577254912
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004219217652958877,
+ "loss": 1.2622,
+ "theoretical_loss": 3.5290713203034922,
+ "tokens_seen": 1577320448
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042190571715145436,
+ "loss": 1.1857,
+ "theoretical_loss": 3.5290586200145464,
+ "tokens_seen": 1577385984
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004218896690070211,
+ "loss": 1.2918,
+ "theoretical_loss": 3.5290459204009896,
+ "tokens_seen": 1577451520
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042187362086258776,
+ "loss": 1.2541,
+ "theoretical_loss": 3.5290332214627576,
+ "tokens_seen": 1577517056
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004218575727181545,
+ "loss": 1.3235,
+ "theoretical_loss": 3.5290205231997867,
+ "tokens_seen": 1577582592
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042184152457372116,
+ "loss": 1.3156,
+ "theoretical_loss": 3.5290078256120134,
+ "tokens_seen": 1577648128
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004218254764292879,
+ "loss": 1.3458,
+ "theoretical_loss": 3.5289951286993726,
+ "tokens_seen": 1577713664
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042180942828485456,
+ "loss": 1.3338,
+ "theoretical_loss": 3.528982432461801,
+ "tokens_seen": 1577779200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004217933801404213,
+ "loss": 1.3097,
+ "theoretical_loss": 3.528969736899235,
+ "tokens_seen": 1577844736
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042177733199598796,
+ "loss": 1.2871,
+ "theoretical_loss": 3.5289570420116103,
+ "tokens_seen": 1577910272
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042176128385155466,
+ "loss": 1.3468,
+ "theoretical_loss": 3.5289443477988627,
+ "tokens_seen": 1577975808
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042174523570712136,
+ "loss": 1.3533,
+ "theoretical_loss": 3.528931654260929,
+ "tokens_seen": 1578041344
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042172918756268806,
+ "loss": 1.2591,
+ "theoretical_loss": 3.5289189613977454,
+ "tokens_seen": 1578106880
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042171313941825477,
+ "loss": 1.2751,
+ "theoretical_loss": 3.528906269209247,
+ "tokens_seen": 1578172416
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042169709127382147,
+ "loss": 1.2951,
+ "theoretical_loss": 3.5288935776953703,
+ "tokens_seen": 1578237952
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042168104312938817,
+ "loss": 1.3469,
+ "theoretical_loss": 3.5288808868560517,
+ "tokens_seen": 1578303488
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042166499498495487,
+ "loss": 1.3073,
+ "theoretical_loss": 3.5288681966912274,
+ "tokens_seen": 1578369024
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042164894684052157,
+ "loss": 1.2532,
+ "theoretical_loss": 3.5288555072008334,
+ "tokens_seen": 1578434560
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042163289869608827,
+ "loss": 1.2275,
+ "theoretical_loss": 3.528842818384806,
+ "tokens_seen": 1578500096
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000421616850551655,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5288301302430813,
+ "tokens_seen": 1578565632
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042160080240722167,
+ "loss": 1.2366,
+ "theoretical_loss": 3.528817442775595,
+ "tokens_seen": 1578631168
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004215847542627884,
+ "loss": 1.297,
+ "theoretical_loss": 3.528804755982284,
+ "tokens_seen": 1578696704
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042156870611835507,
+ "loss": 1.1885,
+ "theoretical_loss": 3.5287920698630835,
+ "tokens_seen": 1578762240
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004215526579739218,
+ "loss": 1.3305,
+ "theoretical_loss": 3.5287793844179314,
+ "tokens_seen": 1578827776
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042153660982948847,
+ "loss": 1.3154,
+ "theoretical_loss": 3.528766699646762,
+ "tokens_seen": 1578893312
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042152056168505523,
+ "loss": 1.2767,
+ "theoretical_loss": 3.528754015549513,
+ "tokens_seen": 1578958848
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004215045135406219,
+ "loss": 1.2104,
+ "theoretical_loss": 3.5287413321261196,
+ "tokens_seen": 1579024384
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042148846539618863,
+ "loss": 1.3082,
+ "theoretical_loss": 3.528728649376519,
+ "tokens_seen": 1579089920
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004214724172517553,
+ "loss": 1.2154,
+ "theoretical_loss": 3.528715967300646,
+ "tokens_seen": 1579155456
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042145636910732203,
+ "loss": 1.3719,
+ "theoretical_loss": 3.5287032858984384,
+ "tokens_seen": 1579220992
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004214403209628887,
+ "loss": 1.3689,
+ "theoretical_loss": 3.5286906051698317,
+ "tokens_seen": 1579286528
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042142427281845543,
+ "loss": 1.3586,
+ "theoretical_loss": 3.5286779251147626,
+ "tokens_seen": 1579352064
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 529316,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2476038932800293,
+ "objective/train/theoretical_loss": 3.5286652457331664,
+ "objective/train/tokens_used": 1599877600,
+ "theoretical_loss": 3.5286652457331664,
+ "tokens_seen": 1579417600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004214082246740221,
+ "loss": 1.3399,
+ "theoretical_loss": 3.5286652457331664,
+ "tokens_seen": 1579417600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042139217652958883,
+ "loss": 1.2368,
+ "theoretical_loss": 3.528652567024981,
+ "tokens_seen": 1579483136
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004213761283851555,
+ "loss": 1.2149,
+ "theoretical_loss": 3.5286398889901407,
+ "tokens_seen": 1579548672
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042136008024072223,
+ "loss": 1.2429,
+ "theoretical_loss": 3.5286272116285833,
+ "tokens_seen": 1579614208
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004213440320962889,
+ "loss": 1.2743,
+ "theoretical_loss": 3.528614534940245,
+ "tokens_seen": 1579679744
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042132798395185563,
+ "loss": 1.2867,
+ "theoretical_loss": 3.5286018589250614,
+ "tokens_seen": 1579745280
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004213119358074223,
+ "loss": 1.1426,
+ "theoretical_loss": 3.5285891835829695,
+ "tokens_seen": 1579810816
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042129588766298904,
+ "loss": 1.2519,
+ "theoretical_loss": 3.528576508913905,
+ "tokens_seen": 1579876352
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004212798395185557,
+ "loss": 1.2552,
+ "theoretical_loss": 3.528563834917805,
+ "tokens_seen": 1579941888
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042126379137412244,
+ "loss": 1.3386,
+ "theoretical_loss": 3.528551161594605,
+ "tokens_seen": 1580007424
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004212477432296891,
+ "loss": 1.3024,
+ "theoretical_loss": 3.528538488944242,
+ "tokens_seen": 1580072960
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042123169508525584,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5285258169666522,
+ "tokens_seen": 1580138496
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004212156469408225,
+ "loss": 1.2856,
+ "theoretical_loss": 3.5285131456617718,
+ "tokens_seen": 1580204032
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042119959879638924,
+ "loss": 1.2508,
+ "theoretical_loss": 3.5285004750295377,
+ "tokens_seen": 1580269568
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004211835506519559,
+ "loss": 1.2427,
+ "theoretical_loss": 3.5284878050698865,
+ "tokens_seen": 1580335104
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042116750250752264,
+ "loss": 1.3434,
+ "theoretical_loss": 3.5284751357827533,
+ "tokens_seen": 1580400640
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004211514543630893,
+ "loss": 1.2483,
+ "theoretical_loss": 3.528462467168075,
+ "tokens_seen": 1580466176
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000421135406218656,
+ "loss": 1.2159,
+ "theoretical_loss": 3.5284497992257893,
+ "tokens_seen": 1580531712
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004211193580742227,
+ "loss": 1.2647,
+ "theoretical_loss": 3.528437131955831,
+ "tokens_seen": 1580597248
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004211033099297894,
+ "loss": 1.3201,
+ "theoretical_loss": 3.528424465358137,
+ "tokens_seen": 1580662784
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210872617853561,
+ "loss": 1.377,
+ "theoretical_loss": 3.5284117994326447,
+ "tokens_seen": 1580728320
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210712136409228,
+ "loss": 1.2718,
+ "theoretical_loss": 3.5283991341792893,
+ "tokens_seen": 1580793856
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210551654964895,
+ "loss": 1.2739,
+ "theoretical_loss": 3.5283864695980083,
+ "tokens_seen": 1580859392
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210391173520562,
+ "loss": 1.2492,
+ "theoretical_loss": 3.5283738056887373,
+ "tokens_seen": 1580924928
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210230692076229,
+ "loss": 1.249,
+ "theoretical_loss": 3.5283611424514127,
+ "tokens_seen": 1580990464
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004210070210631896,
+ "loss": 1.2172,
+ "theoretical_loss": 3.528348479885972,
+ "tokens_seen": 1581056000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042099097291875624,
+ "loss": 1.3031,
+ "theoretical_loss": 3.5283358179923514,
+ "tokens_seen": 1581121536
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000420974924774323,
+ "loss": 1.2905,
+ "theoretical_loss": 3.5283231567704867,
+ "tokens_seen": 1581187072
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042095887662988964,
+ "loss": 1.2849,
+ "theoretical_loss": 3.528310496220315,
+ "tokens_seen": 1581252608
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004209428284854564,
+ "loss": 1.2904,
+ "theoretical_loss": 3.5282978363417725,
+ "tokens_seen": 1581318144
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042092678034102304,
+ "loss": 1.3633,
+ "theoretical_loss": 3.528285177134796,
+ "tokens_seen": 1581383680
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004209107321965898,
+ "loss": 1.1911,
+ "theoretical_loss": 3.528272518599322,
+ "tokens_seen": 1581449216
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042089468405215644,
+ "loss": 1.3286,
+ "theoretical_loss": 3.5282598607352877,
+ "tokens_seen": 1581514752
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004208786359077232,
+ "loss": 1.2025,
+ "theoretical_loss": 3.5282472035426284,
+ "tokens_seen": 1581580288
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042086258776328984,
+ "loss": 1.3582,
+ "theoretical_loss": 3.528234547021281,
+ "tokens_seen": 1581645824
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004208465396188566,
+ "loss": 1.2729,
+ "theoretical_loss": 3.528221891171183,
+ "tokens_seen": 1581711360
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042083049147442325,
+ "loss": 1.4335,
+ "theoretical_loss": 3.52820923599227,
+ "tokens_seen": 1581776896
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042081444332999,
+ "loss": 1.252,
+ "theoretical_loss": 3.528196581484479,
+ "tokens_seen": 1581842432
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042079839518555665,
+ "loss": 1.3587,
+ "theoretical_loss": 3.5281839276477465,
+ "tokens_seen": 1581907968
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004207823470411234,
+ "loss": 1.2302,
+ "theoretical_loss": 3.5281712744820095,
+ "tokens_seen": 1581973504
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042076629889669005,
+ "loss": 1.2506,
+ "theoretical_loss": 3.5281586219872043,
+ "tokens_seen": 1582039040
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004207502507522568,
+ "loss": 1.3141,
+ "theoretical_loss": 3.5281459701632674,
+ "tokens_seen": 1582104576
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042073420260782345,
+ "loss": 1.3975,
+ "theoretical_loss": 3.528133319010135,
+ "tokens_seen": 1582170112
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004207181544633902,
+ "loss": 1.307,
+ "theoretical_loss": 3.528120668527745,
+ "tokens_seen": 1582235648
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042070210631895685,
+ "loss": 1.2699,
+ "theoretical_loss": 3.528108018716033,
+ "tokens_seen": 1582301184
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004206860581745236,
+ "loss": 1.249,
+ "theoretical_loss": 3.5280953695749364,
+ "tokens_seen": 1582366720
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042067001003009025,
+ "loss": 1.2198,
+ "theoretical_loss": 3.5280827211043917,
+ "tokens_seen": 1582432256
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000420653961885657,
+ "loss": 1.33,
+ "theoretical_loss": 3.5280700733043346,
+ "tokens_seen": 1582497792
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042063791374122365,
+ "loss": 1.3147,
+ "theoretical_loss": 3.528057426174703,
+ "tokens_seen": 1582563328
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004206218655967904,
+ "loss": 1.2517,
+ "theoretical_loss": 3.528044779715434,
+ "tokens_seen": 1582628864
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 530619,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1984028816223145,
+ "objective/train/theoretical_loss": 3.5280321339264624,
+ "objective/train/tokens_used": 1603154400,
+ "theoretical_loss": 3.5280321339264624,
+ "tokens_seen": 1582694400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042060581745235705,
+ "loss": 1.3051,
+ "theoretical_loss": 3.5280321339264624,
+ "tokens_seen": 1582694400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004205897693079238,
+ "loss": 1.3102,
+ "theoretical_loss": 3.5280194888077263,
+ "tokens_seen": 1582759936
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042057372116349046,
+ "loss": 1.2483,
+ "theoretical_loss": 3.5280068443591626,
+ "tokens_seen": 1582825472
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004205576730190572,
+ "loss": 1.4325,
+ "theoretical_loss": 3.5279942005807072,
+ "tokens_seen": 1582891008
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042054162487462386,
+ "loss": 1.2919,
+ "theoretical_loss": 3.5279815574722972,
+ "tokens_seen": 1582956544
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004205255767301906,
+ "loss": 1.3096,
+ "theoretical_loss": 3.5279689150338696,
+ "tokens_seen": 1583022080
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042050952858575726,
+ "loss": 1.3325,
+ "theoretical_loss": 3.527956273265361,
+ "tokens_seen": 1583087616
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000420493480441324,
+ "loss": 1.2714,
+ "theoretical_loss": 3.527943632166708,
+ "tokens_seen": 1583153152
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042047743229689066,
+ "loss": 1.2232,
+ "theoretical_loss": 3.527930991737848,
+ "tokens_seen": 1583218688
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004204613841524574,
+ "loss": 1.305,
+ "theoretical_loss": 3.527918351978717,
+ "tokens_seen": 1583284224
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042044533600802406,
+ "loss": 1.2146,
+ "theoretical_loss": 3.5279057128892513,
+ "tokens_seen": 1583349760
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004204292878635908,
+ "loss": 1.3379,
+ "theoretical_loss": 3.5278930744693895,
+ "tokens_seen": 1583415296
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004204132397191575,
+ "loss": 1.3599,
+ "theoretical_loss": 3.527880436719067,
+ "tokens_seen": 1583480832
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004203971915747242,
+ "loss": 1.2844,
+ "theoretical_loss": 3.527867799638221,
+ "tokens_seen": 1583546368
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004203811434302909,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5278551632267883,
+ "tokens_seen": 1583611904
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042036509528585756,
+ "loss": 1.2745,
+ "theoretical_loss": 3.5278425274847063,
+ "tokens_seen": 1583677440
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004203490471414243,
+ "loss": 1.386,
+ "theoretical_loss": 3.527829892411911,
+ "tokens_seen": 1583742976
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042033299899699097,
+ "loss": 1.3353,
+ "theoretical_loss": 3.5278172580083393,
+ "tokens_seen": 1583808512
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004203169508525577,
+ "loss": 1.2358,
+ "theoretical_loss": 3.5278046242739287,
+ "tokens_seen": 1583874048
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042030090270812437,
+ "loss": 1.2919,
+ "theoretical_loss": 3.5277919912086158,
+ "tokens_seen": 1583939584
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004202848545636911,
+ "loss": 1.4091,
+ "theoretical_loss": 3.5277793588123374,
+ "tokens_seen": 1584005120
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042026880641925777,
+ "loss": 1.2692,
+ "theoretical_loss": 3.52776672708503,
+ "tokens_seen": 1584070656
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004202527582748245,
+ "loss": 1.2129,
+ "theoretical_loss": 3.527754096026632,
+ "tokens_seen": 1584136192
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042023671013039117,
+ "loss": 1.3177,
+ "theoretical_loss": 3.527741465637078,
+ "tokens_seen": 1584201728
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004202206619859579,
+ "loss": 1.2905,
+ "theoretical_loss": 3.5277288359163066,
+ "tokens_seen": 1584267264
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042020461384152457,
+ "loss": 1.2648,
+ "theoretical_loss": 3.5277162068642545,
+ "tokens_seen": 1584332800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004201885656970913,
+ "loss": 1.2313,
+ "theoretical_loss": 3.5277035784808577,
+ "tokens_seen": 1584398336
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042017251755265797,
+ "loss": 1.313,
+ "theoretical_loss": 3.5276909507660545,
+ "tokens_seen": 1584463872
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004201564694082247,
+ "loss": 1.3639,
+ "theoretical_loss": 3.527678323719781,
+ "tokens_seen": 1584529408
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004201404212637914,
+ "loss": 1.2945,
+ "theoretical_loss": 3.527665697341974,
+ "tokens_seen": 1584594944
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042012437311935813,
+ "loss": 1.269,
+ "theoretical_loss": 3.527653071632572,
+ "tokens_seen": 1584660480
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004201083249749248,
+ "loss": 1.2947,
+ "theoretical_loss": 3.5276404465915094,
+ "tokens_seen": 1584726016
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042009227683049153,
+ "loss": 1.248,
+ "theoretical_loss": 3.5276278222187254,
+ "tokens_seen": 1584791552
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004200762286860582,
+ "loss": 1.252,
+ "theoretical_loss": 3.5276151985141557,
+ "tokens_seen": 1584857088
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042006018054162493,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5276025754777383,
+ "tokens_seen": 1584922624
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004200441323971916,
+ "loss": 1.2689,
+ "theoretical_loss": 3.527589953109409,
+ "tokens_seen": 1584988160
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00042002808425275833,
+ "loss": 1.2224,
+ "theoretical_loss": 3.527577331409106,
+ "tokens_seen": 1585053696
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000420012036108325,
+ "loss": 1.2175,
+ "theoretical_loss": 3.5275647103767653,
+ "tokens_seen": 1585119232
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041999598796389173,
+ "loss": 1.2662,
+ "theoretical_loss": 3.527552090012325,
+ "tokens_seen": 1585184768
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004199799398194584,
+ "loss": 1.1799,
+ "theoretical_loss": 3.5275394703157215,
+ "tokens_seen": 1585250304
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041996389167502513,
+ "loss": 1.2508,
+ "theoretical_loss": 3.5275268512868916,
+ "tokens_seen": 1585315840
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004199478435305918,
+ "loss": 1.2666,
+ "theoretical_loss": 3.5275142329257734,
+ "tokens_seen": 1585381376
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041993179538615854,
+ "loss": 1.2242,
+ "theoretical_loss": 3.527501615232303,
+ "tokens_seen": 1585446912
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004199157472417252,
+ "loss": 1.3351,
+ "theoretical_loss": 3.5274889982064175,
+ "tokens_seen": 1585512448
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041989969909729194,
+ "loss": 1.302,
+ "theoretical_loss": 3.5274763818480546,
+ "tokens_seen": 1585577984
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004198836509528586,
+ "loss": 1.3432,
+ "theoretical_loss": 3.527463766157151,
+ "tokens_seen": 1585643520
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041986760280842534,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5274511511336435,
+ "tokens_seen": 1585709056
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000419851554663992,
+ "loss": 1.2974,
+ "theoretical_loss": 3.5274385367774697,
+ "tokens_seen": 1585774592
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041983550651955874,
+ "loss": 1.3053,
+ "theoretical_loss": 3.5274259230885665,
+ "tokens_seen": 1585840128
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004198194583751254,
+ "loss": 1.3219,
+ "theoretical_loss": 3.527413310066872,
+ "tokens_seen": 1585905664
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 531508,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2401823997497559,
+ "objective/train/theoretical_loss": 3.5274006977123213,
+ "objective/train/tokens_used": 1606431200,
+ "theoretical_loss": 3.5274006977123213,
+ "tokens_seen": 1585971200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041980341023069214,
+ "loss": 1.2822,
+ "theoretical_loss": 3.5274006977123213,
+ "tokens_seen": 1585971200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004197873620862588,
+ "loss": 1.2111,
+ "theoretical_loss": 3.527388086024853,
+ "tokens_seen": 1586036736
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041977131394182554,
+ "loss": 1.3603,
+ "theoretical_loss": 3.527375475004404,
+ "tokens_seen": 1586102272
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004197552657973922,
+ "loss": 1.3557,
+ "theoretical_loss": 3.527362864650912,
+ "tokens_seen": 1586167808
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004197392176529589,
+ "loss": 1.3688,
+ "theoretical_loss": 3.527350254964313,
+ "tokens_seen": 1586233344
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004197231695085256,
+ "loss": 1.2819,
+ "theoretical_loss": 3.527337645944545,
+ "tokens_seen": 1586298880
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004197071213640923,
+ "loss": 1.3064,
+ "theoretical_loss": 3.527325037591545,
+ "tokens_seen": 1586364416
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000419691073219659,
+ "loss": 1.3509,
+ "theoretical_loss": 3.52731242990525,
+ "tokens_seen": 1586429952
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004196750250752257,
+ "loss": 1.2732,
+ "theoretical_loss": 3.527299822885597,
+ "tokens_seen": 1586495488
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004196589769307924,
+ "loss": 1.2875,
+ "theoretical_loss": 3.527287216532524,
+ "tokens_seen": 1586561024
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004196429287863591,
+ "loss": 1.3171,
+ "theoretical_loss": 3.527274610845968,
+ "tokens_seen": 1586626560
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004196268806419258,
+ "loss": 1.2842,
+ "theoretical_loss": 3.527262005825866,
+ "tokens_seen": 1586692096
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004196108324974925,
+ "loss": 1.2986,
+ "theoretical_loss": 3.5272494014721554,
+ "tokens_seen": 1586757632
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041959478435305914,
+ "loss": 1.2124,
+ "theoretical_loss": 3.5272367977847727,
+ "tokens_seen": 1586823168
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004195787362086259,
+ "loss": 1.4275,
+ "theoretical_loss": 3.5272241947636562,
+ "tokens_seen": 1586888704
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041956268806419254,
+ "loss": 1.3391,
+ "theoretical_loss": 3.527211592408743,
+ "tokens_seen": 1586954240
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004195466399197593,
+ "loss": 1.3117,
+ "theoretical_loss": 3.5271989907199695,
+ "tokens_seen": 1587019776
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041953059177532594,
+ "loss": 1.3273,
+ "theoretical_loss": 3.527186389697274,
+ "tokens_seen": 1587085312
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004195145436308927,
+ "loss": 1.3016,
+ "theoretical_loss": 3.5271737893405932,
+ "tokens_seen": 1587150848
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041949849548645934,
+ "loss": 1.1969,
+ "theoretical_loss": 3.5271611896498647,
+ "tokens_seen": 1587216384
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004194824473420261,
+ "loss": 1.2943,
+ "theoretical_loss": 3.5271485906250257,
+ "tokens_seen": 1587281920
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041946639919759275,
+ "loss": 1.2469,
+ "theoretical_loss": 3.5271359922660137,
+ "tokens_seen": 1587347456
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004194503510531595,
+ "loss": 1.2952,
+ "theoretical_loss": 3.527123394572765,
+ "tokens_seen": 1587412992
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041943430290872615,
+ "loss": 1.2892,
+ "theoretical_loss": 3.5271107975452187,
+ "tokens_seen": 1587478528
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004194182547642929,
+ "loss": 1.3364,
+ "theoretical_loss": 3.5270982011833114,
+ "tokens_seen": 1587544064
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041940220661985955,
+ "loss": 1.237,
+ "theoretical_loss": 3.5270856054869797,
+ "tokens_seen": 1587609600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004193861584754263,
+ "loss": 1.2828,
+ "theoretical_loss": 3.527073010456162,
+ "tokens_seen": 1587675136
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041937011033099295,
+ "loss": 1.3282,
+ "theoretical_loss": 3.5270604160907943,
+ "tokens_seen": 1587740672
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004193540621865597,
+ "loss": 1.4049,
+ "theoretical_loss": 3.5270478223908155,
+ "tokens_seen": 1587806208
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041933801404212635,
+ "loss": 1.3352,
+ "theoretical_loss": 3.5270352293561626,
+ "tokens_seen": 1587871744
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004193219658976931,
+ "loss": 1.2145,
+ "theoretical_loss": 3.5270226369867723,
+ "tokens_seen": 1587937280
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041930591775325975,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5270100452825828,
+ "tokens_seen": 1588002816
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004192898696088265,
+ "loss": 1.3382,
+ "theoretical_loss": 3.5269974542435314,
+ "tokens_seen": 1588068352
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041927382146439315,
+ "loss": 1.2923,
+ "theoretical_loss": 3.5269848638695547,
+ "tokens_seen": 1588133888
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004192577733199599,
+ "loss": 1.2993,
+ "theoretical_loss": 3.526972274160591,
+ "tokens_seen": 1588199424
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041924172517552655,
+ "loss": 1.2885,
+ "theoretical_loss": 3.5269596851165774,
+ "tokens_seen": 1588264960
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004192256770310933,
+ "loss": 1.2316,
+ "theoretical_loss": 3.526947096737451,
+ "tokens_seen": 1588330496
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041920962888666006,
+ "loss": 1.2396,
+ "theoretical_loss": 3.5269345090231505,
+ "tokens_seen": 1588396032
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004191935807422267,
+ "loss": 1.2591,
+ "theoretical_loss": 3.526921921973612,
+ "tokens_seen": 1588461568
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041917753259779347,
+ "loss": 1.2781,
+ "theoretical_loss": 3.5269093355887735,
+ "tokens_seen": 1588527104
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004191614844533601,
+ "loss": 1.2644,
+ "theoretical_loss": 3.5268967498685724,
+ "tokens_seen": 1588592640
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041914543630892687,
+ "loss": 1.356,
+ "theoretical_loss": 3.5268841648129463,
+ "tokens_seen": 1588658176
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004191293881644935,
+ "loss": 1.3601,
+ "theoretical_loss": 3.5268715804218322,
+ "tokens_seen": 1588723712
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041911334002006027,
+ "loss": 1.3441,
+ "theoretical_loss": 3.526858996695169,
+ "tokens_seen": 1588789248
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190972918756269,
+ "loss": 1.2989,
+ "theoretical_loss": 3.5268464136328923,
+ "tokens_seen": 1588854784
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190812437311936,
+ "loss": 1.311,
+ "theoretical_loss": 3.526833831234941,
+ "tokens_seen": 1588920320
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190651955867603,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5268212495012525,
+ "tokens_seen": 1588985856
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000419049147442327,
+ "loss": 1.2975,
+ "theoretical_loss": 3.5268086684317637,
+ "tokens_seen": 1589051392
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190330992978937,
+ "loss": 1.3467,
+ "theoretical_loss": 3.526796088026412,
+ "tokens_seen": 1589116928
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190170511534604,
+ "loss": 1.4015,
+ "theoretical_loss": 3.526783508285136,
+ "tokens_seen": 1589182464
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 532777,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3931702375411987,
+ "objective/train/theoretical_loss": 3.526770929207873,
+ "objective/train/tokens_used": 1609708000,
+ "theoretical_loss": 3.526770929207873,
+ "tokens_seen": 1589248000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004190010030090271,
+ "loss": 1.3721,
+ "theoretical_loss": 3.526770929207873,
+ "tokens_seen": 1589248000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004189849548645938,
+ "loss": 1.3258,
+ "theoretical_loss": 3.5267583507945597,
+ "tokens_seen": 1589313536
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041896890672016047,
+ "loss": 1.3597,
+ "theoretical_loss": 3.5267457730451346,
+ "tokens_seen": 1589379072
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004189528585757272,
+ "loss": 1.3431,
+ "theoretical_loss": 3.526733195959535,
+ "tokens_seen": 1589444608
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041893681043129387,
+ "loss": 1.3008,
+ "theoretical_loss": 3.5267206195376977,
+ "tokens_seen": 1589510144
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004189207622868606,
+ "loss": 1.2865,
+ "theoretical_loss": 3.5267080437795615,
+ "tokens_seen": 1589575680
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041890471414242727,
+ "loss": 1.3016,
+ "theoretical_loss": 3.526695468685064,
+ "tokens_seen": 1589641216
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000418888665997994,
+ "loss": 1.1346,
+ "theoretical_loss": 3.526682894254142,
+ "tokens_seen": 1589706752
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041887261785356067,
+ "loss": 1.3811,
+ "theoretical_loss": 3.5266703204867333,
+ "tokens_seen": 1589772288
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004188565697091274,
+ "loss": 1.2418,
+ "theoretical_loss": 3.526657747382776,
+ "tokens_seen": 1589837824
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041884052156469407,
+ "loss": 1.375,
+ "theoretical_loss": 3.526645174942207,
+ "tokens_seen": 1589903360
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004188244734202608,
+ "loss": 1.2092,
+ "theoretical_loss": 3.5266326031649653,
+ "tokens_seen": 1589968896
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041880842527582747,
+ "loss": 1.3599,
+ "theoretical_loss": 3.526620032050987,
+ "tokens_seen": 1590034432
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004187923771313942,
+ "loss": 1.265,
+ "theoretical_loss": 3.5266074616002108,
+ "tokens_seen": 1590099968
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004187763289869609,
+ "loss": 1.3309,
+ "theoretical_loss": 3.526594891812574,
+ "tokens_seen": 1590165504
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041876028084252763,
+ "loss": 1.2433,
+ "theoretical_loss": 3.526582322688014,
+ "tokens_seen": 1590231040
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004187442326980943,
+ "loss": 1.2087,
+ "theoretical_loss": 3.526569754226469,
+ "tokens_seen": 1590296576
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041872818455366103,
+ "loss": 1.3058,
+ "theoretical_loss": 3.526557186427877,
+ "tokens_seen": 1590362112
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004187121364092277,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5265446192921748,
+ "tokens_seen": 1590427648
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041869608826479443,
+ "loss": 1.2142,
+ "theoretical_loss": 3.5265320528193005,
+ "tokens_seen": 1590493184
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004186800401203611,
+ "loss": 1.3372,
+ "theoretical_loss": 3.5265194870091925,
+ "tokens_seen": 1590558720
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041866399197592783,
+ "loss": 1.3173,
+ "theoretical_loss": 3.5265069218617873,
+ "tokens_seen": 1590624256
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004186479438314945,
+ "loss": 1.4553,
+ "theoretical_loss": 3.5264943573770235,
+ "tokens_seen": 1590689792
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041863189568706123,
+ "loss": 1.3482,
+ "theoretical_loss": 3.526481793554839,
+ "tokens_seen": 1590755328
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004186158475426279,
+ "loss": 1.3073,
+ "theoretical_loss": 3.526469230395171,
+ "tokens_seen": 1590820864
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041859979939819463,
+ "loss": 1.3264,
+ "theoretical_loss": 3.5264566678979574,
+ "tokens_seen": 1590886400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004185837512537613,
+ "loss": 1.2387,
+ "theoretical_loss": 3.526444106063136,
+ "tokens_seen": 1590951936
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041856770310932804,
+ "loss": 1.2303,
+ "theoretical_loss": 3.526431544890645,
+ "tokens_seen": 1591017472
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004185516549648947,
+ "loss": 1.2593,
+ "theoretical_loss": 3.5264189843804212,
+ "tokens_seen": 1591083008
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041853560682046144,
+ "loss": 1.3082,
+ "theoretical_loss": 3.5264064245324036,
+ "tokens_seen": 1591148544
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004185195586760281,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5263938653465297,
+ "tokens_seen": 1591214080
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041850351053159484,
+ "loss": 1.2827,
+ "theoretical_loss": 3.5263813068227368,
+ "tokens_seen": 1591279616
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004184874623871615,
+ "loss": 1.2933,
+ "theoretical_loss": 3.526368748960963,
+ "tokens_seen": 1591345152
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041847141424272824,
+ "loss": 1.363,
+ "theoretical_loss": 3.5263561917611463,
+ "tokens_seen": 1591410688
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004184553660982949,
+ "loss": 1.2557,
+ "theoretical_loss": 3.5263436352232245,
+ "tokens_seen": 1591476224
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041843931795386164,
+ "loss": 1.3466,
+ "theoretical_loss": 3.526331079347135,
+ "tokens_seen": 1591541760
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004184232698094283,
+ "loss": 1.3541,
+ "theoretical_loss": 3.5263185241328165,
+ "tokens_seen": 1591607296
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041840722166499504,
+ "loss": 1.3778,
+ "theoretical_loss": 3.526305969580206,
+ "tokens_seen": 1591672832
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004183911735205617,
+ "loss": 1.3468,
+ "theoretical_loss": 3.5262934156892425,
+ "tokens_seen": 1591738368
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041837512537612844,
+ "loss": 1.3134,
+ "theoretical_loss": 3.5262808624598625,
+ "tokens_seen": 1591803904
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004183590772316951,
+ "loss": 1.2955,
+ "theoretical_loss": 3.526268309892005,
+ "tokens_seen": 1591869440
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041834302908726184,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5262557579856075,
+ "tokens_seen": 1591934976
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004183269809428285,
+ "loss": 1.3082,
+ "theoretical_loss": 3.5262432067406078,
+ "tokens_seen": 1592000512
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004183109327983952,
+ "loss": 1.2465,
+ "theoretical_loss": 3.526230656156944,
+ "tokens_seen": 1592066048
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004182948846539619,
+ "loss": 1.2867,
+ "theoretical_loss": 3.5262181062345537,
+ "tokens_seen": 1592131584
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004182788365095286,
+ "loss": 1.3035,
+ "theoretical_loss": 3.526205556973375,
+ "tokens_seen": 1592197120
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004182627883650953,
+ "loss": 1.3289,
+ "theoretical_loss": 3.5261930083733466,
+ "tokens_seen": 1592262656
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000418246740220662,
+ "loss": 1.3158,
+ "theoretical_loss": 3.5261804604344054,
+ "tokens_seen": 1592328192
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004182306920762287,
+ "loss": 1.3363,
+ "theoretical_loss": 3.52616791315649,
+ "tokens_seen": 1592393728
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004182146439317954,
+ "loss": 1.3114,
+ "theoretical_loss": 3.526155366539538,
+ "tokens_seen": 1592459264
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 534133,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2955944538116455,
+ "objective/train/theoretical_loss": 3.5261428205834875,
+ "objective/train/tokens_used": 1612984800,
+ "theoretical_loss": 3.5261428205834875,
+ "tokens_seen": 1592524800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041819859578736204,
+ "loss": 1.3085,
+ "theoretical_loss": 3.5261428205834875,
+ "tokens_seen": 1592524800
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004181825476429288,
+ "loss": 1.2541,
+ "theoretical_loss": 3.526130275288277,
+ "tokens_seen": 1592590336
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041816649949849544,
+ "loss": 1.4107,
+ "theoretical_loss": 3.5261177306538434,
+ "tokens_seen": 1592655872
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004181504513540622,
+ "loss": 1.294,
+ "theoretical_loss": 3.526105186680126,
+ "tokens_seen": 1592721408
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041813440320962884,
+ "loss": 1.3976,
+ "theoretical_loss": 3.526092643367061,
+ "tokens_seen": 1592786944
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004181183550651956,
+ "loss": 1.3058,
+ "theoretical_loss": 3.526080100714589,
+ "tokens_seen": 1592852480
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041810230692076225,
+ "loss": 1.2832,
+ "theoretical_loss": 3.526067558722646,
+ "tokens_seen": 1592918016
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000418086258776329,
+ "loss": 1.3361,
+ "theoretical_loss": 3.5260550173911707,
+ "tokens_seen": 1592983552
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041807021063189565,
+ "loss": 1.2854,
+ "theoretical_loss": 3.526042476720101,
+ "tokens_seen": 1593049088
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004180541624874624,
+ "loss": 1.3805,
+ "theoretical_loss": 3.5260299367093753,
+ "tokens_seen": 1593114624
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041803811434302905,
+ "loss": 1.3232,
+ "theoretical_loss": 3.526017397358931,
+ "tokens_seen": 1593180160
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004180220661985958,
+ "loss": 1.3571,
+ "theoretical_loss": 3.526004858668707,
+ "tokens_seen": 1593245696
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041800601805416256,
+ "loss": 1.3235,
+ "theoretical_loss": 3.525992320638641,
+ "tokens_seen": 1593311232
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004179899699097292,
+ "loss": 1.0997,
+ "theoretical_loss": 3.525979783268671,
+ "tokens_seen": 1593376768
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041797392176529596,
+ "loss": 1.3121,
+ "theoretical_loss": 3.5259672465587353,
+ "tokens_seen": 1593442304
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004179578736208626,
+ "loss": 1.3656,
+ "theoretical_loss": 3.5259547105087714,
+ "tokens_seen": 1593507840
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041794182547642936,
+ "loss": 1.2168,
+ "theoretical_loss": 3.5259421751187183,
+ "tokens_seen": 1593573376
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000417925777331996,
+ "loss": 1.3795,
+ "theoretical_loss": 3.525929640388514,
+ "tokens_seen": 1593638912
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041790972918756276,
+ "loss": 1.3063,
+ "theoretical_loss": 3.5259171063180963,
+ "tokens_seen": 1593704448
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004178936810431294,
+ "loss": 1.2893,
+ "theoretical_loss": 3.525904572907403,
+ "tokens_seen": 1593769984
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041787763289869616,
+ "loss": 1.2935,
+ "theoretical_loss": 3.5258920401563727,
+ "tokens_seen": 1593835520
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004178615847542628,
+ "loss": 1.3267,
+ "theoretical_loss": 3.5258795080649437,
+ "tokens_seen": 1593901056
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041784553660982956,
+ "loss": 1.3593,
+ "theoretical_loss": 3.525866976633054,
+ "tokens_seen": 1593966592
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004178294884653962,
+ "loss": 1.319,
+ "theoretical_loss": 3.5258544458606416,
+ "tokens_seen": 1594032128
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041781344032096297,
+ "loss": 1.297,
+ "theoretical_loss": 3.5258419157476446,
+ "tokens_seen": 1594097664
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004177973921765296,
+ "loss": 1.2661,
+ "theoretical_loss": 3.525829386294002,
+ "tokens_seen": 1594163200
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041778134403209637,
+ "loss": 1.2155,
+ "theoretical_loss": 3.5258168574996516,
+ "tokens_seen": 1594228736
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000417765295887663,
+ "loss": 1.4208,
+ "theoretical_loss": 3.5258043293645307,
+ "tokens_seen": 1594294272
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041774924774322977,
+ "loss": 1.3209,
+ "theoretical_loss": 3.5257918018885785,
+ "tokens_seen": 1594359808
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004177331995987964,
+ "loss": 1.3237,
+ "theoretical_loss": 3.5257792750717334,
+ "tokens_seen": 1594425344
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041771715145436317,
+ "loss": 1.3139,
+ "theoretical_loss": 3.5257667489139326,
+ "tokens_seen": 1594490880
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004177011033099298,
+ "loss": 1.3462,
+ "theoretical_loss": 3.525754223415116,
+ "tokens_seen": 1594556416
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004176850551654965,
+ "loss": 1.2605,
+ "theoretical_loss": 3.5257416985752195,
+ "tokens_seen": 1594621952
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004176690070210632,
+ "loss": 1.268,
+ "theoretical_loss": 3.525729174394183,
+ "tokens_seen": 1594687488
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004176529588766299,
+ "loss": 1.3794,
+ "theoretical_loss": 3.525716650871945,
+ "tokens_seen": 1594753024
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004176369107321966,
+ "loss": 1.4053,
+ "theoretical_loss": 3.525704128008443,
+ "tokens_seen": 1594818560
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004176208625877633,
+ "loss": 1.3663,
+ "theoretical_loss": 3.5256916058036154,
+ "tokens_seen": 1594884096
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041760481444333,
+ "loss": 1.3703,
+ "theoretical_loss": 3.5256790842574004,
+ "tokens_seen": 1594949632
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004175887662988967,
+ "loss": 1.286,
+ "theoretical_loss": 3.525666563369737,
+ "tokens_seen": 1595015168
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004175727181544634,
+ "loss": 1.3113,
+ "theoretical_loss": 3.5256540431405625,
+ "tokens_seen": 1595080704
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004175566700100301,
+ "loss": 1.3088,
+ "theoretical_loss": 3.5256415235698157,
+ "tokens_seen": 1595146240
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041754062186559677,
+ "loss": 1.2932,
+ "theoretical_loss": 3.5256290046574352,
+ "tokens_seen": 1595211776
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004175245737211635,
+ "loss": 1.3011,
+ "theoretical_loss": 3.525616486403359,
+ "tokens_seen": 1595277312
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041750852557673017,
+ "loss": 1.391,
+ "theoretical_loss": 3.525603968807525,
+ "tokens_seen": 1595342848
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004174924774322969,
+ "loss": 1.355,
+ "theoretical_loss": 3.5255914518698726,
+ "tokens_seen": 1595408384
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041747642928786357,
+ "loss": 1.3184,
+ "theoretical_loss": 3.5255789355903397,
+ "tokens_seen": 1595473920
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004174603811434303,
+ "loss": 1.2601,
+ "theoretical_loss": 3.5255664199688646,
+ "tokens_seen": 1595539456
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041744433299899697,
+ "loss": 1.3963,
+ "theoretical_loss": 3.5255539050053852,
+ "tokens_seen": 1595604992
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004174282848545637,
+ "loss": 1.2895,
+ "theoretical_loss": 3.5255413906998405,
+ "tokens_seen": 1595670528
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041741223671013037,
+ "loss": 1.3094,
+ "theoretical_loss": 3.525528877052169,
+ "tokens_seen": 1595736064
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 535018,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2892045974731445,
+ "objective/train/theoretical_loss": 3.5255163640623084,
+ "objective/train/tokens_used": 1616261600,
+ "theoretical_loss": 3.5255163640623084,
+ "tokens_seen": 1595801600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041739618856569713,
+ "loss": 1.3489,
+ "theoretical_loss": 3.5255163640623084,
+ "tokens_seen": 1595801600
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004173801404212638,
+ "loss": 1.3358,
+ "theoretical_loss": 3.525503851730198,
+ "tokens_seen": 1595867136
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041736409227683053,
+ "loss": 1.3527,
+ "theoretical_loss": 3.5254913400557752,
+ "tokens_seen": 1595932672
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004173480441323972,
+ "loss": 1.3156,
+ "theoretical_loss": 3.5254788290389794,
+ "tokens_seen": 1595998208
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041733199598796393,
+ "loss": 1.2212,
+ "theoretical_loss": 3.5254663186797486,
+ "tokens_seen": 1596063744
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004173159478435306,
+ "loss": 1.2936,
+ "theoretical_loss": 3.525453808978021,
+ "tokens_seen": 1596129280
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041729989969909733,
+ "loss": 1.3786,
+ "theoretical_loss": 3.5254412999337355,
+ "tokens_seen": 1596194816
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000417283851554664,
+ "loss": 1.2774,
+ "theoretical_loss": 3.5254287915468305,
+ "tokens_seen": 1596260352
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041726780341023073,
+ "loss": 1.2742,
+ "theoretical_loss": 3.5254162838172443,
+ "tokens_seen": 1596325888
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004172517552657974,
+ "loss": 1.3532,
+ "theoretical_loss": 3.5254037767449153,
+ "tokens_seen": 1596391424
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041723570712136413,
+ "loss": 1.4497,
+ "theoretical_loss": 3.5253912703297816,
+ "tokens_seen": 1596456960
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004172196589769308,
+ "loss": 1.2876,
+ "theoretical_loss": 3.5253787645717827,
+ "tokens_seen": 1596522496
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041720361083249754,
+ "loss": 1.302,
+ "theoretical_loss": 3.5253662594708564,
+ "tokens_seen": 1596588032
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004171875626880642,
+ "loss": 1.3419,
+ "theoretical_loss": 3.5253537550269414,
+ "tokens_seen": 1596653568
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041717151454363094,
+ "loss": 1.3104,
+ "theoretical_loss": 3.5253412512399764,
+ "tokens_seen": 1596719104
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004171554663991976,
+ "loss": 1.2956,
+ "theoretical_loss": 3.5253287481098994,
+ "tokens_seen": 1596784640
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041713941825476434,
+ "loss": 1.3469,
+ "theoretical_loss": 3.52531624563665,
+ "tokens_seen": 1596850176
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000417123370110331,
+ "loss": 1.1238,
+ "theoretical_loss": 3.5253037438201646,
+ "tokens_seen": 1596915712
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041710732196589774,
+ "loss": 1.3157,
+ "theoretical_loss": 3.5252912426603844,
+ "tokens_seen": 1596981248
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004170912738214644,
+ "loss": 1.2143,
+ "theoretical_loss": 3.525278742157246,
+ "tokens_seen": 1597046784
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041707522567703114,
+ "loss": 1.2819,
+ "theoretical_loss": 3.525266242310689,
+ "tokens_seen": 1597112320
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004170591775325978,
+ "loss": 1.263,
+ "theoretical_loss": 3.5252537431206514,
+ "tokens_seen": 1597177856
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041704312938816454,
+ "loss": 1.3309,
+ "theoretical_loss": 3.5252412445870718,
+ "tokens_seen": 1597243392
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004170270812437312,
+ "loss": 1.1777,
+ "theoretical_loss": 3.5252287467098897,
+ "tokens_seen": 1597308928
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041701103309929794,
+ "loss": 1.2347,
+ "theoretical_loss": 3.525216249489042,
+ "tokens_seen": 1597374464
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004169949849548646,
+ "loss": 1.2991,
+ "theoretical_loss": 3.5252037529244693,
+ "tokens_seen": 1597440000
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041697893681043134,
+ "loss": 1.3162,
+ "theoretical_loss": 3.525191257016109,
+ "tokens_seen": 1597505536
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.000416962888665998,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5251787617638994,
+ "tokens_seen": 1597571072
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041694684052156475,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5251662671677804,
+ "tokens_seen": 1597636608
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004169307923771314,
+ "loss": 1.356,
+ "theoretical_loss": 3.5251537732276894,
+ "tokens_seen": 1597702144
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004169147442326981,
+ "loss": 1.2632,
+ "theoretical_loss": 3.5251412799435657,
+ "tokens_seen": 1597767680
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168986960882648,
+ "loss": 1.3601,
+ "theoretical_loss": 3.525128787315348,
+ "tokens_seen": 1597833216
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168826479438315,
+ "loss": 1.3514,
+ "theoretical_loss": 3.5251162953429747,
+ "tokens_seen": 1597898752
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168665997993982,
+ "loss": 1.2357,
+ "theoretical_loss": 3.5251038040263847,
+ "tokens_seen": 1597964288
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168505516549649,
+ "loss": 1.3505,
+ "theoretical_loss": 3.5250913133655164,
+ "tokens_seen": 1598029824
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168345035105316,
+ "loss": 1.3869,
+ "theoretical_loss": 3.5250788233603085,
+ "tokens_seen": 1598095360
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004168184553660983,
+ "loss": 1.3275,
+ "theoretical_loss": 3.5250663340107,
+ "tokens_seen": 1598160896
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041680240722166505,
+ "loss": 1.3261,
+ "theoretical_loss": 3.525053845316629,
+ "tokens_seen": 1598226432
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004167863590772317,
+ "loss": 1.2504,
+ "theoretical_loss": 3.525041357278035,
+ "tokens_seen": 1598291968
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041677031093279845,
+ "loss": 1.2848,
+ "theoretical_loss": 3.5250288698948564,
+ "tokens_seen": 1598357504
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004167542627883651,
+ "loss": 1.2343,
+ "theoretical_loss": 3.525016383167032,
+ "tokens_seen": 1598423040
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041673821464393185,
+ "loss": 1.2793,
+ "theoretical_loss": 3.5250038970945,
+ "tokens_seen": 1598488576
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004167221664994985,
+ "loss": 1.2619,
+ "theoretical_loss": 3.5249914116772,
+ "tokens_seen": 1598554112
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041670611835506525,
+ "loss": 1.3503,
+ "theoretical_loss": 3.52497892691507,
+ "tokens_seen": 1598619648
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004166900702106319,
+ "loss": 1.2669,
+ "theoretical_loss": 3.5249664428080494,
+ "tokens_seen": 1598685184
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041667402206619866,
+ "loss": 1.2003,
+ "theoretical_loss": 3.5249539593560764,
+ "tokens_seen": 1598750720
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004166579739217653,
+ "loss": 1.2375,
+ "theoretical_loss": 3.52494147655909,
+ "tokens_seen": 1598816256
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041664192577733206,
+ "loss": 1.253,
+ "theoretical_loss": 3.5249289944170297,
+ "tokens_seen": 1598881792
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004166258776328987,
+ "loss": 1.2089,
+ "theoretical_loss": 3.524916512929833,
+ "tokens_seen": 1598947328
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041660982948846546,
+ "loss": 1.3491,
+ "theoretical_loss": 3.5249040320974396,
+ "tokens_seen": 1599012864
+ },
+ {
+ "epoch": 0.48,
+ "objective/train/docs_used": 535874,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3526699542999268,
+ "objective/train/theoretical_loss": 3.524891551919788,
+ "objective/train/tokens_used": 1619538400,
+ "theoretical_loss": 3.524891551919788,
+ "tokens_seen": 1599078400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004165937813440321,
+ "loss": 1.2515,
+ "theoretical_loss": 3.524891551919788,
+ "tokens_seen": 1599078400
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041657773319959886,
+ "loss": 1.2148,
+ "theoretical_loss": 3.524879072396817,
+ "tokens_seen": 1599143936
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004165616850551655,
+ "loss": 1.3939,
+ "theoretical_loss": 3.5248665935284658,
+ "tokens_seen": 1599209472
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041654563691073226,
+ "loss": 1.2845,
+ "theoretical_loss": 3.5248541153146724,
+ "tokens_seen": 1599275008
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004165295887662989,
+ "loss": 1.2864,
+ "theoretical_loss": 3.5248416377553764,
+ "tokens_seen": 1599340544
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041651354062186566,
+ "loss": 1.1897,
+ "theoretical_loss": 3.5248291608505165,
+ "tokens_seen": 1599406080
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004164974924774323,
+ "loss": 1.3159,
+ "theoretical_loss": 3.5248166846000313,
+ "tokens_seen": 1599471616
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041648144433299906,
+ "loss": 1.2669,
+ "theoretical_loss": 3.52480420900386,
+ "tokens_seen": 1599537152
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004164653961885657,
+ "loss": 1.2628,
+ "theoretical_loss": 3.5247917340619415,
+ "tokens_seen": 1599602688
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041644934804413247,
+ "loss": 1.3442,
+ "theoretical_loss": 3.5247792597742147,
+ "tokens_seen": 1599668224
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004164332998996991,
+ "loss": 1.3347,
+ "theoretical_loss": 3.524766786140618,
+ "tokens_seen": 1599733760
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041641725175526587,
+ "loss": 1.2555,
+ "theoretical_loss": 3.5247543131610906,
+ "tokens_seen": 1599799296
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004164012036108325,
+ "loss": 1.2195,
+ "theoretical_loss": 3.5247418408355715,
+ "tokens_seen": 1599864832
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041638515546639927,
+ "loss": 1.25,
+ "theoretical_loss": 3.5247293691639996,
+ "tokens_seen": 1599930368
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004163691073219659,
+ "loss": 1.2788,
+ "theoretical_loss": 3.5247168981463144,
+ "tokens_seen": 1599995904
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041635305917753267,
+ "loss": 1.3879,
+ "theoretical_loss": 3.5247044277824533,
+ "tokens_seen": 1600061440
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004163370110330993,
+ "loss": 1.3778,
+ "theoretical_loss": 3.5246919580723564,
+ "tokens_seen": 1600126976
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.00041632096288866607,
+ "loss": 1.3168,
+ "theoretical_loss": 3.5246794890159627,
+ "tokens_seen": 1600192512
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004163049147442327,
+ "loss": 1.3215,
+ "theoretical_loss": 3.5246670206132107,
+ "tokens_seen": 1600258048
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004162888665997994,
+ "loss": 1.3128,
+ "theoretical_loss": 3.52465455286404,
+ "tokens_seen": 1600323584
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004162728184553661,
+ "loss": 1.249,
+ "theoretical_loss": 3.524642085768388,
+ "tokens_seen": 1600389120
+ },
+ {
+ "epoch": 0.48,
+ "learning_rate": 0.0004162567703109328,
+ "loss": 1.2697,
+ "theoretical_loss": 3.5246296193261957,
+ "tokens_seen": 1600454656
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004162407221664995,
+ "loss": 1.3321,
+ "theoretical_loss": 3.524617153537401,
+ "tokens_seen": 1600520192
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004162246740220662,
+ "loss": 1.4486,
+ "theoretical_loss": 3.524604688401943,
+ "tokens_seen": 1600585728
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004162086258776329,
+ "loss": 1.2488,
+ "theoretical_loss": 3.524592223919761,
+ "tokens_seen": 1600651264
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004161925777331996,
+ "loss": 1.2404,
+ "theoretical_loss": 3.524579760090794,
+ "tokens_seen": 1600716800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004161765295887663,
+ "loss": 1.391,
+ "theoretical_loss": 3.524567296914981,
+ "tokens_seen": 1600782336
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000416160481444333,
+ "loss": 1.269,
+ "theoretical_loss": 3.5245548343922604,
+ "tokens_seen": 1600847872
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041614443329989967,
+ "loss": 1.2758,
+ "theoretical_loss": 3.5245423725225713,
+ "tokens_seen": 1600913408
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004161283851554664,
+ "loss": 1.2227,
+ "theoretical_loss": 3.524529911305854,
+ "tokens_seen": 1600978944
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041611233701103307,
+ "loss": 1.2996,
+ "theoretical_loss": 3.5245174507420467,
+ "tokens_seen": 1601044480
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004160962888665998,
+ "loss": 1.2715,
+ "theoretical_loss": 3.524504990831088,
+ "tokens_seen": 1601110016
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041608024072216647,
+ "loss": 1.3514,
+ "theoretical_loss": 3.5244925315729176,
+ "tokens_seen": 1601175552
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004160641925777332,
+ "loss": 1.3901,
+ "theoretical_loss": 3.5244800729674752,
+ "tokens_seen": 1601241088
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041604814443329987,
+ "loss": 1.2613,
+ "theoretical_loss": 3.5244676150146983,
+ "tokens_seen": 1601306624
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041603209628886663,
+ "loss": 1.27,
+ "theoretical_loss": 3.5244551577145273,
+ "tokens_seen": 1601372160
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004160160481444333,
+ "loss": 1.2595,
+ "theoretical_loss": 3.524442701066901,
+ "tokens_seen": 1601437696
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041600000000000003,
+ "loss": 1.3035,
+ "theoretical_loss": 3.524430245071758,
+ "tokens_seen": 1601503232
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004159839518555667,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5244177897290383,
+ "tokens_seen": 1601568768
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041596790371113343,
+ "loss": 1.2935,
+ "theoretical_loss": 3.52440533503868,
+ "tokens_seen": 1601634304
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004159518555667001,
+ "loss": 1.1936,
+ "theoretical_loss": 3.524392881000623,
+ "tokens_seen": 1601699840
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041593580742226683,
+ "loss": 1.3264,
+ "theoretical_loss": 3.524380427614806,
+ "tokens_seen": 1601765376
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004159197592778335,
+ "loss": 1.4175,
+ "theoretical_loss": 3.5243679748811685,
+ "tokens_seen": 1601830912
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041590371113340023,
+ "loss": 1.3324,
+ "theoretical_loss": 3.5243555227996497,
+ "tokens_seen": 1601896448
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004158876629889669,
+ "loss": 1.2627,
+ "theoretical_loss": 3.524343071370189,
+ "tokens_seen": 1601961984
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041587161484453363,
+ "loss": 1.252,
+ "theoretical_loss": 3.5243306205927247,
+ "tokens_seen": 1602027520
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004158555667001003,
+ "loss": 1.286,
+ "theoretical_loss": 3.5243181704671964,
+ "tokens_seen": 1602093056
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041583951855566703,
+ "loss": 1.2345,
+ "theoretical_loss": 3.524305720993544,
+ "tokens_seen": 1602158592
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004158234704112337,
+ "loss": 1.2133,
+ "theoretical_loss": 3.5242932721717057,
+ "tokens_seen": 1602224128
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041580742226680044,
+ "loss": 1.2693,
+ "theoretical_loss": 3.524280824001621,
+ "tokens_seen": 1602289664
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 537090,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3611112833023071,
+ "objective/train/theoretical_loss": 3.524268376483229,
+ "objective/train/tokens_used": 1622815200,
+ "theoretical_loss": 3.524268376483229,
+ "tokens_seen": 1602355200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004157913741223671,
+ "loss": 1.2988,
+ "theoretical_loss": 3.524268376483229,
+ "tokens_seen": 1602355200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041577532597793384,
+ "loss": 1.283,
+ "theoretical_loss": 3.5242559296164697,
+ "tokens_seen": 1602420736
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004157592778335005,
+ "loss": 1.403,
+ "theoretical_loss": 3.524243483401282,
+ "tokens_seen": 1602486272
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041574322968906724,
+ "loss": 1.2206,
+ "theoretical_loss": 3.5242310378376045,
+ "tokens_seen": 1602551808
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004157271815446339,
+ "loss": 1.2726,
+ "theoretical_loss": 3.5242185929253766,
+ "tokens_seen": 1602617344
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041571113340020064,
+ "loss": 1.351,
+ "theoretical_loss": 3.5242061486645384,
+ "tokens_seen": 1602682880
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004156950852557673,
+ "loss": 1.3489,
+ "theoretical_loss": 3.5241937050550285,
+ "tokens_seen": 1602748416
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041567903711133404,
+ "loss": 1.2575,
+ "theoretical_loss": 3.5241812620967865,
+ "tokens_seen": 1602813952
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004156629889669007,
+ "loss": 1.2989,
+ "theoretical_loss": 3.524168819789751,
+ "tokens_seen": 1602879488
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041564694082246744,
+ "loss": 1.2948,
+ "theoretical_loss": 3.5241563781338625,
+ "tokens_seen": 1602945024
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004156308926780341,
+ "loss": 1.3361,
+ "theoretical_loss": 3.5241439371290593,
+ "tokens_seen": 1603010560
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041561484453360084,
+ "loss": 1.2728,
+ "theoretical_loss": 3.524131496775281,
+ "tokens_seen": 1603076096
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041559879638916754,
+ "loss": 1.3738,
+ "theoretical_loss": 3.5241190570724665,
+ "tokens_seen": 1603141632
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041558274824473425,
+ "loss": 1.2662,
+ "theoretical_loss": 3.5241066180205562,
+ "tokens_seen": 1603207168
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041556670010030095,
+ "loss": 1.297,
+ "theoretical_loss": 3.524094179619489,
+ "tokens_seen": 1603272704
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041555065195586765,
+ "loss": 1.2055,
+ "theoretical_loss": 3.524081741869203,
+ "tokens_seen": 1603338240
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041553460381143435,
+ "loss": 1.2949,
+ "theoretical_loss": 3.5240693047696396,
+ "tokens_seen": 1603403776
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000415518555667001,
+ "loss": 1.357,
+ "theoretical_loss": 3.5240568683207365,
+ "tokens_seen": 1603469312
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041550250752256775,
+ "loss": 1.2265,
+ "theoretical_loss": 3.524044432522434,
+ "tokens_seen": 1603534848
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004154864593781344,
+ "loss": 1.3354,
+ "theoretical_loss": 3.524031997374671,
+ "tokens_seen": 1603600384
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041547041123370115,
+ "loss": 1.2541,
+ "theoretical_loss": 3.5240195628773874,
+ "tokens_seen": 1603665920
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004154543630892678,
+ "loss": 1.3173,
+ "theoretical_loss": 3.5240071290305224,
+ "tokens_seen": 1603731456
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041543831494483455,
+ "loss": 1.2055,
+ "theoretical_loss": 3.523994695834015,
+ "tokens_seen": 1603796992
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004154222668004012,
+ "loss": 1.2651,
+ "theoretical_loss": 3.523982263287805,
+ "tokens_seen": 1603862528
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041540621865596795,
+ "loss": 1.2806,
+ "theoretical_loss": 3.5239698313918315,
+ "tokens_seen": 1603928064
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004153901705115346,
+ "loss": 1.3639,
+ "theoretical_loss": 3.5239574001460348,
+ "tokens_seen": 1603993600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041537412236710135,
+ "loss": 1.2573,
+ "theoretical_loss": 3.5239449695503526,
+ "tokens_seen": 1604059136
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000415358074222668,
+ "loss": 1.2743,
+ "theoretical_loss": 3.523932539604726,
+ "tokens_seen": 1604124672
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041534202607823475,
+ "loss": 1.2824,
+ "theoretical_loss": 3.523920110309094,
+ "tokens_seen": 1604190208
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004153259779338014,
+ "loss": 1.1919,
+ "theoretical_loss": 3.5239076816633954,
+ "tokens_seen": 1604255744
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041530992978936816,
+ "loss": 1.3154,
+ "theoretical_loss": 3.523895253667571,
+ "tokens_seen": 1604321280
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004152938816449348,
+ "loss": 1.2457,
+ "theoretical_loss": 3.523882826321559,
+ "tokens_seen": 1604386816
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041527783350050156,
+ "loss": 1.2227,
+ "theoretical_loss": 3.523870399625299,
+ "tokens_seen": 1604452352
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004152617853560682,
+ "loss": 1.2721,
+ "theoretical_loss": 3.5238579735787314,
+ "tokens_seen": 1604517888
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041524573721163496,
+ "loss": 1.3266,
+ "theoretical_loss": 3.5238455481817947,
+ "tokens_seen": 1604583424
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004152296890672016,
+ "loss": 1.2273,
+ "theoretical_loss": 3.523833123434429,
+ "tokens_seen": 1604648960
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041521364092276836,
+ "loss": 1.2599,
+ "theoretical_loss": 3.5238206993365737,
+ "tokens_seen": 1604714496
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000415197592778335,
+ "loss": 1.3976,
+ "theoretical_loss": 3.5238082758881686,
+ "tokens_seen": 1604780032
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041518154463390176,
+ "loss": 1.2827,
+ "theoretical_loss": 3.5237958530891524,
+ "tokens_seen": 1604845568
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004151654964894684,
+ "loss": 1.3363,
+ "theoretical_loss": 3.5237834309394653,
+ "tokens_seen": 1604911104
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041514944834503516,
+ "loss": 1.2726,
+ "theoretical_loss": 3.523771009439047,
+ "tokens_seen": 1604976640
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004151334002006018,
+ "loss": 1.2827,
+ "theoretical_loss": 3.523758588587836,
+ "tokens_seen": 1605042176
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041511735205616856,
+ "loss": 1.2786,
+ "theoretical_loss": 3.5237461683857734,
+ "tokens_seen": 1605107712
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004151013039117352,
+ "loss": 1.2373,
+ "theoretical_loss": 3.5237337488327976,
+ "tokens_seen": 1605173248
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041508525576730196,
+ "loss": 1.2837,
+ "theoretical_loss": 3.523721329928849,
+ "tokens_seen": 1605238784
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004150692076228686,
+ "loss": 1.3342,
+ "theoretical_loss": 3.523708911673866,
+ "tokens_seen": 1605304320
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041505315947843537,
+ "loss": 1.2353,
+ "theoretical_loss": 3.523696494067789,
+ "tokens_seen": 1605369856
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000415037111334002,
+ "loss": 1.3092,
+ "theoretical_loss": 3.5236840771105578,
+ "tokens_seen": 1605435392
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041502106318956877,
+ "loss": 1.2923,
+ "theoretical_loss": 3.523671660802112,
+ "tokens_seen": 1605500928
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004150050150451354,
+ "loss": 1.2769,
+ "theoretical_loss": 3.5236592451423907,
+ "tokens_seen": 1605566464
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 538406,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2975478172302246,
+ "objective/train/theoretical_loss": 3.5236468301313337,
+ "objective/train/tokens_used": 1626092000,
+ "theoretical_loss": 3.5236468301313337,
+ "tokens_seen": 1605632000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041498896690070217,
+ "loss": 1.3045,
+ "theoretical_loss": 3.5236468301313337,
+ "tokens_seen": 1605632000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004149729187562688,
+ "loss": 1.3784,
+ "theoretical_loss": 3.523634415768881,
+ "tokens_seen": 1605697536
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041495687061183557,
+ "loss": 1.2807,
+ "theoretical_loss": 3.523622002054972,
+ "tokens_seen": 1605763072
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004149408224674022,
+ "loss": 1.286,
+ "theoretical_loss": 3.5236095889895465,
+ "tokens_seen": 1605828608
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041492477432296897,
+ "loss": 1.3701,
+ "theoretical_loss": 3.523597176572544,
+ "tokens_seen": 1605894144
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004149087261785356,
+ "loss": 1.3172,
+ "theoretical_loss": 3.5235847648039034,
+ "tokens_seen": 1605959680
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004148926780341023,
+ "loss": 1.2887,
+ "theoretical_loss": 3.5235723536835657,
+ "tokens_seen": 1606025216
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000414876629889669,
+ "loss": 1.2816,
+ "theoretical_loss": 3.52355994321147,
+ "tokens_seen": 1606090752
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004148605817452357,
+ "loss": 1.2408,
+ "theoretical_loss": 3.523547533387556,
+ "tokens_seen": 1606156288
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004148445336008024,
+ "loss": 1.1945,
+ "theoretical_loss": 3.5235351242117634,
+ "tokens_seen": 1606221824
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004148284854563691,
+ "loss": 1.3582,
+ "theoretical_loss": 3.523522715684032,
+ "tokens_seen": 1606287360
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004148124373119358,
+ "loss": 1.3528,
+ "theoretical_loss": 3.523510307804301,
+ "tokens_seen": 1606352896
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004147963891675025,
+ "loss": 1.3063,
+ "theoretical_loss": 3.5234979005725116,
+ "tokens_seen": 1606418432
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004147803410230692,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5234854939886016,
+ "tokens_seen": 1606483968
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004147642928786359,
+ "loss": 1.2798,
+ "theoretical_loss": 3.523473088052512,
+ "tokens_seen": 1606549504
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041474824473420257,
+ "loss": 1.3439,
+ "theoretical_loss": 3.523460682764182,
+ "tokens_seen": 1606615040
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004147321965897693,
+ "loss": 1.3044,
+ "theoretical_loss": 3.523448278123552,
+ "tokens_seen": 1606680576
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041471614844533597,
+ "loss": 1.2384,
+ "theoretical_loss": 3.523435874130561,
+ "tokens_seen": 1606746112
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004147001003009027,
+ "loss": 1.3326,
+ "theoretical_loss": 3.523423470785149,
+ "tokens_seen": 1606811648
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041468405215646937,
+ "loss": 1.3018,
+ "theoretical_loss": 3.523411068087256,
+ "tokens_seen": 1606877184
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004146680040120361,
+ "loss": 1.3179,
+ "theoretical_loss": 3.5233986660368215,
+ "tokens_seen": 1606942720
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004146519558676028,
+ "loss": 1.296,
+ "theoretical_loss": 3.5233862646337855,
+ "tokens_seen": 1607008256
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041463590772316953,
+ "loss": 1.3282,
+ "theoretical_loss": 3.523373863878088,
+ "tokens_seen": 1607073792
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004146198595787362,
+ "loss": 1.2659,
+ "theoretical_loss": 3.5233614637696684,
+ "tokens_seen": 1607139328
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041460381143430293,
+ "loss": 1.3368,
+ "theoretical_loss": 3.5233490643084666,
+ "tokens_seen": 1607204864
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004145877632898696,
+ "loss": 1.3547,
+ "theoretical_loss": 3.5233366654944227,
+ "tokens_seen": 1607270400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041457171514543633,
+ "loss": 1.3301,
+ "theoretical_loss": 3.5233242673274763,
+ "tokens_seen": 1607335936
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000414555667001003,
+ "loss": 1.2426,
+ "theoretical_loss": 3.5233118698075674,
+ "tokens_seen": 1607401472
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041453961885656973,
+ "loss": 1.3547,
+ "theoretical_loss": 3.523299472934636,
+ "tokens_seen": 1607467008
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004145235707121364,
+ "loss": 1.2979,
+ "theoretical_loss": 3.5232870767086215,
+ "tokens_seen": 1607532544
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041450752256770313,
+ "loss": 1.3082,
+ "theoretical_loss": 3.5232746811294637,
+ "tokens_seen": 1607598080
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004144914744232698,
+ "loss": 1.3847,
+ "theoretical_loss": 3.5232622861971032,
+ "tokens_seen": 1607663616
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041447542627883653,
+ "loss": 1.3044,
+ "theoretical_loss": 3.5232498919114787,
+ "tokens_seen": 1607729152
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004144593781344032,
+ "loss": 1.2915,
+ "theoretical_loss": 3.5232374982725316,
+ "tokens_seen": 1607794688
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041444332998996994,
+ "loss": 1.3189,
+ "theoretical_loss": 3.523225105280201,
+ "tokens_seen": 1607860224
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004144272818455366,
+ "loss": 1.3514,
+ "theoretical_loss": 3.523212712934427,
+ "tokens_seen": 1607925760
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041441123370110334,
+ "loss": 1.2246,
+ "theoretical_loss": 3.523200321235149,
+ "tokens_seen": 1607991296
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004143951855566701,
+ "loss": 1.3455,
+ "theoretical_loss": 3.5231879301823072,
+ "tokens_seen": 1608056832
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041437913741223674,
+ "loss": 1.2461,
+ "theoretical_loss": 3.523175539775842,
+ "tokens_seen": 1608122368
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004143630892678035,
+ "loss": 1.1826,
+ "theoretical_loss": 3.523163150015693,
+ "tokens_seen": 1608187904
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041434704112337014,
+ "loss": 1.2882,
+ "theoretical_loss": 3.5231507609018,
+ "tokens_seen": 1608253440
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004143309929789369,
+ "loss": 1.3647,
+ "theoretical_loss": 3.5231383724341034,
+ "tokens_seen": 1608318976
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041431494483450354,
+ "loss": 1.4542,
+ "theoretical_loss": 3.523125984612542,
+ "tokens_seen": 1608384512
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004142988966900703,
+ "loss": 1.4548,
+ "theoretical_loss": 3.5231135974370575,
+ "tokens_seen": 1608450048
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041428284854563694,
+ "loss": 1.223,
+ "theoretical_loss": 3.5231012109075888,
+ "tokens_seen": 1608515584
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041426680040120364,
+ "loss": 1.245,
+ "theoretical_loss": 3.523088825024076,
+ "tokens_seen": 1608581120
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041425075225677034,
+ "loss": 1.2947,
+ "theoretical_loss": 3.5230764397864593,
+ "tokens_seen": 1608646656
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041423470411233704,
+ "loss": 1.1774,
+ "theoretical_loss": 3.523064055194679,
+ "tokens_seen": 1608712192
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041421865596790374,
+ "loss": 1.3004,
+ "theoretical_loss": 3.523051671248674,
+ "tokens_seen": 1608777728
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041420260782347045,
+ "loss": 1.2679,
+ "theoretical_loss": 3.5230392879483854,
+ "tokens_seen": 1608843264
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 539279,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.215774416923523,
+ "objective/train/theoretical_loss": 3.5230269052937526,
+ "objective/train/tokens_used": 1629368800,
+ "theoretical_loss": 3.5230269052937526,
+ "tokens_seen": 1608908800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041418655967903715,
+ "loss": 1.2308,
+ "theoretical_loss": 3.5230269052937526,
+ "tokens_seen": 1608908800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041417051153460385,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5230145232847163,
+ "tokens_seen": 1608974336
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041415446339017055,
+ "loss": 1.444,
+ "theoretical_loss": 3.523002141921216,
+ "tokens_seen": 1609039872
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041413841524573725,
+ "loss": 1.275,
+ "theoretical_loss": 3.522989761203192,
+ "tokens_seen": 1609105408
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004141223671013039,
+ "loss": 1.3311,
+ "theoretical_loss": 3.522977381130584,
+ "tokens_seen": 1609170944
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041410631895687065,
+ "loss": 1.3435,
+ "theoretical_loss": 3.5229650017033327,
+ "tokens_seen": 1609236480
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004140902708124373,
+ "loss": 1.2941,
+ "theoretical_loss": 3.5229526229213777,
+ "tokens_seen": 1609302016
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041407422266800405,
+ "loss": 1.2829,
+ "theoretical_loss": 3.5229402447846594,
+ "tokens_seen": 1609367552
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004140581745235707,
+ "loss": 1.2612,
+ "theoretical_loss": 3.5229278672931175,
+ "tokens_seen": 1609433088
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041404212637913745,
+ "loss": 1.2628,
+ "theoretical_loss": 3.5229154904466924,
+ "tokens_seen": 1609498624
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004140260782347041,
+ "loss": 1.2724,
+ "theoretical_loss": 3.522903114245324,
+ "tokens_seen": 1609564160
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041401003009027085,
+ "loss": 1.3474,
+ "theoretical_loss": 3.522890738688953,
+ "tokens_seen": 1609629696
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004139939819458375,
+ "loss": 1.2274,
+ "theoretical_loss": 3.5228783637775183,
+ "tokens_seen": 1609695232
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041397793380140425,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5228659895109615,
+ "tokens_seen": 1609760768
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004139618856569709,
+ "loss": 1.2198,
+ "theoretical_loss": 3.522853615889222,
+ "tokens_seen": 1609826304
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041394583751253766,
+ "loss": 1.3051,
+ "theoretical_loss": 3.5228412429122393,
+ "tokens_seen": 1609891840
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004139297893681043,
+ "loss": 1.3384,
+ "theoretical_loss": 3.5228288705799544,
+ "tokens_seen": 1609957376
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041391374122367106,
+ "loss": 1.2414,
+ "theoretical_loss": 3.5228164988923076,
+ "tokens_seen": 1610022912
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004138976930792377,
+ "loss": 1.364,
+ "theoretical_loss": 3.5228041278492395,
+ "tokens_seen": 1610088448
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041388164493480446,
+ "loss": 1.3697,
+ "theoretical_loss": 3.5227917574506886,
+ "tokens_seen": 1610153984
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004138655967903711,
+ "loss": 1.2494,
+ "theoretical_loss": 3.5227793876965965,
+ "tokens_seen": 1610219520
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041384954864593786,
+ "loss": 1.2935,
+ "theoretical_loss": 3.5227670185869027,
+ "tokens_seen": 1610285056
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004138335005015045,
+ "loss": 1.3039,
+ "theoretical_loss": 3.5227546501215476,
+ "tokens_seen": 1610350592
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041381745235707126,
+ "loss": 1.3171,
+ "theoretical_loss": 3.522742282300472,
+ "tokens_seen": 1610416128
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004138014042126379,
+ "loss": 1.1883,
+ "theoretical_loss": 3.522729915123615,
+ "tokens_seen": 1610481664
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041378535606820466,
+ "loss": 1.2982,
+ "theoretical_loss": 3.522717548590918,
+ "tokens_seen": 1610547200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004137693079237713,
+ "loss": 1.2177,
+ "theoretical_loss": 3.5227051827023206,
+ "tokens_seen": 1610612736
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041375325977933806,
+ "loss": 1.3035,
+ "theoretical_loss": 3.522692817457763,
+ "tokens_seen": 1610678272
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004137372116349047,
+ "loss": 1.269,
+ "theoretical_loss": 3.522680452857186,
+ "tokens_seen": 1610743808
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041372116349047146,
+ "loss": 1.3868,
+ "theoretical_loss": 3.522668088900528,
+ "tokens_seen": 1610809344
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004137051153460381,
+ "loss": 1.3411,
+ "theoretical_loss": 3.5226557255877324,
+ "tokens_seen": 1610874880
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041368906720160487,
+ "loss": 1.2988,
+ "theoretical_loss": 3.5226433629187373,
+ "tokens_seen": 1610940416
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004136730190571715,
+ "loss": 1.1887,
+ "theoretical_loss": 3.522631000893483,
+ "tokens_seen": 1611005952
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041365697091273827,
+ "loss": 1.202,
+ "theoretical_loss": 3.5226186395119106,
+ "tokens_seen": 1611071488
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004136409227683049,
+ "loss": 1.2691,
+ "theoretical_loss": 3.52260627877396,
+ "tokens_seen": 1611137024
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041362487462387167,
+ "loss": 1.2519,
+ "theoretical_loss": 3.5225939186795716,
+ "tokens_seen": 1611202560
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004136088264794383,
+ "loss": 1.3052,
+ "theoretical_loss": 3.5225815592286853,
+ "tokens_seen": 1611268096
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041359277833500507,
+ "loss": 1.3524,
+ "theoretical_loss": 3.5225692004212426,
+ "tokens_seen": 1611333632
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004135767301905717,
+ "loss": 1.1798,
+ "theoretical_loss": 3.522556842257183,
+ "tokens_seen": 1611399168
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041356068204613847,
+ "loss": 1.2865,
+ "theoretical_loss": 3.5225444847364464,
+ "tokens_seen": 1611464704
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004135446339017051,
+ "loss": 1.3008,
+ "theoretical_loss": 3.5225321278589736,
+ "tokens_seen": 1611530240
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041352858575727187,
+ "loss": 1.3017,
+ "theoretical_loss": 3.5225197716247054,
+ "tokens_seen": 1611595776
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004135125376128385,
+ "loss": 1.3895,
+ "theoretical_loss": 3.522507416033582,
+ "tokens_seen": 1611661312
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134964894684052,
+ "loss": 1.2815,
+ "theoretical_loss": 3.522495061085543,
+ "tokens_seen": 1611726848
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134804413239719,
+ "loss": 1.3243,
+ "theoretical_loss": 3.5224827067805293,
+ "tokens_seen": 1611792384
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134643931795386,
+ "loss": 1.3513,
+ "theoretical_loss": 3.5224703531184813,
+ "tokens_seen": 1611857920
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134483450351053,
+ "loss": 1.3026,
+ "theoretical_loss": 3.5224580000993395,
+ "tokens_seen": 1611923456
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000413432296890672,
+ "loss": 1.3368,
+ "theoretical_loss": 3.522445647723045,
+ "tokens_seen": 1611988992
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134162487462387,
+ "loss": 1.2917,
+ "theoretical_loss": 3.5224332959895364,
+ "tokens_seen": 1612054528
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004134002006018054,
+ "loss": 1.3536,
+ "theoretical_loss": 3.522420944898756,
+ "tokens_seen": 1612120064
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 540479,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2837307453155518,
+ "objective/train/theoretical_loss": 3.522408594450643,
+ "objective/train/tokens_used": 1632645600,
+ "theoretical_loss": 3.522408594450643,
+ "tokens_seen": 1612185600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004133841524573721,
+ "loss": 1.3376,
+ "theoretical_loss": 3.522408594450643,
+ "tokens_seen": 1612185600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004133681043129388,
+ "loss": 1.263,
+ "theoretical_loss": 3.5223962446451376,
+ "tokens_seen": 1612251136
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041335205616850547,
+ "loss": 1.2734,
+ "theoretical_loss": 3.522383895482182,
+ "tokens_seen": 1612316672
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004133360080240722,
+ "loss": 1.382,
+ "theoretical_loss": 3.5223715469617147,
+ "tokens_seen": 1612382208
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041331995987963887,
+ "loss": 1.2277,
+ "theoretical_loss": 3.5223591990836773,
+ "tokens_seen": 1612447744
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004133039117352056,
+ "loss": 1.3789,
+ "theoretical_loss": 3.5223468518480097,
+ "tokens_seen": 1612513280
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004132878635907723,
+ "loss": 1.2571,
+ "theoretical_loss": 3.522334505254653,
+ "tokens_seen": 1612578816
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041327181544633903,
+ "loss": 1.2769,
+ "theoretical_loss": 3.522322159303547,
+ "tokens_seen": 1612644352
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004132557673019057,
+ "loss": 1.3262,
+ "theoretical_loss": 3.5223098139946325,
+ "tokens_seen": 1612709888
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041323971915747243,
+ "loss": 1.3533,
+ "theoretical_loss": 3.5222974693278504,
+ "tokens_seen": 1612775424
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004132236710130391,
+ "loss": 1.1954,
+ "theoretical_loss": 3.5222851253031404,
+ "tokens_seen": 1612840960
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041320762286860583,
+ "loss": 1.3715,
+ "theoretical_loss": 3.522272781920444,
+ "tokens_seen": 1612906496
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004131915747241726,
+ "loss": 1.3869,
+ "theoretical_loss": 3.5222604391797008,
+ "tokens_seen": 1612972032
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041317552657973923,
+ "loss": 1.4016,
+ "theoretical_loss": 3.5222480970808516,
+ "tokens_seen": 1613037568
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000413159478435306,
+ "loss": 1.3609,
+ "theoretical_loss": 3.522235755623837,
+ "tokens_seen": 1613103104
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041314343029087263,
+ "loss": 1.2471,
+ "theoretical_loss": 3.5222234148085976,
+ "tokens_seen": 1613168640
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004131273821464394,
+ "loss": 1.3479,
+ "theoretical_loss": 3.522211074635074,
+ "tokens_seen": 1613234176
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041311133400200603,
+ "loss": 1.4156,
+ "theoretical_loss": 3.5221987351032067,
+ "tokens_seen": 1613299712
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004130952858575728,
+ "loss": 1.2331,
+ "theoretical_loss": 3.5221863962129367,
+ "tokens_seen": 1613365248
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041307923771313944,
+ "loss": 1.2116,
+ "theoretical_loss": 3.5221740579642034,
+ "tokens_seen": 1613430784
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004130631895687062,
+ "loss": 1.2599,
+ "theoretical_loss": 3.5221617203569484,
+ "tokens_seen": 1613496320
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041304714142427284,
+ "loss": 1.3422,
+ "theoretical_loss": 3.5221493833911124,
+ "tokens_seen": 1613561856
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004130310932798396,
+ "loss": 1.1559,
+ "theoretical_loss": 3.522137047066635,
+ "tokens_seen": 1613627392
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041301504513540624,
+ "loss": 1.2957,
+ "theoretical_loss": 3.5221247113834577,
+ "tokens_seen": 1613692928
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000412998996990973,
+ "loss": 1.2715,
+ "theoretical_loss": 3.5221123763415214,
+ "tokens_seen": 1613758464
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041298294884653964,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5221000419407655,
+ "tokens_seen": 1613824000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004129669007021064,
+ "loss": 1.2881,
+ "theoretical_loss": 3.5220877081811315,
+ "tokens_seen": 1613889536
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041295085255767304,
+ "loss": 1.3578,
+ "theoretical_loss": 3.52207537506256,
+ "tokens_seen": 1613955072
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004129348044132398,
+ "loss": 1.3039,
+ "theoretical_loss": 3.5220630425849917,
+ "tokens_seen": 1614020608
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041291875626880644,
+ "loss": 1.3073,
+ "theoretical_loss": 3.522050710748367,
+ "tokens_seen": 1614086144
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004129027081243732,
+ "loss": 1.3265,
+ "theoretical_loss": 3.522038379552626,
+ "tokens_seen": 1614151680
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041288665997993984,
+ "loss": 1.3409,
+ "theoretical_loss": 3.522026048997711,
+ "tokens_seen": 1614217216
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004128706118355066,
+ "loss": 1.273,
+ "theoretical_loss": 3.522013719083561,
+ "tokens_seen": 1614282752
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041285456369107324,
+ "loss": 1.2903,
+ "theoretical_loss": 3.522001389810118,
+ "tokens_seen": 1614348288
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041283851554663995,
+ "loss": 1.3074,
+ "theoretical_loss": 3.5219890611773215,
+ "tokens_seen": 1614413824
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041282246740220665,
+ "loss": 1.4072,
+ "theoretical_loss": 3.521976733185113,
+ "tokens_seen": 1614479360
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041280641925777335,
+ "loss": 1.2841,
+ "theoretical_loss": 3.521964405833433,
+ "tokens_seen": 1614544896
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041279037111334005,
+ "loss": 1.4054,
+ "theoretical_loss": 3.5219520791222223,
+ "tokens_seen": 1614610432
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041277432296890675,
+ "loss": 1.3945,
+ "theoretical_loss": 3.521939753051422,
+ "tokens_seen": 1614675968
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041275827482447345,
+ "loss": 1.3681,
+ "theoretical_loss": 3.521927427620972,
+ "tokens_seen": 1614741504
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041274222668004015,
+ "loss": 1.3696,
+ "theoretical_loss": 3.521915102830813,
+ "tokens_seen": 1614807040
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004127261785356068,
+ "loss": 1.3058,
+ "theoretical_loss": 3.521902778680887,
+ "tokens_seen": 1614872576
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041271013039117355,
+ "loss": 1.2813,
+ "theoretical_loss": 3.5218904551711336,
+ "tokens_seen": 1614938112
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004126940822467402,
+ "loss": 1.3238,
+ "theoretical_loss": 3.5218781323014943,
+ "tokens_seen": 1615003648
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041267803410230695,
+ "loss": 1.4571,
+ "theoretical_loss": 3.521865810071909,
+ "tokens_seen": 1615069184
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004126619859578736,
+ "loss": 1.3464,
+ "theoretical_loss": 3.521853488482319,
+ "tokens_seen": 1615134720
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041264593781344035,
+ "loss": 1.2765,
+ "theoretical_loss": 3.5218411675326653,
+ "tokens_seen": 1615200256
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000412629889669007,
+ "loss": 1.3163,
+ "theoretical_loss": 3.5218288472228885,
+ "tokens_seen": 1615265792
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041261384152457375,
+ "loss": 1.349,
+ "theoretical_loss": 3.5218165275529296,
+ "tokens_seen": 1615331328
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004125977933801404,
+ "loss": 1.3393,
+ "theoretical_loss": 3.5218042085227292,
+ "tokens_seen": 1615396864
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 541462,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3924484252929688,
+ "objective/train/theoretical_loss": 3.521791890132228,
+ "objective/train/tokens_used": 1635922400,
+ "theoretical_loss": 3.521791890132228,
+ "tokens_seen": 1615462400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041258174523570716,
+ "loss": 1.3301,
+ "theoretical_loss": 3.521791890132228,
+ "tokens_seen": 1615462400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004125656970912738,
+ "loss": 1.2946,
+ "theoretical_loss": 3.5217795723813667,
+ "tokens_seen": 1615527936
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041254964894684056,
+ "loss": 1.2791,
+ "theoretical_loss": 3.521767255270087,
+ "tokens_seen": 1615593472
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004125336008024072,
+ "loss": 1.2704,
+ "theoretical_loss": 3.521754938798329,
+ "tokens_seen": 1615659008
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041251755265797396,
+ "loss": 1.2761,
+ "theoretical_loss": 3.521742622966033,
+ "tokens_seen": 1615724544
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004125015045135406,
+ "loss": 1.2553,
+ "theoretical_loss": 3.521730307773142,
+ "tokens_seen": 1615790080
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041248545636910736,
+ "loss": 1.3563,
+ "theoretical_loss": 3.5217179932195943,
+ "tokens_seen": 1615855616
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000412469408224674,
+ "loss": 1.3129,
+ "theoretical_loss": 3.521705679305332,
+ "tokens_seen": 1615921152
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041245336008024076,
+ "loss": 1.3465,
+ "theoretical_loss": 3.5216933660302967,
+ "tokens_seen": 1615986688
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004124373119358074,
+ "loss": 1.2953,
+ "theoretical_loss": 3.5216810533944276,
+ "tokens_seen": 1616052224
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041242126379137416,
+ "loss": 1.3918,
+ "theoretical_loss": 3.521668741397667,
+ "tokens_seen": 1616117760
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004124052156469408,
+ "loss": 1.3707,
+ "theoretical_loss": 3.5216564300399558,
+ "tokens_seen": 1616183296
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041238916750250756,
+ "loss": 1.3014,
+ "theoretical_loss": 3.5216441193212336,
+ "tokens_seen": 1616248832
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004123731193580742,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5216318092414425,
+ "tokens_seen": 1616314368
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041235707121364096,
+ "loss": 1.2968,
+ "theoretical_loss": 3.5216194998005235,
+ "tokens_seen": 1616379904
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004123410230692076,
+ "loss": 1.2852,
+ "theoretical_loss": 3.5216071909984166,
+ "tokens_seen": 1616445440
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041232497492477437,
+ "loss": 1.2701,
+ "theoretical_loss": 3.5215948828350636,
+ "tokens_seen": 1616510976
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000412308926780341,
+ "loss": 1.306,
+ "theoretical_loss": 3.521582575310405,
+ "tokens_seen": 1616576512
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041229287863590777,
+ "loss": 1.3116,
+ "theoretical_loss": 3.521570268424382,
+ "tokens_seen": 1616642048
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004122768304914744,
+ "loss": 1.2724,
+ "theoretical_loss": 3.521557962176936,
+ "tokens_seen": 1616707584
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041226078234704117,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5215456565680068,
+ "tokens_seen": 1616773120
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004122447342026078,
+ "loss": 1.232,
+ "theoretical_loss": 3.5215333515975367,
+ "tokens_seen": 1616838656
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041222868605817457,
+ "loss": 1.3332,
+ "theoretical_loss": 3.521521047265465,
+ "tokens_seen": 1616904192
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004122126379137412,
+ "loss": 1.2532,
+ "theoretical_loss": 3.521508743571735,
+ "tokens_seen": 1616969728
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041219658976930797,
+ "loss": 1.2531,
+ "theoretical_loss": 3.521496440516286,
+ "tokens_seen": 1617035264
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004121805416248746,
+ "loss": 1.3661,
+ "theoretical_loss": 3.5214841380990594,
+ "tokens_seen": 1617100800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041216449348044137,
+ "loss": 1.3272,
+ "theoretical_loss": 3.5214718363199964,
+ "tokens_seen": 1617166336
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000412148445336008,
+ "loss": 1.2134,
+ "theoretical_loss": 3.521459535179038,
+ "tokens_seen": 1617231872
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004121323971915748,
+ "loss": 1.1857,
+ "theoretical_loss": 3.521447234676126,
+ "tokens_seen": 1617297408
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004121163490471414,
+ "loss": 1.4602,
+ "theoretical_loss": 3.5214349348111997,
+ "tokens_seen": 1617362944
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004121003009027082,
+ "loss": 1.303,
+ "theoretical_loss": 3.5214226355842015,
+ "tokens_seen": 1617428480
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120842527582748,
+ "loss": 1.3819,
+ "theoretical_loss": 3.521410336995072,
+ "tokens_seen": 1617494016
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120682046138415,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5213980390437527,
+ "tokens_seen": 1617559552
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120521564694082,
+ "loss": 1.2673,
+ "theoretical_loss": 3.521385741730184,
+ "tokens_seen": 1617625088
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120361083249749,
+ "loss": 1.3548,
+ "theoretical_loss": 3.5213734450543077,
+ "tokens_seen": 1617690624
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120200601805416,
+ "loss": 1.2943,
+ "theoretical_loss": 3.521361149016064,
+ "tokens_seen": 1617756160
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004120040120361083,
+ "loss": 1.3091,
+ "theoretical_loss": 3.521348853615395,
+ "tokens_seen": 1617821696
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004119879638916751,
+ "loss": 1.354,
+ "theoretical_loss": 3.5213365588522416,
+ "tokens_seen": 1617887232
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004119719157472417,
+ "loss": 1.3145,
+ "theoretical_loss": 3.5213242647265446,
+ "tokens_seen": 1617952768
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004119558676028085,
+ "loss": 1.2963,
+ "theoretical_loss": 3.521311971238245,
+ "tokens_seen": 1618018304
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004119398194583751,
+ "loss": 1.321,
+ "theoretical_loss": 3.521299678387284,
+ "tokens_seen": 1618083840
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004119237713139419,
+ "loss": 1.3095,
+ "theoretical_loss": 3.5212873861736034,
+ "tokens_seen": 1618149376
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041190772316950853,
+ "loss": 1.2735,
+ "theoretical_loss": 3.5212750945971436,
+ "tokens_seen": 1618214912
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004118916750250753,
+ "loss": 1.367,
+ "theoretical_loss": 3.521262803657846,
+ "tokens_seen": 1618280448
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041187562688064193,
+ "loss": 1.2877,
+ "theoretical_loss": 3.521250513355652,
+ "tokens_seen": 1618345984
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004118595787362087,
+ "loss": 1.306,
+ "theoretical_loss": 3.5212382236905024,
+ "tokens_seen": 1618411520
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041184353059177533,
+ "loss": 1.2761,
+ "theoretical_loss": 3.521225934662339,
+ "tokens_seen": 1618477056
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004118274824473421,
+ "loss": 1.327,
+ "theoretical_loss": 3.521213646271102,
+ "tokens_seen": 1618542592
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041181143430290873,
+ "loss": 1.3305,
+ "theoretical_loss": 3.521201358516733,
+ "tokens_seen": 1618608128
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004117953861584755,
+ "loss": 1.2317,
+ "theoretical_loss": 3.5211890713991743,
+ "tokens_seen": 1618673664
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 542790,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.221559762954712,
+ "objective/train/theoretical_loss": 3.5211767849183655,
+ "objective/train/tokens_used": 1639199200,
+ "theoretical_loss": 3.5211767849183655,
+ "tokens_seen": 1618739200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041177933801404213,
+ "loss": 1.2886,
+ "theoretical_loss": 3.5211767849183655,
+ "tokens_seen": 1618739200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004117632898696089,
+ "loss": 1.3146,
+ "theoretical_loss": 3.521164499074249,
+ "tokens_seen": 1618804736
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041174724172517553,
+ "loss": 1.2495,
+ "theoretical_loss": 3.521152213866765,
+ "tokens_seen": 1618870272
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004117311935807423,
+ "loss": 1.2029,
+ "theoretical_loss": 3.521139929295856,
+ "tokens_seen": 1618935808
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041171514543630894,
+ "loss": 1.2901,
+ "theoretical_loss": 3.521127645361462,
+ "tokens_seen": 1619001344
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004116990972918757,
+ "loss": 1.3533,
+ "theoretical_loss": 3.521115362063525,
+ "tokens_seen": 1619066880
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041168304914744234,
+ "loss": 1.3077,
+ "theoretical_loss": 3.5211030794019855,
+ "tokens_seen": 1619132416
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004116670010030091,
+ "loss": 1.4194,
+ "theoretical_loss": 3.521090797376786,
+ "tokens_seen": 1619197952
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041165095285857574,
+ "loss": 1.292,
+ "theoretical_loss": 3.521078515987867,
+ "tokens_seen": 1619263488
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004116349047141425,
+ "loss": 1.337,
+ "theoretical_loss": 3.5210662352351694,
+ "tokens_seen": 1619329024
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041161885656970914,
+ "loss": 1.2442,
+ "theoretical_loss": 3.521053955118636,
+ "tokens_seen": 1619394560
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004116028084252759,
+ "loss": 1.4051,
+ "theoretical_loss": 3.521041675638206,
+ "tokens_seen": 1619460096
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041158676028084254,
+ "loss": 1.329,
+ "theoretical_loss": 3.521029396793822,
+ "tokens_seen": 1619525632
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004115707121364093,
+ "loss": 1.345,
+ "theoretical_loss": 3.521017118585426,
+ "tokens_seen": 1619591168
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041155466399197594,
+ "loss": 1.2474,
+ "theoretical_loss": 3.5210048410129575,
+ "tokens_seen": 1619656704
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004115386158475427,
+ "loss": 1.294,
+ "theoretical_loss": 3.5209925640763595,
+ "tokens_seen": 1619722240
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041152256770310934,
+ "loss": 1.3307,
+ "theoretical_loss": 3.520980287775572,
+ "tokens_seen": 1619787776
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004115065195586761,
+ "loss": 1.2483,
+ "theoretical_loss": 3.5209680121105373,
+ "tokens_seen": 1619853312
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041149047141424274,
+ "loss": 1.2847,
+ "theoretical_loss": 3.520955737081197,
+ "tokens_seen": 1619918848
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004114744232698095,
+ "loss": 1.4058,
+ "theoretical_loss": 3.520943462687491,
+ "tokens_seen": 1619984384
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041145837512537615,
+ "loss": 1.2474,
+ "theoretical_loss": 3.5209311889293615,
+ "tokens_seen": 1620049920
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041144232698094285,
+ "loss": 1.3477,
+ "theoretical_loss": 3.5209189158067504,
+ "tokens_seen": 1620115456
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041142627883650955,
+ "loss": 1.2323,
+ "theoretical_loss": 3.5209066433195986,
+ "tokens_seen": 1620180992
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041141023069207625,
+ "loss": 1.3001,
+ "theoretical_loss": 3.5208943714678473,
+ "tokens_seen": 1620246528
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041139418254764295,
+ "loss": 1.3605,
+ "theoretical_loss": 3.5208821002514386,
+ "tokens_seen": 1620312064
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041137813440320965,
+ "loss": 1.3282,
+ "theoretical_loss": 3.520869829670313,
+ "tokens_seen": 1620377600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041136208625877635,
+ "loss": 1.2522,
+ "theoretical_loss": 3.5208575597244125,
+ "tokens_seen": 1620443136
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041134603811434305,
+ "loss": 1.2853,
+ "theoretical_loss": 3.5208452904136784,
+ "tokens_seen": 1620508672
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041132998996990975,
+ "loss": 1.2942,
+ "theoretical_loss": 3.520833021738052,
+ "tokens_seen": 1620574208
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041131394182547645,
+ "loss": 1.3175,
+ "theoretical_loss": 3.5208207536974747,
+ "tokens_seen": 1620639744
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004112978936810431,
+ "loss": 1.3737,
+ "theoretical_loss": 3.520808486291889,
+ "tokens_seen": 1620705280
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041128184553660985,
+ "loss": 1.1986,
+ "theoretical_loss": 3.520796219521235,
+ "tokens_seen": 1620770816
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004112657973921765,
+ "loss": 1.2954,
+ "theoretical_loss": 3.520783953385454,
+ "tokens_seen": 1620836352
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041124974924774325,
+ "loss": 1.2996,
+ "theoretical_loss": 3.520771687884489,
+ "tokens_seen": 1620901888
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004112337011033099,
+ "loss": 1.322,
+ "theoretical_loss": 3.52075942301828,
+ "tokens_seen": 1620967424
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041121765295887666,
+ "loss": 1.397,
+ "theoretical_loss": 3.5207471587867696,
+ "tokens_seen": 1621032960
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004112016048144433,
+ "loss": 1.2729,
+ "theoretical_loss": 3.5207348951898982,
+ "tokens_seen": 1621098496
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041118555667001006,
+ "loss": 1.3236,
+ "theoretical_loss": 3.520722632227608,
+ "tokens_seen": 1621164032
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004111695085255767,
+ "loss": 1.3361,
+ "theoretical_loss": 3.520710369899841,
+ "tokens_seen": 1621229568
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041115346038114346,
+ "loss": 1.3386,
+ "theoretical_loss": 3.520698108206538,
+ "tokens_seen": 1621295104
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004111374122367101,
+ "loss": 1.3073,
+ "theoretical_loss": 3.5206858471476403,
+ "tokens_seen": 1621360640
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041112136409227686,
+ "loss": 1.434,
+ "theoretical_loss": 3.52067358672309,
+ "tokens_seen": 1621426176
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004111053159478435,
+ "loss": 1.3304,
+ "theoretical_loss": 3.520661326932828,
+ "tokens_seen": 1621491712
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041108926780341026,
+ "loss": 1.2513,
+ "theoretical_loss": 3.5206490677767968,
+ "tokens_seen": 1621557248
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004110732196589769,
+ "loss": 1.2836,
+ "theoretical_loss": 3.5206368092549374,
+ "tokens_seen": 1621622784
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041105717151454366,
+ "loss": 1.2449,
+ "theoretical_loss": 3.5206245513671908,
+ "tokens_seen": 1621688320
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004110411233701103,
+ "loss": 1.2701,
+ "theoretical_loss": 3.5206122941134996,
+ "tokens_seen": 1621753856
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041102507522567706,
+ "loss": 1.33,
+ "theoretical_loss": 3.5206000374938053,
+ "tokens_seen": 1621819392
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004110090270812437,
+ "loss": 1.3547,
+ "theoretical_loss": 3.520587781508049,
+ "tokens_seen": 1621884928
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041099297893681046,
+ "loss": 1.3295,
+ "theoretical_loss": 3.520575526156172,
+ "tokens_seen": 1621950464
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 543728,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1965467929840088,
+ "objective/train/theoretical_loss": 3.5205632714381165,
+ "objective/train/tokens_used": 1642476000,
+ "theoretical_loss": 3.5205632714381165,
+ "tokens_seen": 1622016000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004109769307923771,
+ "loss": 1.1936,
+ "theoretical_loss": 3.5205632714381165,
+ "tokens_seen": 1622016000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041096088264794387,
+ "loss": 1.2342,
+ "theoretical_loss": 3.5205510173538244,
+ "tokens_seen": 1622081536
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004109448345035105,
+ "loss": 1.3057,
+ "theoretical_loss": 3.520538763903237,
+ "tokens_seen": 1622147072
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041092878635907727,
+ "loss": 1.301,
+ "theoretical_loss": 3.5205265110862953,
+ "tokens_seen": 1622212608
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004109127382146439,
+ "loss": 1.3495,
+ "theoretical_loss": 3.5205142589029412,
+ "tokens_seen": 1622278144
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041089669007021067,
+ "loss": 1.3808,
+ "theoretical_loss": 3.5205020073531172,
+ "tokens_seen": 1622343680
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004108806419257773,
+ "loss": 1.3109,
+ "theoretical_loss": 3.5204897564367643,
+ "tokens_seen": 1622409216
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041086459378134407,
+ "loss": 1.265,
+ "theoretical_loss": 3.520477506153824,
+ "tokens_seen": 1622474752
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004108485456369107,
+ "loss": 1.3811,
+ "theoretical_loss": 3.520465256504238,
+ "tokens_seen": 1622540288
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041083249749247747,
+ "loss": 1.284,
+ "theoretical_loss": 3.5204530074879488,
+ "tokens_seen": 1622605824
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004108164493480441,
+ "loss": 1.2275,
+ "theoretical_loss": 3.520440759104897,
+ "tokens_seen": 1622671360
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041080040120361087,
+ "loss": 1.3408,
+ "theoretical_loss": 3.5204285113550244,
+ "tokens_seen": 1622736896
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041078435305917757,
+ "loss": 1.3113,
+ "theoretical_loss": 3.5204162642382735,
+ "tokens_seen": 1622802432
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041076830491474427,
+ "loss": 1.2345,
+ "theoretical_loss": 3.5204040177545854,
+ "tokens_seen": 1622867968
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000410752256770311,
+ "loss": 1.3309,
+ "theoretical_loss": 3.5203917719039017,
+ "tokens_seen": 1622933504
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004107362086258777,
+ "loss": 1.3104,
+ "theoretical_loss": 3.520379526686165,
+ "tokens_seen": 1622999040
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004107201604814444,
+ "loss": 1.3521,
+ "theoretical_loss": 3.520367282101316,
+ "tokens_seen": 1623064576
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004107041123370111,
+ "loss": 1.2375,
+ "theoretical_loss": 3.5203550381492965,
+ "tokens_seen": 1623130112
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106880641925778,
+ "loss": 1.367,
+ "theoretical_loss": 3.520342794830049,
+ "tokens_seen": 1623195648
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106720160481444,
+ "loss": 1.36,
+ "theoretical_loss": 3.520330552143515,
+ "tokens_seen": 1623261184
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106559679037112,
+ "loss": 1.3016,
+ "theoretical_loss": 3.520318310089636,
+ "tokens_seen": 1623326720
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106399197592778,
+ "loss": 1.2955,
+ "theoretical_loss": 3.5203060686683534,
+ "tokens_seen": 1623392256
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106238716148446,
+ "loss": 1.3719,
+ "theoretical_loss": 3.52029382787961,
+ "tokens_seen": 1623457792
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004106078234704112,
+ "loss": 1.2813,
+ "theoretical_loss": 3.5202815877233467,
+ "tokens_seen": 1623523328
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000410591775325978,
+ "loss": 1.2415,
+ "theoretical_loss": 3.5202693481995055,
+ "tokens_seen": 1623588864
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004105757271815446,
+ "loss": 1.2761,
+ "theoretical_loss": 3.5202571093080284,
+ "tokens_seen": 1623654400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004105596790371114,
+ "loss": 1.2857,
+ "theoretical_loss": 3.520244871048857,
+ "tokens_seen": 1623719936
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041054363089267803,
+ "loss": 1.3139,
+ "theoretical_loss": 3.5202326334219336,
+ "tokens_seen": 1623785472
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004105275827482448,
+ "loss": 1.2739,
+ "theoretical_loss": 3.520220396427199,
+ "tokens_seen": 1623851008
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041051153460381143,
+ "loss": 1.2938,
+ "theoretical_loss": 3.5202081600645965,
+ "tokens_seen": 1623916544
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004104954864593782,
+ "loss": 1.2656,
+ "theoretical_loss": 3.520195924334067,
+ "tokens_seen": 1623982080
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041047943831494483,
+ "loss": 1.3287,
+ "theoretical_loss": 3.5201836892355516,
+ "tokens_seen": 1624047616
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004104633901705116,
+ "loss": 1.315,
+ "theoretical_loss": 3.5201714547689935,
+ "tokens_seen": 1624113152
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041044734202607823,
+ "loss": 1.3726,
+ "theoretical_loss": 3.5201592209343344,
+ "tokens_seen": 1624178688
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000410431293881645,
+ "loss": 1.2545,
+ "theoretical_loss": 3.5201469877315157,
+ "tokens_seen": 1624244224
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041041524573721163,
+ "loss": 1.3051,
+ "theoretical_loss": 3.520134755160479,
+ "tokens_seen": 1624309760
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004103991975927784,
+ "loss": 1.297,
+ "theoretical_loss": 3.520122523221167,
+ "tokens_seen": 1624375296
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041038314944834503,
+ "loss": 1.3264,
+ "theoretical_loss": 3.520110291913521,
+ "tokens_seen": 1624440832
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004103671013039118,
+ "loss": 1.2975,
+ "theoretical_loss": 3.5200980612374835,
+ "tokens_seen": 1624506368
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041035105315947844,
+ "loss": 1.3084,
+ "theoretical_loss": 3.5200858311929952,
+ "tokens_seen": 1624571904
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004103350050150452,
+ "loss": 1.3527,
+ "theoretical_loss": 3.520073601779999,
+ "tokens_seen": 1624637440
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041031895687061184,
+ "loss": 1.203,
+ "theoretical_loss": 3.5200613729984376,
+ "tokens_seen": 1624702976
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004103029087261786,
+ "loss": 1.2778,
+ "theoretical_loss": 3.520049144848251,
+ "tokens_seen": 1624768512
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041028686058174524,
+ "loss": 1.3016,
+ "theoretical_loss": 3.5200369173293824,
+ "tokens_seen": 1624834048
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000410270812437312,
+ "loss": 1.2638,
+ "theoretical_loss": 3.5200246904417734,
+ "tokens_seen": 1624899584
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041025476429287864,
+ "loss": 1.2902,
+ "theoretical_loss": 3.5200124641853656,
+ "tokens_seen": 1624965120
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004102387161484454,
+ "loss": 1.3101,
+ "theoretical_loss": 3.520000238560102,
+ "tokens_seen": 1625030656
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041022266800401204,
+ "loss": 1.2553,
+ "theoretical_loss": 3.519988013565923,
+ "tokens_seen": 1625096192
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004102066198595788,
+ "loss": 1.2852,
+ "theoretical_loss": 3.5199757892027725,
+ "tokens_seen": 1625161728
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041019057171514544,
+ "loss": 1.3299,
+ "theoretical_loss": 3.519963565470591,
+ "tokens_seen": 1625227264
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 545041,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.421590805053711,
+ "objective/train/theoretical_loss": 3.519951342369321,
+ "objective/train/tokens_used": 1645752800,
+ "theoretical_loss": 3.519951342369321,
+ "tokens_seen": 1625292800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004101745235707122,
+ "loss": 1.434,
+ "theoretical_loss": 3.519951342369321,
+ "tokens_seen": 1625292800
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041015847542627884,
+ "loss": 1.3512,
+ "theoretical_loss": 3.5199391198989045,
+ "tokens_seen": 1625358336
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004101424272818456,
+ "loss": 1.2329,
+ "theoretical_loss": 3.5199268980592837,
+ "tokens_seen": 1625423872
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041012637913741224,
+ "loss": 1.3568,
+ "theoretical_loss": 3.5199146768503997,
+ "tokens_seen": 1625489408
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000410110330992979,
+ "loss": 1.1842,
+ "theoretical_loss": 3.519902456272196,
+ "tokens_seen": 1625554944
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041009428284854565,
+ "loss": 1.3386,
+ "theoretical_loss": 3.5198902363246134,
+ "tokens_seen": 1625620480
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004100782347041124,
+ "loss": 1.3737,
+ "theoretical_loss": 3.5198780170075947,
+ "tokens_seen": 1625686016
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041006218655967905,
+ "loss": 1.3289,
+ "theoretical_loss": 3.519865798321081,
+ "tokens_seen": 1625751552
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041004613841524575,
+ "loss": 1.3245,
+ "theoretical_loss": 3.519853580265015,
+ "tokens_seen": 1625817088
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041003009027081245,
+ "loss": 1.2956,
+ "theoretical_loss": 3.5198413628393395,
+ "tokens_seen": 1625882624
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00041001404212637915,
+ "loss": 1.2515,
+ "theoretical_loss": 3.519829146043995,
+ "tokens_seen": 1625948160
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040999799398194585,
+ "loss": 1.3645,
+ "theoretical_loss": 3.519816929878925,
+ "tokens_seen": 1626013696
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040998194583751255,
+ "loss": 1.2441,
+ "theoretical_loss": 3.5198047143440707,
+ "tokens_seen": 1626079232
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040996589769307925,
+ "loss": 1.2035,
+ "theoretical_loss": 3.519792499439374,
+ "tokens_seen": 1626144768
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040994984954864595,
+ "loss": 1.2841,
+ "theoretical_loss": 3.519780285164778,
+ "tokens_seen": 1626210304
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040993380140421265,
+ "loss": 1.3427,
+ "theoretical_loss": 3.5197680715202244,
+ "tokens_seen": 1626275840
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040991775325977935,
+ "loss": 1.2832,
+ "theoretical_loss": 3.519755858505655,
+ "tokens_seen": 1626341376
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000409901705115346,
+ "loss": 1.3184,
+ "theoretical_loss": 3.519743646121012,
+ "tokens_seen": 1626406912
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040988565697091275,
+ "loss": 1.3831,
+ "theoretical_loss": 3.5197314343662374,
+ "tokens_seen": 1626472448
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004098696088264794,
+ "loss": 1.3433,
+ "theoretical_loss": 3.5197192232412733,
+ "tokens_seen": 1626537984
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040985356068204615,
+ "loss": 1.2464,
+ "theoretical_loss": 3.5197070127460623,
+ "tokens_seen": 1626603520
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004098375125376128,
+ "loss": 1.2645,
+ "theoretical_loss": 3.519694802880547,
+ "tokens_seen": 1626669056
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040982146439317956,
+ "loss": 1.329,
+ "theoretical_loss": 3.519682593644668,
+ "tokens_seen": 1626734592
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004098054162487462,
+ "loss": 1.2149,
+ "theoretical_loss": 3.519670385038369,
+ "tokens_seen": 1626800128
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040978936810431296,
+ "loss": 1.191,
+ "theoretical_loss": 3.5196581770615913,
+ "tokens_seen": 1626865664
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004097733199598796,
+ "loss": 1.278,
+ "theoretical_loss": 3.5196459697142775,
+ "tokens_seen": 1626931200
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040975727181544636,
+ "loss": 1.2842,
+ "theoretical_loss": 3.519633762996369,
+ "tokens_seen": 1626996736
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000409741223671013,
+ "loss": 1.2341,
+ "theoretical_loss": 3.5196215569078095,
+ "tokens_seen": 1627062272
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040972517552657976,
+ "loss": 1.1918,
+ "theoretical_loss": 3.5196093514485396,
+ "tokens_seen": 1627127808
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004097091273821464,
+ "loss": 1.3009,
+ "theoretical_loss": 3.5195971466185023,
+ "tokens_seen": 1627193344
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040969307923771316,
+ "loss": 1.2998,
+ "theoretical_loss": 3.51958494241764,
+ "tokens_seen": 1627258880
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004096770310932798,
+ "loss": 1.259,
+ "theoretical_loss": 3.519572738845895,
+ "tokens_seen": 1627324416
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040966098294884656,
+ "loss": 1.3007,
+ "theoretical_loss": 3.5195605359032087,
+ "tokens_seen": 1627389952
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004096449348044132,
+ "loss": 1.2726,
+ "theoretical_loss": 3.519548333589524,
+ "tokens_seen": 1627455488
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040962888665997996,
+ "loss": 1.3173,
+ "theoretical_loss": 3.5195361319047826,
+ "tokens_seen": 1627521024
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004096128385155466,
+ "loss": 1.1967,
+ "theoretical_loss": 3.519523930848928,
+ "tokens_seen": 1627586560
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040959679037111336,
+ "loss": 1.326,
+ "theoretical_loss": 3.5195117304219012,
+ "tokens_seen": 1627652096
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004095807422266801,
+ "loss": 1.2693,
+ "theoretical_loss": 3.5194995306236443,
+ "tokens_seen": 1627717632
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040956469408224677,
+ "loss": 1.37,
+ "theoretical_loss": 3.5194873314541013,
+ "tokens_seen": 1627783168
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004095486459378135,
+ "loss": 1.2091,
+ "theoretical_loss": 3.519475132913213,
+ "tokens_seen": 1627848704
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040953259779338017,
+ "loss": 1.3749,
+ "theoretical_loss": 3.5194629350009214,
+ "tokens_seen": 1627914240
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004095165496489469,
+ "loss": 1.1392,
+ "theoretical_loss": 3.51945073771717,
+ "tokens_seen": 1627979776
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040950050150451357,
+ "loss": 1.1767,
+ "theoretical_loss": 3.5194385410619002,
+ "tokens_seen": 1628045312
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004094844533600803,
+ "loss": 1.2558,
+ "theoretical_loss": 3.5194263450350554,
+ "tokens_seen": 1628110848
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040946840521564697,
+ "loss": 1.3181,
+ "theoretical_loss": 3.519414149636577,
+ "tokens_seen": 1628176384
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004094523570712137,
+ "loss": 1.3602,
+ "theoretical_loss": 3.5194019548664075,
+ "tokens_seen": 1628241920
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040943630892678037,
+ "loss": 1.267,
+ "theoretical_loss": 3.519389760724489,
+ "tokens_seen": 1628307456
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040942026078234707,
+ "loss": 1.2681,
+ "theoretical_loss": 3.5193775672107646,
+ "tokens_seen": 1628372992
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040940421263791377,
+ "loss": 1.2576,
+ "theoretical_loss": 3.519365374325176,
+ "tokens_seen": 1628438528
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004093881644934805,
+ "loss": 1.3498,
+ "theoretical_loss": 3.5193531820676656,
+ "tokens_seen": 1628504064
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 545970,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.416658878326416,
+ "objective/train/theoretical_loss": 3.5193409904381765,
+ "objective/train/tokens_used": 1649029600,
+ "theoretical_loss": 3.5193409904381765,
+ "tokens_seen": 1628569600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004093721163490472,
+ "loss": 1.3766,
+ "theoretical_loss": 3.5193409904381765,
+ "tokens_seen": 1628569600
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004093560682046139,
+ "loss": 1.2934,
+ "theoretical_loss": 3.51932879943665,
+ "tokens_seen": 1628635136
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004093400200601806,
+ "loss": 1.3387,
+ "theoretical_loss": 3.5193166090630292,
+ "tokens_seen": 1628700672
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004093239719157473,
+ "loss": 1.3228,
+ "theoretical_loss": 3.5193044193172565,
+ "tokens_seen": 1628766208
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000409307923771314,
+ "loss": 1.2837,
+ "theoretical_loss": 3.519292230199274,
+ "tokens_seen": 1628831744
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092918756268807,
+ "loss": 1.3637,
+ "theoretical_loss": 3.519280041709024,
+ "tokens_seen": 1628897280
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092758274824473,
+ "loss": 1.2393,
+ "theoretical_loss": 3.5192678538464492,
+ "tokens_seen": 1628962816
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092597793380141,
+ "loss": 1.3145,
+ "theoretical_loss": 3.5192556666114925,
+ "tokens_seen": 1629028352
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092437311935807,
+ "loss": 1.3558,
+ "theoretical_loss": 3.5192434800040955,
+ "tokens_seen": 1629093888
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092276830491475,
+ "loss": 1.2483,
+ "theoretical_loss": 3.5192312940242005,
+ "tokens_seen": 1629159424
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004092116349047141,
+ "loss": 1.2581,
+ "theoretical_loss": 3.519219108671751,
+ "tokens_seen": 1629224960
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004091955867602809,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5192069239466885,
+ "tokens_seen": 1629290496
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040917953861584753,
+ "loss": 1.4091,
+ "theoretical_loss": 3.519194739848956,
+ "tokens_seen": 1629356032
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004091634904714143,
+ "loss": 1.2028,
+ "theoretical_loss": 3.5191825563784955,
+ "tokens_seen": 1629421568
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040914744232698093,
+ "loss": 1.23,
+ "theoretical_loss": 3.5191703735352498,
+ "tokens_seen": 1629487104
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004091313941825477,
+ "loss": 1.307,
+ "theoretical_loss": 3.5191581913191614,
+ "tokens_seen": 1629552640
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040911534603811433,
+ "loss": 1.3266,
+ "theoretical_loss": 3.519146009730173,
+ "tokens_seen": 1629618176
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004090992978936811,
+ "loss": 1.3664,
+ "theoretical_loss": 3.519133828768227,
+ "tokens_seen": 1629683712
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040908324974924773,
+ "loss": 1.2227,
+ "theoretical_loss": 3.5191216484332655,
+ "tokens_seen": 1629749248
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004090672016048145,
+ "loss": 1.2636,
+ "theoretical_loss": 3.5191094687252313,
+ "tokens_seen": 1629814784
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040905115346038113,
+ "loss": 1.2572,
+ "theoretical_loss": 3.519097289644067,
+ "tokens_seen": 1629880320
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004090351053159479,
+ "loss": 1.2375,
+ "theoretical_loss": 3.519085111189715,
+ "tokens_seen": 1629945856
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040901905717151453,
+ "loss": 1.2524,
+ "theoretical_loss": 3.519072933362118,
+ "tokens_seen": 1630011392
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004090030090270813,
+ "loss": 1.3324,
+ "theoretical_loss": 3.5190607561612177,
+ "tokens_seen": 1630076928
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040898696088264793,
+ "loss": 1.399,
+ "theoretical_loss": 3.5190485795869577,
+ "tokens_seen": 1630142464
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004089709127382147,
+ "loss": 1.3517,
+ "theoretical_loss": 3.519036403639281,
+ "tokens_seen": 1630208000
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040895486459378134,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5190242283181288,
+ "tokens_seen": 1630273536
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004089388164493481,
+ "loss": 1.1723,
+ "theoretical_loss": 3.519012053623444,
+ "tokens_seen": 1630339072
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040892276830491474,
+ "loss": 1.3063,
+ "theoretical_loss": 3.5189998795551696,
+ "tokens_seen": 1630404608
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004089067201604815,
+ "loss": 1.2878,
+ "theoretical_loss": 3.5189877061132484,
+ "tokens_seen": 1630470144
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040889067201604814,
+ "loss": 1.3226,
+ "theoretical_loss": 3.5189755332976222,
+ "tokens_seen": 1630535680
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004088746238716149,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5189633611082347,
+ "tokens_seen": 1630601216
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040885857572718154,
+ "loss": 1.2736,
+ "theoretical_loss": 3.518951189545027,
+ "tokens_seen": 1630666752
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004088425275827483,
+ "loss": 1.2574,
+ "theoretical_loss": 3.5189390186079437,
+ "tokens_seen": 1630732288
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040882647943831494,
+ "loss": 1.3744,
+ "theoretical_loss": 3.5189268482969256,
+ "tokens_seen": 1630797824
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004088104312938817,
+ "loss": 1.3011,
+ "theoretical_loss": 3.5189146786119165,
+ "tokens_seen": 1630863360
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040879438314944834,
+ "loss": 1.3871,
+ "theoretical_loss": 3.5189025095528583,
+ "tokens_seen": 1630928896
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004087783350050151,
+ "loss": 1.3542,
+ "theoretical_loss": 3.518890341119694,
+ "tokens_seen": 1630994432
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040876228686058174,
+ "loss": 1.259,
+ "theoretical_loss": 3.518878173312366,
+ "tokens_seen": 1631059968
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004087462387161485,
+ "loss": 1.3816,
+ "theoretical_loss": 3.518866006130818,
+ "tokens_seen": 1631125504
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040873019057171515,
+ "loss": 1.3348,
+ "theoretical_loss": 3.5188538395749918,
+ "tokens_seen": 1631191040
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004087141424272819,
+ "loss": 1.2186,
+ "theoretical_loss": 3.5188416736448294,
+ "tokens_seen": 1631256576
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040869809428284855,
+ "loss": 1.2514,
+ "theoretical_loss": 3.518829508340275,
+ "tokens_seen": 1631322112
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004086820461384153,
+ "loss": 1.3279,
+ "theoretical_loss": 3.5188173436612695,
+ "tokens_seen": 1631387648
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040866599799398195,
+ "loss": 1.2843,
+ "theoretical_loss": 3.5188051796077575,
+ "tokens_seen": 1631453184
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040864994984954865,
+ "loss": 1.1867,
+ "theoretical_loss": 3.518793016179681,
+ "tokens_seen": 1631518720
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040863390170511535,
+ "loss": 1.2986,
+ "theoretical_loss": 3.518780853376982,
+ "tokens_seen": 1631584256
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040861785356068205,
+ "loss": 1.2974,
+ "theoretical_loss": 3.518768691199605,
+ "tokens_seen": 1631649792
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040860180541624875,
+ "loss": 1.2806,
+ "theoretical_loss": 3.51875652964749,
+ "tokens_seen": 1631715328
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040858575727181545,
+ "loss": 1.2822,
+ "theoretical_loss": 3.518744368720582,
+ "tokens_seen": 1631780864
+ },
+ {
+ "epoch": 0.49,
+ "objective/train/docs_used": 547214,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2601799964904785,
+ "objective/train/theoretical_loss": 3.5187322084188235,
+ "objective/train/tokens_used": 1652306400,
+ "theoretical_loss": 3.5187322084188235,
+ "tokens_seen": 1631846400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040856970912738215,
+ "loss": 1.2906,
+ "theoretical_loss": 3.5187322084188235,
+ "tokens_seen": 1631846400
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040855366098294885,
+ "loss": 1.3311,
+ "theoretical_loss": 3.518720048742156,
+ "tokens_seen": 1631911936
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040853761283851555,
+ "loss": 1.2151,
+ "theoretical_loss": 3.518707889690524,
+ "tokens_seen": 1631977472
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040852156469408225,
+ "loss": 1.2723,
+ "theoretical_loss": 3.5186957312638683,
+ "tokens_seen": 1632043008
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004085055165496489,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5186835734621336,
+ "tokens_seen": 1632108544
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040848946840521565,
+ "loss": 1.2423,
+ "theoretical_loss": 3.5186714162852613,
+ "tokens_seen": 1632174080
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004084734202607823,
+ "loss": 1.3532,
+ "theoretical_loss": 3.518659259733195,
+ "tokens_seen": 1632239616
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040845737211634906,
+ "loss": 1.3451,
+ "theoretical_loss": 3.5186471038058773,
+ "tokens_seen": 1632305152
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004084413239719157,
+ "loss": 1.3717,
+ "theoretical_loss": 3.5186349485032506,
+ "tokens_seen": 1632370688
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040842527582748246,
+ "loss": 1.2511,
+ "theoretical_loss": 3.518622793825258,
+ "tokens_seen": 1632436224
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004084092276830491,
+ "loss": 1.2722,
+ "theoretical_loss": 3.518610639771843,
+ "tokens_seen": 1632501760
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040839317953861586,
+ "loss": 1.2873,
+ "theoretical_loss": 3.518598486342947,
+ "tokens_seen": 1632567296
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004083771313941826,
+ "loss": 1.3081,
+ "theoretical_loss": 3.518586333538514,
+ "tokens_seen": 1632632832
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040836108324974926,
+ "loss": 1.2417,
+ "theoretical_loss": 3.518574181358487,
+ "tokens_seen": 1632698368
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000408345035105316,
+ "loss": 1.399,
+ "theoretical_loss": 3.5185620298028075,
+ "tokens_seen": 1632763904
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040832898696088266,
+ "loss": 1.2732,
+ "theoretical_loss": 3.5185498788714202,
+ "tokens_seen": 1632829440
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004083129388164494,
+ "loss": 1.3301,
+ "theoretical_loss": 3.518537728564266,
+ "tokens_seen": 1632894976
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040829689067201606,
+ "loss": 1.3286,
+ "theoretical_loss": 3.51852557888129,
+ "tokens_seen": 1632960512
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004082808425275828,
+ "loss": 1.3132,
+ "theoretical_loss": 3.518513429822433,
+ "tokens_seen": 1633026048
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040826479438314946,
+ "loss": 1.3872,
+ "theoretical_loss": 3.518501281387639,
+ "tokens_seen": 1633091584
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004082487462387162,
+ "loss": 1.3259,
+ "theoretical_loss": 3.5184891335768507,
+ "tokens_seen": 1633157120
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040823269809428286,
+ "loss": 1.2861,
+ "theoretical_loss": 3.5184769863900103,
+ "tokens_seen": 1633222656
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.0004082166499498496,
+ "loss": 1.2396,
+ "theoretical_loss": 3.518464839827062,
+ "tokens_seen": 1633288192
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040820060180541627,
+ "loss": 1.3408,
+ "theoretical_loss": 3.5184526938879483,
+ "tokens_seen": 1633353728
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.000408184553660983,
+ "loss": 1.3001,
+ "theoretical_loss": 3.518440548572612,
+ "tokens_seen": 1633419264
+ },
+ {
+ "epoch": 0.49,
+ "learning_rate": 0.00040816850551654967,
+ "loss": 1.3264,
+ "theoretical_loss": 3.5184284038809954,
+ "tokens_seen": 1633484800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004081524573721164,
+ "loss": 1.2697,
+ "theoretical_loss": 3.5184162598130424,
+ "tokens_seen": 1633550336
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040813640922768307,
+ "loss": 1.3854,
+ "theoretical_loss": 3.5184041163686954,
+ "tokens_seen": 1633615872
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004081203610832498,
+ "loss": 1.3554,
+ "theoretical_loss": 3.5183919735478977,
+ "tokens_seen": 1633681408
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040810431293881647,
+ "loss": 1.2912,
+ "theoretical_loss": 3.5183798313505923,
+ "tokens_seen": 1633746944
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004080882647943832,
+ "loss": 1.2618,
+ "theoretical_loss": 3.5183676897767215,
+ "tokens_seen": 1633812480
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040807221664994987,
+ "loss": 1.3732,
+ "theoretical_loss": 3.5183555488262295,
+ "tokens_seen": 1633878016
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004080561685055166,
+ "loss": 1.2545,
+ "theoretical_loss": 3.518343408499058,
+ "tokens_seen": 1633943552
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040804012036108327,
+ "loss": 1.3064,
+ "theoretical_loss": 3.5183312687951505,
+ "tokens_seen": 1634009088
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040802407221665,
+ "loss": 1.3529,
+ "theoretical_loss": 3.5183191297144507,
+ "tokens_seen": 1634074624
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004080080240722167,
+ "loss": 1.2502,
+ "theoretical_loss": 3.5183069912569005,
+ "tokens_seen": 1634140160
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079919759277834,
+ "loss": 1.2531,
+ "theoretical_loss": 3.5182948534224434,
+ "tokens_seen": 1634205696
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079759277833501,
+ "loss": 1.2107,
+ "theoretical_loss": 3.518282716211022,
+ "tokens_seen": 1634271232
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079598796389168,
+ "loss": 1.3794,
+ "theoretical_loss": 3.518270579622581,
+ "tokens_seen": 1634336768
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079438314944835,
+ "loss": 1.2892,
+ "theoretical_loss": 3.5182584436570616,
+ "tokens_seen": 1634402304
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079277833500502,
+ "loss": 1.2906,
+ "theoretical_loss": 3.5182463083144073,
+ "tokens_seen": 1634467840
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004079117352056169,
+ "loss": 1.3662,
+ "theoretical_loss": 3.5182341735945615,
+ "tokens_seen": 1634533376
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004078956870611836,
+ "loss": 1.194,
+ "theoretical_loss": 3.5182220394974673,
+ "tokens_seen": 1634598912
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004078796389167502,
+ "loss": 1.2531,
+ "theoretical_loss": 3.5182099060230674,
+ "tokens_seen": 1634664448
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000407863590772317,
+ "loss": 1.3946,
+ "theoretical_loss": 3.518197773171305,
+ "tokens_seen": 1634729984
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004078475426278836,
+ "loss": 1.2397,
+ "theoretical_loss": 3.5181856409421233,
+ "tokens_seen": 1634795520
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004078314944834504,
+ "loss": 1.3523,
+ "theoretical_loss": 3.5181735093354654,
+ "tokens_seen": 1634861056
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000407815446339017,
+ "loss": 1.3561,
+ "theoretical_loss": 3.5181613783512744,
+ "tokens_seen": 1634926592
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004077993981945838,
+ "loss": 1.317,
+ "theoretical_loss": 3.518149247989493,
+ "tokens_seen": 1634992128
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040778335005015043,
+ "loss": 1.4313,
+ "theoretical_loss": 3.518137118250065,
+ "tokens_seen": 1635057664
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 548116,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2663007974624634,
+ "objective/train/theoretical_loss": 3.518124989132933,
+ "objective/train/tokens_used": 1655583200,
+ "theoretical_loss": 3.518124989132933,
+ "tokens_seen": 1635123200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004077673019057172,
+ "loss": 1.28,
+ "theoretical_loss": 3.518124989132933,
+ "tokens_seen": 1635123200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040775125376128383,
+ "loss": 1.3439,
+ "theoretical_loss": 3.5181128606380407,
+ "tokens_seen": 1635188736
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004077352056168506,
+ "loss": 1.2223,
+ "theoretical_loss": 3.5181007327653306,
+ "tokens_seen": 1635254272
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040771915747241723,
+ "loss": 1.2541,
+ "theoretical_loss": 3.5180886055147464,
+ "tokens_seen": 1635319808
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000407703109327984,
+ "loss": 1.3701,
+ "theoretical_loss": 3.518076478886231,
+ "tokens_seen": 1635385344
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040768706118355063,
+ "loss": 1.2392,
+ "theoretical_loss": 3.5180643528797275,
+ "tokens_seen": 1635450880
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004076710130391174,
+ "loss": 1.313,
+ "theoretical_loss": 3.518052227495179,
+ "tokens_seen": 1635516416
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040765496489468403,
+ "loss": 1.3045,
+ "theoretical_loss": 3.5180401027325288,
+ "tokens_seen": 1635581952
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004076389167502508,
+ "loss": 1.3109,
+ "theoretical_loss": 3.51802797859172,
+ "tokens_seen": 1635647488
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040762286860581743,
+ "loss": 1.3282,
+ "theoretical_loss": 3.5180158550726963,
+ "tokens_seen": 1635713024
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004076068204613842,
+ "loss": 1.2817,
+ "theoretical_loss": 3.5180037321754,
+ "tokens_seen": 1635778560
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040759077231695084,
+ "loss": 1.507,
+ "theoretical_loss": 3.517991609899775,
+ "tokens_seen": 1635844096
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004075747241725176,
+ "loss": 1.2632,
+ "theoretical_loss": 3.5179794882457642,
+ "tokens_seen": 1635909632
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040755867602808424,
+ "loss": 1.371,
+ "theoretical_loss": 3.5179673672133114,
+ "tokens_seen": 1635975168
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000407542627883651,
+ "loss": 1.3235,
+ "theoretical_loss": 3.5179552468023587,
+ "tokens_seen": 1636040704
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040752657973921764,
+ "loss": 1.3234,
+ "theoretical_loss": 3.5179431270128507,
+ "tokens_seen": 1636106240
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004075105315947844,
+ "loss": 1.2365,
+ "theoretical_loss": 3.5179310078447297,
+ "tokens_seen": 1636171776
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040749448345035104,
+ "loss": 1.3097,
+ "theoretical_loss": 3.517918889297939,
+ "tokens_seen": 1636237312
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004074784353059178,
+ "loss": 1.3809,
+ "theoretical_loss": 3.517906771372422,
+ "tokens_seen": 1636302848
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040746238716148444,
+ "loss": 1.3125,
+ "theoretical_loss": 3.5178946540681224,
+ "tokens_seen": 1636368384
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004074463390170512,
+ "loss": 1.2686,
+ "theoretical_loss": 3.517882537384983,
+ "tokens_seen": 1636433920
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040743029087261784,
+ "loss": 1.3453,
+ "theoretical_loss": 3.5178704213229475,
+ "tokens_seen": 1636499456
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004074142427281846,
+ "loss": 1.2856,
+ "theoretical_loss": 3.5178583058819584,
+ "tokens_seen": 1636564992
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040739819458375124,
+ "loss": 1.2855,
+ "theoretical_loss": 3.5178461910619596,
+ "tokens_seen": 1636630528
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000407382146439318,
+ "loss": 1.3232,
+ "theoretical_loss": 3.5178340768628944,
+ "tokens_seen": 1636696064
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040736609829488464,
+ "loss": 1.2578,
+ "theoretical_loss": 3.517821963284706,
+ "tokens_seen": 1636761600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004073500501504514,
+ "loss": 1.2877,
+ "theoretical_loss": 3.517809850327337,
+ "tokens_seen": 1636827136
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040733400200601805,
+ "loss": 1.3605,
+ "theoretical_loss": 3.5177977379907324,
+ "tokens_seen": 1636892672
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004073179538615848,
+ "loss": 1.2611,
+ "theoretical_loss": 3.5177856262748346,
+ "tokens_seen": 1636958208
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040730190571715145,
+ "loss": 1.2498,
+ "theoretical_loss": 3.5177735151795866,
+ "tokens_seen": 1637023744
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004072858575727182,
+ "loss": 1.4699,
+ "theoretical_loss": 3.517761404704932,
+ "tokens_seen": 1637089280
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040726980942828485,
+ "loss": 1.2863,
+ "theoretical_loss": 3.5177492948508147,
+ "tokens_seen": 1637154816
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040725376128385155,
+ "loss": 1.2188,
+ "theoretical_loss": 3.517737185617177,
+ "tokens_seen": 1637220352
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040723771313941825,
+ "loss": 1.3205,
+ "theoretical_loss": 3.5177250770039636,
+ "tokens_seen": 1637285888
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040722166499498495,
+ "loss": 1.2454,
+ "theoretical_loss": 3.5177129690111166,
+ "tokens_seen": 1637351424
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004072056168505517,
+ "loss": 1.3755,
+ "theoretical_loss": 3.5177008616385805,
+ "tokens_seen": 1637416960
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040718956870611835,
+ "loss": 1.2752,
+ "theoretical_loss": 3.517688754886297,
+ "tokens_seen": 1637482496
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004071735205616851,
+ "loss": 1.3169,
+ "theoretical_loss": 3.517676648754212,
+ "tokens_seen": 1637548032
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040715747241725175,
+ "loss": 1.2806,
+ "theoretical_loss": 3.517664543242267,
+ "tokens_seen": 1637613568
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004071414242728185,
+ "loss": 1.2345,
+ "theoretical_loss": 3.5176524383504058,
+ "tokens_seen": 1637679104
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040712537612838515,
+ "loss": 1.2848,
+ "theoretical_loss": 3.517640334078572,
+ "tokens_seen": 1637744640
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004071093279839519,
+ "loss": 1.2902,
+ "theoretical_loss": 3.5176282304267095,
+ "tokens_seen": 1637810176
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040709327983951856,
+ "loss": 1.3012,
+ "theoretical_loss": 3.517616127394761,
+ "tokens_seen": 1637875712
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004070772316950853,
+ "loss": 1.3301,
+ "theoretical_loss": 3.51760402498267,
+ "tokens_seen": 1637941248
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040706118355065196,
+ "loss": 1.2662,
+ "theoretical_loss": 3.5175919231903805,
+ "tokens_seen": 1638006784
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004070451354062187,
+ "loss": 1.3013,
+ "theoretical_loss": 3.5175798220178356,
+ "tokens_seen": 1638072320
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040702908726178536,
+ "loss": 1.2981,
+ "theoretical_loss": 3.5175677214649785,
+ "tokens_seen": 1638137856
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004070130391173521,
+ "loss": 1.2958,
+ "theoretical_loss": 3.5175556215317534,
+ "tokens_seen": 1638203392
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040699699097291876,
+ "loss": 1.348,
+ "theoretical_loss": 3.5175435222181033,
+ "tokens_seen": 1638268928
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004069809428284855,
+ "loss": 1.3486,
+ "theoretical_loss": 3.5175314235239714,
+ "tokens_seen": 1638334464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.7322549377064853,
+ "debugging/entropy-1-grams": 5.133293776725962,
+ "debugging/length": 605.75,
+ "debugging/num_segments": 8,
+ "debugging/score": 0.044954849742907824,
+ "debugging/score_std": 0.025932746393242948,
+ "epoch": 0.5,
+ "objective/train/docs_used": 549338,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.18026864528656,
+ "objective/train/theoretical_loss": 3.517519325449302,
+ "objective/train/tokens_used": 1658860000,
+ "theoretical_loss": 3.517519325449302,
+ "tokens_seen": 1638400000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040696489468405216,
+ "loss": 1.1928,
+ "theoretical_loss": 3.517519325449302,
+ "tokens_seen": 1638400000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004069488465396189,
+ "loss": 1.3697,
+ "theoretical_loss": 3.517507227994038,
+ "tokens_seen": 1638465536
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040693279839518556,
+ "loss": 1.3529,
+ "theoretical_loss": 3.517495131158123,
+ "tokens_seen": 1638531072
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004069167502507523,
+ "loss": 1.1634,
+ "theoretical_loss": 3.5174830349415007,
+ "tokens_seen": 1638596608
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040690070210631896,
+ "loss": 1.3304,
+ "theoretical_loss": 3.517470939344114,
+ "tokens_seen": 1638662144
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004068846539618857,
+ "loss": 1.1961,
+ "theoretical_loss": 3.5174588443659074,
+ "tokens_seen": 1638727680
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040686860581745236,
+ "loss": 1.3084,
+ "theoretical_loss": 3.5174467500068243,
+ "tokens_seen": 1638793216
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004068525576730191,
+ "loss": 1.3103,
+ "theoretical_loss": 3.5174346562668077,
+ "tokens_seen": 1638858752
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040683650952858577,
+ "loss": 1.2826,
+ "theoretical_loss": 3.5174225631458014,
+ "tokens_seen": 1638924288
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004068204613841525,
+ "loss": 1.3464,
+ "theoretical_loss": 3.517410470643749,
+ "tokens_seen": 1638989824
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040680441323971917,
+ "loss": 1.2669,
+ "theoretical_loss": 3.5173983787605945,
+ "tokens_seen": 1639055360
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004067883650952859,
+ "loss": 1.2666,
+ "theoretical_loss": 3.5173862874962802,
+ "tokens_seen": 1639120896
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040677231695085257,
+ "loss": 1.2279,
+ "theoretical_loss": 3.517374196850751,
+ "tokens_seen": 1639186432
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004067562688064193,
+ "loss": 1.1905,
+ "theoretical_loss": 3.51736210682395,
+ "tokens_seen": 1639251968
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040674022066198597,
+ "loss": 1.3524,
+ "theoretical_loss": 3.5173500174158208,
+ "tokens_seen": 1639317504
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004067241725175527,
+ "loss": 1.3141,
+ "theoretical_loss": 3.517337928626307,
+ "tokens_seen": 1639383040
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040670812437311937,
+ "loss": 1.3674,
+ "theoretical_loss": 3.517325840455352,
+ "tokens_seen": 1639448576
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004066920762286861,
+ "loss": 1.3128,
+ "theoretical_loss": 3.5173137529029006,
+ "tokens_seen": 1639514112
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040667602808425277,
+ "loss": 1.2473,
+ "theoretical_loss": 3.5173016659688945,
+ "tokens_seen": 1639579648
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004066599799398195,
+ "loss": 1.3721,
+ "theoretical_loss": 3.517289579653279,
+ "tokens_seen": 1639645184
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004066439317953862,
+ "loss": 1.2469,
+ "theoretical_loss": 3.5172774939559965,
+ "tokens_seen": 1639710720
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040662788365095293,
+ "loss": 1.2926,
+ "theoretical_loss": 3.517265408876992,
+ "tokens_seen": 1639776256
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004066118355065196,
+ "loss": 1.3442,
+ "theoretical_loss": 3.517253324416208,
+ "tokens_seen": 1639841792
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004065957873620863,
+ "loss": 1.3684,
+ "theoretical_loss": 3.517241240573588,
+ "tokens_seen": 1639907328
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000406579739217653,
+ "loss": 1.2832,
+ "theoretical_loss": 3.5172291573490773,
+ "tokens_seen": 1639972864
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004065636910732197,
+ "loss": 1.2527,
+ "theoretical_loss": 3.517217074742618,
+ "tokens_seen": 1640038400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004065476429287864,
+ "loss": 1.3512,
+ "theoretical_loss": 3.517204992754154,
+ "tokens_seen": 1640103936
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004065315947843531,
+ "loss": 1.3242,
+ "theoretical_loss": 3.5171929113836295,
+ "tokens_seen": 1640169472
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004065155466399198,
+ "loss": 1.3171,
+ "theoretical_loss": 3.5171808306309886,
+ "tokens_seen": 1640235008
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064994984954865,
+ "loss": 1.2756,
+ "theoretical_loss": 3.5171687504961744,
+ "tokens_seen": 1640300544
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064834503510531,
+ "loss": 1.2805,
+ "theoretical_loss": 3.51715667097913,
+ "tokens_seen": 1640366080
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064674022066199,
+ "loss": 1.311,
+ "theoretical_loss": 3.5171445920798003,
+ "tokens_seen": 1640431616
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064513540621865,
+ "loss": 1.267,
+ "theoretical_loss": 3.5171325137981286,
+ "tokens_seen": 1640497152
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064353059177533,
+ "loss": 1.2189,
+ "theoretical_loss": 3.517120436134058,
+ "tokens_seen": 1640562688
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040641925777331993,
+ "loss": 1.4002,
+ "theoretical_loss": 3.5171083590875334,
+ "tokens_seen": 1640628224
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004064032096288867,
+ "loss": 1.3636,
+ "theoretical_loss": 3.517096282658498,
+ "tokens_seen": 1640693760
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040638716148445333,
+ "loss": 1.3992,
+ "theoretical_loss": 3.5170842068468953,
+ "tokens_seen": 1640759296
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004063711133400201,
+ "loss": 1.3882,
+ "theoretical_loss": 3.5170721316526694,
+ "tokens_seen": 1640824832
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040635506519558673,
+ "loss": 1.2632,
+ "theoretical_loss": 3.517060057075764,
+ "tokens_seen": 1640890368
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004063390170511535,
+ "loss": 1.2589,
+ "theoretical_loss": 3.517047983116123,
+ "tokens_seen": 1640955904
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040632296890672013,
+ "loss": 1.3114,
+ "theoretical_loss": 3.51703590977369,
+ "tokens_seen": 1641021440
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004063069207622869,
+ "loss": 1.2914,
+ "theoretical_loss": 3.517023837048409,
+ "tokens_seen": 1641086976
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040629087261785353,
+ "loss": 1.2638,
+ "theoretical_loss": 3.5170117649402233,
+ "tokens_seen": 1641152512
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004062748244734203,
+ "loss": 1.2527,
+ "theoretical_loss": 3.5169996934490775,
+ "tokens_seen": 1641218048
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040625877632898693,
+ "loss": 1.2254,
+ "theoretical_loss": 3.516987622574915,
+ "tokens_seen": 1641283584
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004062427281845537,
+ "loss": 1.3077,
+ "theoretical_loss": 3.51697555231768,
+ "tokens_seen": 1641349120
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040622668004012034,
+ "loss": 1.4039,
+ "theoretical_loss": 3.5169634826773155,
+ "tokens_seen": 1641414656
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004062106318956871,
+ "loss": 1.3378,
+ "theoretical_loss": 3.516951413653766,
+ "tokens_seen": 1641480192
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040619458375125374,
+ "loss": 1.2719,
+ "theoretical_loss": 3.516939345246975,
+ "tokens_seen": 1641545728
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004061785356068205,
+ "loss": 1.2856,
+ "theoretical_loss": 3.5169272774568867,
+ "tokens_seen": 1641611264
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 550613,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3520652055740356,
+ "objective/train/theoretical_loss": 3.516915210283445,
+ "objective/train/tokens_used": 1662136800,
+ "theoretical_loss": 3.516915210283445,
+ "tokens_seen": 1641676800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040616248746238714,
+ "loss": 1.2895,
+ "theoretical_loss": 3.516915210283445,
+ "tokens_seen": 1641676800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004061464393179539,
+ "loss": 1.312,
+ "theoretical_loss": 3.5169031437265934,
+ "tokens_seen": 1641742336
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040613039117352054,
+ "loss": 1.2866,
+ "theoretical_loss": 3.5168910777862763,
+ "tokens_seen": 1641807872
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004061143430290873,
+ "loss": 1.2786,
+ "theoretical_loss": 3.516879012462437,
+ "tokens_seen": 1641873408
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040609829488465394,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5168669477550196,
+ "tokens_seen": 1641938944
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004060822467402207,
+ "loss": 1.254,
+ "theoretical_loss": 3.5168548836639686,
+ "tokens_seen": 1642004480
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040606619859578734,
+ "loss": 1.3026,
+ "theoretical_loss": 3.5168428201892272,
+ "tokens_seen": 1642070016
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004060501504513541,
+ "loss": 1.4237,
+ "theoretical_loss": 3.516830757330739,
+ "tokens_seen": 1642135552
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040603410230692074,
+ "loss": 1.2869,
+ "theoretical_loss": 3.5168186950884492,
+ "tokens_seen": 1642201088
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004060180541624875,
+ "loss": 1.2253,
+ "theoretical_loss": 3.5168066334623003,
+ "tokens_seen": 1642266624
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040600200601805425,
+ "loss": 1.2861,
+ "theoretical_loss": 3.5167945724522376,
+ "tokens_seen": 1642332160
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004059859578736209,
+ "loss": 1.3845,
+ "theoretical_loss": 3.516782512058204,
+ "tokens_seen": 1642397696
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004059699097291876,
+ "loss": 1.2079,
+ "theoretical_loss": 3.516770452280144,
+ "tokens_seen": 1642463232
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004059538615847543,
+ "loss": 1.3241,
+ "theoretical_loss": 3.5167583931180015,
+ "tokens_seen": 1642528768
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405937813440321,
+ "loss": 1.2881,
+ "theoretical_loss": 3.51674633457172,
+ "tokens_seen": 1642594304
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004059217652958877,
+ "loss": 1.2206,
+ "theoretical_loss": 3.5167342766412437,
+ "tokens_seen": 1642659840
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004059057171514544,
+ "loss": 1.3041,
+ "theoretical_loss": 3.516722219326517,
+ "tokens_seen": 1642725376
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004058896690070211,
+ "loss": 1.527,
+ "theoretical_loss": 3.5167101626274837,
+ "tokens_seen": 1642790912
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004058736208625878,
+ "loss": 1.2195,
+ "theoretical_loss": 3.5166981065440877,
+ "tokens_seen": 1642856448
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004058575727181545,
+ "loss": 1.2176,
+ "theoretical_loss": 3.516686051076273,
+ "tokens_seen": 1642921984
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004058415245737212,
+ "loss": 1.4429,
+ "theoretical_loss": 3.5166739962239832,
+ "tokens_seen": 1642987520
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040582547642928785,
+ "loss": 1.4121,
+ "theoretical_loss": 3.516661941987164,
+ "tokens_seen": 1643053056
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004058094282848546,
+ "loss": 1.2556,
+ "theoretical_loss": 3.516649888365757,
+ "tokens_seen": 1643118592
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040579338014042125,
+ "loss": 1.1572,
+ "theoretical_loss": 3.516637835359708,
+ "tokens_seen": 1643184128
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405777331995988,
+ "loss": 1.1544,
+ "theoretical_loss": 3.5166257829689602,
+ "tokens_seen": 1643249664
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040576128385155465,
+ "loss": 1.2845,
+ "theoretical_loss": 3.516613731193458,
+ "tokens_seen": 1643315200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004057452357071214,
+ "loss": 1.165,
+ "theoretical_loss": 3.5166016800331454,
+ "tokens_seen": 1643380736
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040572918756268806,
+ "loss": 1.332,
+ "theoretical_loss": 3.5165896294879664,
+ "tokens_seen": 1643446272
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004057131394182548,
+ "loss": 1.3102,
+ "theoretical_loss": 3.5165775795578655,
+ "tokens_seen": 1643511808
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040569709127382146,
+ "loss": 1.2905,
+ "theoretical_loss": 3.516565530242786,
+ "tokens_seen": 1643577344
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004056810431293882,
+ "loss": 1.3407,
+ "theoretical_loss": 3.5165534815426724,
+ "tokens_seen": 1643642880
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040566499498495486,
+ "loss": 1.3558,
+ "theoretical_loss": 3.516541433457469,
+ "tokens_seen": 1643708416
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004056489468405216,
+ "loss": 1.2896,
+ "theoretical_loss": 3.51652938598712,
+ "tokens_seen": 1643773952
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040563289869608826,
+ "loss": 1.281,
+ "theoretical_loss": 3.5165173391315685,
+ "tokens_seen": 1643839488
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405616850551655,
+ "loss": 1.2944,
+ "theoretical_loss": 3.516505292890759,
+ "tokens_seen": 1643905024
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040560080240722166,
+ "loss": 1.2427,
+ "theoretical_loss": 3.516493247264637,
+ "tokens_seen": 1643970560
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004055847542627884,
+ "loss": 1.2297,
+ "theoretical_loss": 3.516481202253145,
+ "tokens_seen": 1644036096
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040556870611835506,
+ "loss": 1.2446,
+ "theoretical_loss": 3.5164691578562284,
+ "tokens_seen": 1644101632
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004055526579739218,
+ "loss": 1.3117,
+ "theoretical_loss": 3.51645711407383,
+ "tokens_seen": 1644167168
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040553660982948846,
+ "loss": 1.2907,
+ "theoretical_loss": 3.5164450709058945,
+ "tokens_seen": 1644232704
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004055205616850552,
+ "loss": 1.3179,
+ "theoretical_loss": 3.516433028352367,
+ "tokens_seen": 1644298240
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040550451354062186,
+ "loss": 1.4179,
+ "theoretical_loss": 3.51642098641319,
+ "tokens_seen": 1644363776
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004054884653961886,
+ "loss": 1.324,
+ "theoretical_loss": 3.5164089450883087,
+ "tokens_seen": 1644429312
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040547241725175527,
+ "loss": 1.1669,
+ "theoretical_loss": 3.5163969043776673,
+ "tokens_seen": 1644494848
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405456369107322,
+ "loss": 1.3281,
+ "theoretical_loss": 3.5163848642812097,
+ "tokens_seen": 1644560384
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040544032096288867,
+ "loss": 1.2432,
+ "theoretical_loss": 3.5163728247988804,
+ "tokens_seen": 1644625920
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004054242728184554,
+ "loss": 1.2673,
+ "theoretical_loss": 3.516360785930623,
+ "tokens_seen": 1644691456
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040540822467402207,
+ "loss": 1.2448,
+ "theoretical_loss": 3.516348747676382,
+ "tokens_seen": 1644756992
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004053921765295888,
+ "loss": 1.3363,
+ "theoretical_loss": 3.5163367100361023,
+ "tokens_seen": 1644822528
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040537612838515547,
+ "loss": 1.2778,
+ "theoretical_loss": 3.516324673009727,
+ "tokens_seen": 1644888064
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 551496,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3743598461151123,
+ "objective/train/theoretical_loss": 3.5163126365972017,
+ "objective/train/tokens_used": 1665413600,
+ "theoretical_loss": 3.5163126365972017,
+ "tokens_seen": 1644953600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004053600802407222,
+ "loss": 1.3564,
+ "theoretical_loss": 3.5163126365972017,
+ "tokens_seen": 1644953600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040534403209628887,
+ "loss": 1.2824,
+ "theoretical_loss": 3.516300600798469,
+ "tokens_seen": 1645019136
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004053279839518556,
+ "loss": 1.2393,
+ "theoretical_loss": 3.5162885656134746,
+ "tokens_seen": 1645084672
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040531193580742227,
+ "loss": 1.4257,
+ "theoretical_loss": 3.5162765310421618,
+ "tokens_seen": 1645150208
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405295887662989,
+ "loss": 1.2824,
+ "theoretical_loss": 3.5162644970844754,
+ "tokens_seen": 1645215744
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004052798395185557,
+ "loss": 1.3281,
+ "theoretical_loss": 3.516252463740359,
+ "tokens_seen": 1645281280
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040526379137412243,
+ "loss": 1.2545,
+ "theoretical_loss": 3.516240431009758,
+ "tokens_seen": 1645346816
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004052477432296891,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5162283988926157,
+ "tokens_seen": 1645412352
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040523169508525583,
+ "loss": 1.3082,
+ "theoretical_loss": 3.5162163673888767,
+ "tokens_seen": 1645477888
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004052156469408225,
+ "loss": 1.2894,
+ "theoretical_loss": 3.516204336498485,
+ "tokens_seen": 1645543424
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051995987963892,
+ "loss": 1.2994,
+ "theoretical_loss": 3.516192306221386,
+ "tokens_seen": 1645608960
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051835506519559,
+ "loss": 1.3521,
+ "theoretical_loss": 3.516180276557523,
+ "tokens_seen": 1645674496
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051675025075226,
+ "loss": 1.3075,
+ "theoretical_loss": 3.5161682475068403,
+ "tokens_seen": 1645740032
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051514543630893,
+ "loss": 1.2642,
+ "theoretical_loss": 3.5161562190692823,
+ "tokens_seen": 1645805568
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000405135406218656,
+ "loss": 1.2422,
+ "theoretical_loss": 3.516144191244794,
+ "tokens_seen": 1645871104
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051193580742227,
+ "loss": 1.3944,
+ "theoretical_loss": 3.516132164033319,
+ "tokens_seen": 1645936640
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004051033099297894,
+ "loss": 1.2794,
+ "theoretical_loss": 3.516120137434802,
+ "tokens_seen": 1646002176
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004050872617853561,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5161081114491877,
+ "tokens_seen": 1646067712
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004050712136409228,
+ "loss": 1.3001,
+ "theoretical_loss": 3.51609608607642,
+ "tokens_seen": 1646133248
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040505516549648943,
+ "loss": 1.2354,
+ "theoretical_loss": 3.516084061316443,
+ "tokens_seen": 1646198784
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004050391173520562,
+ "loss": 1.1901,
+ "theoretical_loss": 3.5160720371692014,
+ "tokens_seen": 1646264320
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040502306920762283,
+ "loss": 1.3966,
+ "theoretical_loss": 3.5160600136346396,
+ "tokens_seen": 1646329856
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004050070210631896,
+ "loss": 1.3421,
+ "theoretical_loss": 3.5160479907127016,
+ "tokens_seen": 1646395392
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040499097291875623,
+ "loss": 1.3397,
+ "theoretical_loss": 3.516035968403333,
+ "tokens_seen": 1646460928
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000404974924774323,
+ "loss": 1.4182,
+ "theoretical_loss": 3.516023946706477,
+ "tokens_seen": 1646526464
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040495887662988963,
+ "loss": 1.3631,
+ "theoretical_loss": 3.5160119256220783,
+ "tokens_seen": 1646592000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004049428284854564,
+ "loss": 1.2533,
+ "theoretical_loss": 3.515999905150082,
+ "tokens_seen": 1646657536
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040492678034102303,
+ "loss": 1.3535,
+ "theoretical_loss": 3.515987885290431,
+ "tokens_seen": 1646723072
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004049107321965898,
+ "loss": 1.3533,
+ "theoretical_loss": 3.515975866043071,
+ "tokens_seen": 1646788608
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040489468405215643,
+ "loss": 1.3462,
+ "theoretical_loss": 3.5159638474079467,
+ "tokens_seen": 1646854144
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004048786359077232,
+ "loss": 1.4025,
+ "theoretical_loss": 3.515951829385002,
+ "tokens_seen": 1646919680
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040486258776328984,
+ "loss": 1.2922,
+ "theoretical_loss": 3.5159398119741807,
+ "tokens_seen": 1646985216
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004048465396188566,
+ "loss": 1.2653,
+ "theoretical_loss": 3.515927795175428,
+ "tokens_seen": 1647050752
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040483049147442324,
+ "loss": 1.1922,
+ "theoretical_loss": 3.5159157789886892,
+ "tokens_seen": 1647116288
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040481444332999,
+ "loss": 1.3475,
+ "theoretical_loss": 3.515903763413907,
+ "tokens_seen": 1647181824
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040479839518555675,
+ "loss": 1.2216,
+ "theoretical_loss": 3.515891748451027,
+ "tokens_seen": 1647247360
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004047823470411234,
+ "loss": 1.2698,
+ "theoretical_loss": 3.515879734099993,
+ "tokens_seen": 1647312896
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040476629889669015,
+ "loss": 1.3232,
+ "theoretical_loss": 3.5158677203607507,
+ "tokens_seen": 1647378432
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004047502507522568,
+ "loss": 1.3189,
+ "theoretical_loss": 3.5158557072332437,
+ "tokens_seen": 1647443968
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040473420260782355,
+ "loss": 1.2758,
+ "theoretical_loss": 3.5158436947174168,
+ "tokens_seen": 1647509504
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004047181544633902,
+ "loss": 1.3033,
+ "theoretical_loss": 3.5158316828132143,
+ "tokens_seen": 1647575040
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040470210631895695,
+ "loss": 1.3382,
+ "theoretical_loss": 3.5158196715205805,
+ "tokens_seen": 1647640576
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004046860581745236,
+ "loss": 1.2258,
+ "theoretical_loss": 3.5158076608394606,
+ "tokens_seen": 1647706112
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040467001003009035,
+ "loss": 1.3701,
+ "theoretical_loss": 3.5157956507697987,
+ "tokens_seen": 1647771648
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000404653961885657,
+ "loss": 1.313,
+ "theoretical_loss": 3.5157836413115398,
+ "tokens_seen": 1647837184
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040463791374122375,
+ "loss": 1.3229,
+ "theoretical_loss": 3.515771632464628,
+ "tokens_seen": 1647902720
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004046218655967904,
+ "loss": 1.3026,
+ "theoretical_loss": 3.515759624229008,
+ "tokens_seen": 1647968256
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040460581745235715,
+ "loss": 1.2709,
+ "theoretical_loss": 3.5157476166046244,
+ "tokens_seen": 1648033792
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045897693079238,
+ "loss": 1.3436,
+ "theoretical_loss": 3.5157356095914216,
+ "tokens_seen": 1648099328
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045737211634905,
+ "loss": 1.3777,
+ "theoretical_loss": 3.5157236031893446,
+ "tokens_seen": 1648164864
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 552711,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2650483846664429,
+ "objective/train/theoretical_loss": 3.5157115973983375,
+ "objective/train/tokens_used": 1668690400,
+ "theoretical_loss": 3.5157115973983375,
+ "tokens_seen": 1648230400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045576730190572,
+ "loss": 1.3071,
+ "theoretical_loss": 3.5157115973983375,
+ "tokens_seen": 1648230400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045416248746239,
+ "loss": 1.2882,
+ "theoretical_loss": 3.5156995922183456,
+ "tokens_seen": 1648295936
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045255767301906,
+ "loss": 1.3203,
+ "theoretical_loss": 3.515687587649313,
+ "tokens_seen": 1648361472
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004045095285857573,
+ "loss": 1.3522,
+ "theoretical_loss": 3.515675583691184,
+ "tokens_seen": 1648427008
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000404493480441324,
+ "loss": 1.3041,
+ "theoretical_loss": 3.515663580343904,
+ "tokens_seen": 1648492544
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004044774322968907,
+ "loss": 1.319,
+ "theoretical_loss": 3.515651577607417,
+ "tokens_seen": 1648558080
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004044613841524574,
+ "loss": 1.4277,
+ "theoretical_loss": 3.515639575481668,
+ "tokens_seen": 1648623616
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004044453360080241,
+ "loss": 1.439,
+ "theoretical_loss": 3.5156275739666016,
+ "tokens_seen": 1648689152
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040442928786359075,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5156155730621625,
+ "tokens_seen": 1648754688
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004044132397191575,
+ "loss": 1.2366,
+ "theoretical_loss": 3.5156035727682946,
+ "tokens_seen": 1648820224
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040439719157472415,
+ "loss": 1.2742,
+ "theoretical_loss": 3.515591573084944,
+ "tokens_seen": 1648885760
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004043811434302909,
+ "loss": 1.1554,
+ "theoretical_loss": 3.5155795740120546,
+ "tokens_seen": 1648951296
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040436509528585755,
+ "loss": 1.2962,
+ "theoretical_loss": 3.5155675755495706,
+ "tokens_seen": 1649016832
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004043490471414243,
+ "loss": 1.3395,
+ "theoretical_loss": 3.5155555776974374,
+ "tokens_seen": 1649082368
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040433299899699096,
+ "loss": 1.3604,
+ "theoretical_loss": 3.5155435804555997,
+ "tokens_seen": 1649147904
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004043169508525577,
+ "loss": 1.2655,
+ "theoretical_loss": 3.5155315838240018,
+ "tokens_seen": 1649213440
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040430090270812436,
+ "loss": 1.2242,
+ "theoretical_loss": 3.5155195878025887,
+ "tokens_seen": 1649278976
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004042848545636911,
+ "loss": 1.3321,
+ "theoretical_loss": 3.515507592391305,
+ "tokens_seen": 1649344512
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040426880641925776,
+ "loss": 1.3798,
+ "theoretical_loss": 3.5154955975900952,
+ "tokens_seen": 1649410048
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004042527582748245,
+ "loss": 1.3723,
+ "theoretical_loss": 3.5154836033989048,
+ "tokens_seen": 1649475584
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040423671013039116,
+ "loss": 1.307,
+ "theoretical_loss": 3.5154716098176775,
+ "tokens_seen": 1649541120
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004042206619859579,
+ "loss": 1.3331,
+ "theoretical_loss": 3.5154596168463588,
+ "tokens_seen": 1649606656
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040420461384152456,
+ "loss": 1.3565,
+ "theoretical_loss": 3.515447624484893,
+ "tokens_seen": 1649672192
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004041885656970913,
+ "loss": 1.2387,
+ "theoretical_loss": 3.515435632733225,
+ "tokens_seen": 1649737728
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040417251755265796,
+ "loss": 1.3197,
+ "theoretical_loss": 3.5154236415913003,
+ "tokens_seen": 1649803264
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004041564694082247,
+ "loss": 1.2871,
+ "theoretical_loss": 3.5154116510590625,
+ "tokens_seen": 1649868800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040414042126379136,
+ "loss": 1.2499,
+ "theoretical_loss": 3.515399661136457,
+ "tokens_seen": 1649934336
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004041243731193581,
+ "loss": 1.3688,
+ "theoretical_loss": 3.5153876718234285,
+ "tokens_seen": 1649999872
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040410832497492477,
+ "loss": 1.2212,
+ "theoretical_loss": 3.515375683119922,
+ "tokens_seen": 1650065408
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004040922768304915,
+ "loss": 1.4593,
+ "theoretical_loss": 3.515363695025882,
+ "tokens_seen": 1650130944
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040407622868605817,
+ "loss": 1.2195,
+ "theoretical_loss": 3.515351707541253,
+ "tokens_seen": 1650196480
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004040601805416249,
+ "loss": 1.2406,
+ "theoretical_loss": 3.515339720665981,
+ "tokens_seen": 1650262016
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040404413239719157,
+ "loss": 1.2425,
+ "theoretical_loss": 3.51532773440001,
+ "tokens_seen": 1650327552
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004040280842527583,
+ "loss": 1.3901,
+ "theoretical_loss": 3.5153157487432845,
+ "tokens_seen": 1650393088
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040401203610832497,
+ "loss": 1.3186,
+ "theoretical_loss": 3.5153037636957496,
+ "tokens_seen": 1650458624
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004039959879638917,
+ "loss": 1.2464,
+ "theoretical_loss": 3.5152917792573506,
+ "tokens_seen": 1650524160
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040397993981945837,
+ "loss": 1.2504,
+ "theoretical_loss": 3.5152797954280324,
+ "tokens_seen": 1650589696
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004039638916750251,
+ "loss": 1.3325,
+ "theoretical_loss": 3.515267812207739,
+ "tokens_seen": 1650655232
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040394784353059177,
+ "loss": 1.2496,
+ "theoretical_loss": 3.5152558295964162,
+ "tokens_seen": 1650720768
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004039317953861585,
+ "loss": 1.2919,
+ "theoretical_loss": 3.515243847594008,
+ "tokens_seen": 1650786304
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040391574724172517,
+ "loss": 1.237,
+ "theoretical_loss": 3.5152318662004602,
+ "tokens_seen": 1650851840
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040389969909729193,
+ "loss": 1.2685,
+ "theoretical_loss": 3.515219885415717,
+ "tokens_seen": 1650917376
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004038836509528586,
+ "loss": 1.2956,
+ "theoretical_loss": 3.515207905239724,
+ "tokens_seen": 1650982912
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040386760280842533,
+ "loss": 1.2047,
+ "theoretical_loss": 3.5151959256724252,
+ "tokens_seen": 1651048448
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000403851554663992,
+ "loss": 1.3302,
+ "theoretical_loss": 3.5151839467137656,
+ "tokens_seen": 1651113984
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040383550651955873,
+ "loss": 1.2759,
+ "theoretical_loss": 3.515171968363691,
+ "tokens_seen": 1651179520
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004038194583751254,
+ "loss": 1.3293,
+ "theoretical_loss": 3.515159990622146,
+ "tokens_seen": 1651245056
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004038034102306921,
+ "loss": 1.1966,
+ "theoretical_loss": 3.5151480134890747,
+ "tokens_seen": 1651310592
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037873620862588,
+ "loss": 1.262,
+ "theoretical_loss": 3.515136036964423,
+ "tokens_seen": 1651376128
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037713139418255,
+ "loss": 1.2622,
+ "theoretical_loss": 3.5151240610481356,
+ "tokens_seen": 1651441664
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 553645,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1858956813812256,
+ "objective/train/theoretical_loss": 3.5151120857401574,
+ "objective/train/tokens_used": 1671967200,
+ "theoretical_loss": 3.5151120857401574,
+ "tokens_seen": 1651507200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037552657973922,
+ "loss": 1.194,
+ "theoretical_loss": 3.5151120857401574,
+ "tokens_seen": 1651507200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037392176529589,
+ "loss": 1.3638,
+ "theoretical_loss": 3.5151001110404336,
+ "tokens_seen": 1651572736
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037231695085256,
+ "loss": 1.2681,
+ "theoretical_loss": 3.5150881369489086,
+ "tokens_seen": 1651638272
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004037071213640923,
+ "loss": 1.2787,
+ "theoretical_loss": 3.515076163465528,
+ "tokens_seen": 1651703808
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000403691073219659,
+ "loss": 1.2985,
+ "theoretical_loss": 3.5150641905902367,
+ "tokens_seen": 1651769344
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004036750250752257,
+ "loss": 1.2556,
+ "theoretical_loss": 3.515052218322979,
+ "tokens_seen": 1651834880
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040365897693079233,
+ "loss": 1.2907,
+ "theoretical_loss": 3.5150402466637,
+ "tokens_seen": 1651900416
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004036429287863591,
+ "loss": 1.2359,
+ "theoretical_loss": 3.515028275612346,
+ "tokens_seen": 1651965952
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040362688064192573,
+ "loss": 1.3341,
+ "theoretical_loss": 3.5150163051688605,
+ "tokens_seen": 1652031488
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004036108324974925,
+ "loss": 1.2262,
+ "theoretical_loss": 3.5150043353331895,
+ "tokens_seen": 1652097024
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040359478435305924,
+ "loss": 1.4032,
+ "theoretical_loss": 3.514992366105277,
+ "tokens_seen": 1652162560
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004035787362086259,
+ "loss": 1.2673,
+ "theoretical_loss": 3.5149803974850697,
+ "tokens_seen": 1652228096
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040356268806419264,
+ "loss": 1.2494,
+ "theoretical_loss": 3.5149684294725114,
+ "tokens_seen": 1652293632
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004035466399197593,
+ "loss": 1.3535,
+ "theoretical_loss": 3.5149564620675466,
+ "tokens_seen": 1652359168
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040353059177532604,
+ "loss": 1.3012,
+ "theoretical_loss": 3.514944495270122,
+ "tokens_seen": 1652424704
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004035145436308927,
+ "loss": 1.4052,
+ "theoretical_loss": 3.5149325290801814,
+ "tokens_seen": 1652490240
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040349849548645944,
+ "loss": 1.3336,
+ "theoretical_loss": 3.5149205634976703,
+ "tokens_seen": 1652555776
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004034824473420261,
+ "loss": 1.3278,
+ "theoretical_loss": 3.514908598522534,
+ "tokens_seen": 1652621312
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040346639919759284,
+ "loss": 1.2521,
+ "theoretical_loss": 3.5148966341547174,
+ "tokens_seen": 1652686848
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004034503510531595,
+ "loss": 1.2629,
+ "theoretical_loss": 3.5148846703941654,
+ "tokens_seen": 1652752384
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040343430290872625,
+ "loss": 1.2761,
+ "theoretical_loss": 3.514872707240823,
+ "tokens_seen": 1652817920
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004034182547642929,
+ "loss": 1.4867,
+ "theoretical_loss": 3.5148607446946354,
+ "tokens_seen": 1652883456
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040340220661985965,
+ "loss": 1.3313,
+ "theoretical_loss": 3.514848782755548,
+ "tokens_seen": 1652948992
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004033861584754263,
+ "loss": 1.2909,
+ "theoretical_loss": 3.514836821423506,
+ "tokens_seen": 1653014528
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040337011033099305,
+ "loss": 1.2385,
+ "theoretical_loss": 3.5148248606984547,
+ "tokens_seen": 1653080064
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004033540621865597,
+ "loss": 1.2123,
+ "theoretical_loss": 3.5148129005803384,
+ "tokens_seen": 1653145600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040333801404212645,
+ "loss": 1.1998,
+ "theoretical_loss": 3.514800941069103,
+ "tokens_seen": 1653211136
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004033219658976931,
+ "loss": 1.2844,
+ "theoretical_loss": 3.5147889821646925,
+ "tokens_seen": 1653276672
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040330591775325985,
+ "loss": 1.2154,
+ "theoretical_loss": 3.5147770238670537,
+ "tokens_seen": 1653342208
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004032898696088265,
+ "loss": 1.279,
+ "theoretical_loss": 3.51476506617613,
+ "tokens_seen": 1653407744
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040327382146439325,
+ "loss": 1.2855,
+ "theoretical_loss": 3.5147531090918687,
+ "tokens_seen": 1653473280
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004032577733199599,
+ "loss": 1.2695,
+ "theoretical_loss": 3.514741152614213,
+ "tokens_seen": 1653538816
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040324172517552665,
+ "loss": 1.2805,
+ "theoretical_loss": 3.5147291967431094,
+ "tokens_seen": 1653604352
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004032256770310933,
+ "loss": 1.2239,
+ "theoretical_loss": 3.514717241478502,
+ "tokens_seen": 1653669888
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040320962888666005,
+ "loss": 1.2177,
+ "theoretical_loss": 3.5147052868203374,
+ "tokens_seen": 1653735424
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031935807422267,
+ "loss": 1.3711,
+ "theoretical_loss": 3.5146933327685597,
+ "tokens_seen": 1653800960
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031775325977934,
+ "loss": 1.3291,
+ "theoretical_loss": 3.5146813793231138,
+ "tokens_seen": 1653866496
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031614844533601,
+ "loss": 1.2799,
+ "theoretical_loss": 3.5146694264839464,
+ "tokens_seen": 1653932032
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031454363089268,
+ "loss": 1.2196,
+ "theoretical_loss": 3.5146574742510013,
+ "tokens_seen": 1653997568
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031293881644935,
+ "loss": 1.2649,
+ "theoretical_loss": 3.5146455226242246,
+ "tokens_seen": 1654063104
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004031133400200602,
+ "loss": 1.2337,
+ "theoretical_loss": 3.514633571603561,
+ "tokens_seen": 1654128640
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004030972918756269,
+ "loss": 1.2707,
+ "theoretical_loss": 3.514621621188956,
+ "tokens_seen": 1654194176
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004030812437311936,
+ "loss": 1.4027,
+ "theoretical_loss": 3.514609671380355,
+ "tokens_seen": 1654259712
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004030651955867603,
+ "loss": 1.2711,
+ "theoretical_loss": 3.5145977221777027,
+ "tokens_seen": 1654325248
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000403049147442327,
+ "loss": 1.3762,
+ "theoretical_loss": 3.5145857735809454,
+ "tokens_seen": 1654390784
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040303309929789365,
+ "loss": 1.2744,
+ "theoretical_loss": 3.514573825590027,
+ "tokens_seen": 1654456320
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004030170511534604,
+ "loss": 1.2372,
+ "theoretical_loss": 3.5145618782048937,
+ "tokens_seen": 1654521856
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040300100300902705,
+ "loss": 1.3249,
+ "theoretical_loss": 3.514549931425491,
+ "tokens_seen": 1654587392
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004029849548645938,
+ "loss": 1.2668,
+ "theoretical_loss": 3.5145379852517635,
+ "tokens_seen": 1654652928
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040296890672016046,
+ "loss": 1.3107,
+ "theoretical_loss": 3.5145260396836573,
+ "tokens_seen": 1654718464
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 554730,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0662038326263428,
+ "objective/train/theoretical_loss": 3.514514094721117,
+ "objective/train/tokens_used": 1675244000,
+ "theoretical_loss": 3.514514094721117,
+ "tokens_seen": 1654784000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004029528585757272,
+ "loss": 1.2191,
+ "theoretical_loss": 3.514514094721117,
+ "tokens_seen": 1654784000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040293681043129386,
+ "loss": 1.2078,
+ "theoretical_loss": 3.514502150364088,
+ "tokens_seen": 1654849536
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004029207622868606,
+ "loss": 1.2636,
+ "theoretical_loss": 3.514490206612516,
+ "tokens_seen": 1654915072
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040290471414242726,
+ "loss": 1.2449,
+ "theoretical_loss": 3.514478263466346,
+ "tokens_seen": 1654980608
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402888665997994,
+ "loss": 1.2674,
+ "theoretical_loss": 3.5144663209255236,
+ "tokens_seen": 1655046144
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040287261785356066,
+ "loss": 1.3643,
+ "theoretical_loss": 3.514454378989994,
+ "tokens_seen": 1655111680
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004028565697091274,
+ "loss": 1.3403,
+ "theoretical_loss": 3.514442437659702,
+ "tokens_seen": 1655177216
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040284052156469406,
+ "loss": 1.3899,
+ "theoretical_loss": 3.5144304969345947,
+ "tokens_seen": 1655242752
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004028244734202608,
+ "loss": 1.3013,
+ "theoretical_loss": 3.5144185568146153,
+ "tokens_seen": 1655308288
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040280842527582746,
+ "loss": 1.3463,
+ "theoretical_loss": 3.5144066172997106,
+ "tokens_seen": 1655373824
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004027923771313942,
+ "loss": 1.4002,
+ "theoretical_loss": 3.5143946783898254,
+ "tokens_seen": 1655439360
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040277632898696086,
+ "loss": 1.2795,
+ "theoretical_loss": 3.5143827400849057,
+ "tokens_seen": 1655504896
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004027602808425276,
+ "loss": 1.3027,
+ "theoretical_loss": 3.514370802384896,
+ "tokens_seen": 1655570432
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040274423269809426,
+ "loss": 1.3428,
+ "theoretical_loss": 3.514358865289742,
+ "tokens_seen": 1655635968
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402728184553661,
+ "loss": 1.2307,
+ "theoretical_loss": 3.51434692879939,
+ "tokens_seen": 1655701504
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040271213640922767,
+ "loss": 1.2406,
+ "theoretical_loss": 3.514334992913784,
+ "tokens_seen": 1655767040
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004026960882647944,
+ "loss": 1.3787,
+ "theoretical_loss": 3.5143230576328706,
+ "tokens_seen": 1655832576
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040268004012036107,
+ "loss": 1.271,
+ "theoretical_loss": 3.5143111229565944,
+ "tokens_seen": 1655898112
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004026639919759278,
+ "loss": 1.2659,
+ "theoretical_loss": 3.514299188884902,
+ "tokens_seen": 1655963648
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040264794383149447,
+ "loss": 1.2307,
+ "theoretical_loss": 3.514287255417737,
+ "tokens_seen": 1656029184
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004026318956870612,
+ "loss": 1.2847,
+ "theoretical_loss": 3.5142753225550463,
+ "tokens_seen": 1656094720
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040261584754262787,
+ "loss": 1.2566,
+ "theoretical_loss": 3.514263390296775,
+ "tokens_seen": 1656160256
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004025997993981946,
+ "loss": 1.2322,
+ "theoretical_loss": 3.5142514586428684,
+ "tokens_seen": 1656225792
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040258375125376127,
+ "loss": 1.2389,
+ "theoretical_loss": 3.5142395275932725,
+ "tokens_seen": 1656291328
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402567703109328,
+ "loss": 1.3053,
+ "theoretical_loss": 3.514227597147932,
+ "tokens_seen": 1656356864
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040255165496489467,
+ "loss": 1.2863,
+ "theoretical_loss": 3.514215667306793,
+ "tokens_seen": 1656422400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040253560682046143,
+ "loss": 1.2838,
+ "theoretical_loss": 3.514203738069801,
+ "tokens_seen": 1656487936
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004025195586760281,
+ "loss": 1.308,
+ "theoretical_loss": 3.514191809436901,
+ "tokens_seen": 1656553472
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040250351053159483,
+ "loss": 1.2878,
+ "theoretical_loss": 3.5141798814080385,
+ "tokens_seen": 1656619008
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004024874623871615,
+ "loss": 1.3471,
+ "theoretical_loss": 3.5141679539831596,
+ "tokens_seen": 1656684544
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040247141424272823,
+ "loss": 1.4304,
+ "theoretical_loss": 3.5141560271622097,
+ "tokens_seen": 1656750080
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004024553660982949,
+ "loss": 1.233,
+ "theoretical_loss": 3.514144100945134,
+ "tokens_seen": 1656815616
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040243931795386163,
+ "loss": 1.3339,
+ "theoretical_loss": 3.5141321753318784,
+ "tokens_seen": 1656881152
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004024232698094283,
+ "loss": 1.1742,
+ "theoretical_loss": 3.514120250322388,
+ "tokens_seen": 1656946688
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402407221664995,
+ "loss": 1.2235,
+ "theoretical_loss": 3.5141083259166086,
+ "tokens_seen": 1657012224
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040239117352056173,
+ "loss": 1.2988,
+ "theoretical_loss": 3.514096402114486,
+ "tokens_seen": 1657077760
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004023751253761284,
+ "loss": 1.3111,
+ "theoretical_loss": 3.514084478915966,
+ "tokens_seen": 1657143296
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040235907723169513,
+ "loss": 1.2148,
+ "theoretical_loss": 3.514072556320993,
+ "tokens_seen": 1657208832
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004023430290872618,
+ "loss": 1.34,
+ "theoretical_loss": 3.5140606343295135,
+ "tokens_seen": 1657274368
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040232698094282854,
+ "loss": 1.1875,
+ "theoretical_loss": 3.514048712941473,
+ "tokens_seen": 1657339904
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004023109327983952,
+ "loss": 1.2747,
+ "theoretical_loss": 3.514036792156817,
+ "tokens_seen": 1657405440
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040229488465396194,
+ "loss": 1.2087,
+ "theoretical_loss": 3.514024871975491,
+ "tokens_seen": 1657470976
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004022788365095286,
+ "loss": 1.3389,
+ "theoretical_loss": 3.514012952397441,
+ "tokens_seen": 1657536512
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040226278836509534,
+ "loss": 1.3444,
+ "theoretical_loss": 3.5140010334226117,
+ "tokens_seen": 1657602048
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402246740220662,
+ "loss": 1.2374,
+ "theoretical_loss": 3.5139891150509497,
+ "tokens_seen": 1657667584
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040223069207622874,
+ "loss": 1.372,
+ "theoretical_loss": 3.5139771972824008,
+ "tokens_seen": 1657733120
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004022146439317954,
+ "loss": 1.2957,
+ "theoretical_loss": 3.513965280116909,
+ "tokens_seen": 1657798656
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040219859578736214,
+ "loss": 1.3287,
+ "theoretical_loss": 3.513953363554422,
+ "tokens_seen": 1657864192
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004021825476429288,
+ "loss": 1.2013,
+ "theoretical_loss": 3.5139414475948842,
+ "tokens_seen": 1657929728
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040216649949849554,
+ "loss": 1.2265,
+ "theoretical_loss": 3.513929532238242,
+ "tokens_seen": 1657995264
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 555622,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.201681137084961,
+ "objective/train/theoretical_loss": 3.5139176174844398,
+ "objective/train/tokens_used": 1678520800,
+ "theoretical_loss": 3.5139176174844398,
+ "tokens_seen": 1658060800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004021504513540622,
+ "loss": 1.2452,
+ "theoretical_loss": 3.5139176174844398,
+ "tokens_seen": 1658060800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040213440320962894,
+ "loss": 1.2517,
+ "theoretical_loss": 3.5139057033334247,
+ "tokens_seen": 1658126336
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004021183550651956,
+ "loss": 1.2518,
+ "theoretical_loss": 3.5138937897851417,
+ "tokens_seen": 1658191872
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004021183550651956,
+ "loss": 1.2347,
+ "theoretical_loss": 3.5138818768395366,
+ "tokens_seen": 1658257408
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040210230692076234,
+ "loss": 1.213,
+ "theoretical_loss": 3.513869964496555,
+ "tokens_seen": 1658322944
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000402086258776329,
+ "loss": 1.385,
+ "theoretical_loss": 3.513858052756143,
+ "tokens_seen": 1658388480
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040207021063189575,
+ "loss": 1.3498,
+ "theoretical_loss": 3.513846141618246,
+ "tokens_seen": 1658454016
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004020541624874624,
+ "loss": 1.3418,
+ "theoretical_loss": 3.513834231082809,
+ "tokens_seen": 1658519552
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040203811434302915,
+ "loss": 1.2901,
+ "theoretical_loss": 3.513822321149779,
+ "tokens_seen": 1658585088
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004020220661985958,
+ "loss": 1.2277,
+ "theoretical_loss": 3.5138104118191014,
+ "tokens_seen": 1658650624
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040200601805416255,
+ "loss": 1.2502,
+ "theoretical_loss": 3.5137985030907215,
+ "tokens_seen": 1658716160
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004019899699097292,
+ "loss": 1.328,
+ "theoretical_loss": 3.513786594964585,
+ "tokens_seen": 1658781696
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040197392176529595,
+ "loss": 1.3518,
+ "theoretical_loss": 3.5137746874406384,
+ "tokens_seen": 1658847232
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004019578736208626,
+ "loss": 1.3002,
+ "theoretical_loss": 3.5137627805188263,
+ "tokens_seen": 1658912768
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040194182547642935,
+ "loss": 1.2317,
+ "theoretical_loss": 3.5137508741990953,
+ "tokens_seen": 1658978304
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000401925777331996,
+ "loss": 1.2738,
+ "theoretical_loss": 3.5137389684813916,
+ "tokens_seen": 1659043840
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040190972918756275,
+ "loss": 1.3161,
+ "theoretical_loss": 3.5137270633656597,
+ "tokens_seen": 1659109376
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004018936810431294,
+ "loss": 1.2779,
+ "theoretical_loss": 3.5137151588518467,
+ "tokens_seen": 1659174912
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040187763289869615,
+ "loss": 1.3001,
+ "theoretical_loss": 3.513703254939897,
+ "tokens_seen": 1659240448
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004018615847542628,
+ "loss": 1.2651,
+ "theoretical_loss": 3.513691351629758,
+ "tokens_seen": 1659305984
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040184553660982955,
+ "loss": 1.2889,
+ "theoretical_loss": 3.5136794489213736,
+ "tokens_seen": 1659371520
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004018294884653962,
+ "loss": 1.287,
+ "theoretical_loss": 3.5136675468146916,
+ "tokens_seen": 1659437056
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040181344032096296,
+ "loss": 1.2384,
+ "theoretical_loss": 3.513655645309657,
+ "tokens_seen": 1659502592
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017973921765296,
+ "loss": 1.3519,
+ "theoretical_loss": 3.513643744406215,
+ "tokens_seen": 1659568128
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017813440320963,
+ "loss": 1.3794,
+ "theoretical_loss": 3.513631844104312,
+ "tokens_seen": 1659633664
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000401765295887663,
+ "loss": 1.2307,
+ "theoretical_loss": 3.513619944403894,
+ "tokens_seen": 1659699200
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017492477432297,
+ "loss": 1.3276,
+ "theoretical_loss": 3.5136080453049066,
+ "tokens_seen": 1659764736
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017331995987964,
+ "loss": 1.1935,
+ "theoretical_loss": 3.513596146807296,
+ "tokens_seen": 1659830272
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017171514543631,
+ "loss": 1.2553,
+ "theoretical_loss": 3.513584248911007,
+ "tokens_seen": 1659895808
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004017011033099298,
+ "loss": 1.2243,
+ "theoretical_loss": 3.513572351615987,
+ "tokens_seen": 1659961344
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004016850551654965,
+ "loss": 1.328,
+ "theoretical_loss": 3.513560454922181,
+ "tokens_seen": 1660026880
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004016690070210632,
+ "loss": 1.2381,
+ "theoretical_loss": 3.5135485588295348,
+ "tokens_seen": 1660092416
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004016529588766299,
+ "loss": 1.2497,
+ "theoretical_loss": 3.513536663337995,
+ "tokens_seen": 1660157952
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040163691073219655,
+ "loss": 1.3023,
+ "theoretical_loss": 3.513524768447507,
+ "tokens_seen": 1660223488
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004016208625877633,
+ "loss": 1.2794,
+ "theoretical_loss": 3.5135128741580157,
+ "tokens_seen": 1660289024
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040160481444332996,
+ "loss": 1.2542,
+ "theoretical_loss": 3.513500980469469,
+ "tokens_seen": 1660354560
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004015887662988967,
+ "loss": 1.308,
+ "theoretical_loss": 3.513489087381812,
+ "tokens_seen": 1660420096
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040157271815446336,
+ "loss": 1.3863,
+ "theoretical_loss": 3.51347719489499,
+ "tokens_seen": 1660485632
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004015566700100301,
+ "loss": 1.302,
+ "theoretical_loss": 3.513465303008949,
+ "tokens_seen": 1660551168
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040154062186559676,
+ "loss": 1.3184,
+ "theoretical_loss": 3.513453411723636,
+ "tokens_seen": 1660616704
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004015245737211635,
+ "loss": 1.3139,
+ "theoretical_loss": 3.513441521038996,
+ "tokens_seen": 1660682240
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040150852557673016,
+ "loss": 1.2198,
+ "theoretical_loss": 3.5134296309549757,
+ "tokens_seen": 1660747776
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004014924774322969,
+ "loss": 1.2535,
+ "theoretical_loss": 3.51341774147152,
+ "tokens_seen": 1660813312
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040147642928786356,
+ "loss": 1.2408,
+ "theoretical_loss": 3.513405852588576,
+ "tokens_seen": 1660878848
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004014603811434303,
+ "loss": 1.3916,
+ "theoretical_loss": 3.513393964306089,
+ "tokens_seen": 1660944384
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040144433299899696,
+ "loss": 1.3197,
+ "theoretical_loss": 3.513382076624005,
+ "tokens_seen": 1661009920
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004014282848545637,
+ "loss": 1.1997,
+ "theoretical_loss": 3.51337018954227,
+ "tokens_seen": 1661075456
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040141223671013036,
+ "loss": 1.2186,
+ "theoretical_loss": 3.5133583030608304,
+ "tokens_seen": 1661140992
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004013961885656971,
+ "loss": 1.3315,
+ "theoretical_loss": 3.513346417179632,
+ "tokens_seen": 1661206528
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040138014042126376,
+ "loss": 1.3006,
+ "theoretical_loss": 3.5133345318986207,
+ "tokens_seen": 1661272064
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 556798,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1186680793762207,
+ "objective/train/theoretical_loss": 3.513322647217742,
+ "objective/train/tokens_used": 1681797600,
+ "theoretical_loss": 3.513322647217742,
+ "tokens_seen": 1661337600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004013640922768305,
+ "loss": 1.1569,
+ "theoretical_loss": 3.513322647217742,
+ "tokens_seen": 1661337600
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040134804413239717,
+ "loss": 1.2233,
+ "theoretical_loss": 3.5133107631369436,
+ "tokens_seen": 1661403136
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004013319959879639,
+ "loss": 1.3372,
+ "theoretical_loss": 3.51329887965617,
+ "tokens_seen": 1661468672
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040131594784353057,
+ "loss": 1.1737,
+ "theoretical_loss": 3.513286996775367,
+ "tokens_seen": 1661534208
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004012998996990973,
+ "loss": 1.3467,
+ "theoretical_loss": 3.513275114494482,
+ "tokens_seen": 1661599744
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040128385155466397,
+ "loss": 1.327,
+ "theoretical_loss": 3.5132632328134603,
+ "tokens_seen": 1661665280
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004012678034102307,
+ "loss": 1.333,
+ "theoretical_loss": 3.5132513517322477,
+ "tokens_seen": 1661730816
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040125175526579737,
+ "loss": 1.3408,
+ "theoretical_loss": 3.513239471250791,
+ "tokens_seen": 1661796352
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004012357071213641,
+ "loss": 1.2169,
+ "theoretical_loss": 3.513227591369035,
+ "tokens_seen": 1661861888
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040121965897693077,
+ "loss": 1.2781,
+ "theoretical_loss": 3.513215712086928,
+ "tokens_seen": 1661927424
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004012036108324975,
+ "loss": 1.3716,
+ "theoretical_loss": 3.513203833404414,
+ "tokens_seen": 1661992960
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004011875626880643,
+ "loss": 1.299,
+ "theoretical_loss": 3.5131919553214397,
+ "tokens_seen": 1662058496
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004011715145436309,
+ "loss": 1.2452,
+ "theoretical_loss": 3.513180077837951,
+ "tokens_seen": 1662124032
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040115546639919763,
+ "loss": 1.2357,
+ "theoretical_loss": 3.5131682009538947,
+ "tokens_seen": 1662189568
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040113941825476433,
+ "loss": 1.3702,
+ "theoretical_loss": 3.513156324669217,
+ "tokens_seen": 1662255104
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040112337011033103,
+ "loss": 1.2116,
+ "theoretical_loss": 3.513144448983863,
+ "tokens_seen": 1662320640
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040110732196589773,
+ "loss": 1.3076,
+ "theoretical_loss": 3.5131325738977797,
+ "tokens_seen": 1662386176
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040109127382146443,
+ "loss": 1.2959,
+ "theoretical_loss": 3.513120699410913,
+ "tokens_seen": 1662451712
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040107522567703113,
+ "loss": 1.3756,
+ "theoretical_loss": 3.513108825523209,
+ "tokens_seen": 1662517248
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040105917753259783,
+ "loss": 1.2802,
+ "theoretical_loss": 3.513096952234613,
+ "tokens_seen": 1662582784
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040104312938816453,
+ "loss": 1.3151,
+ "theoretical_loss": 3.513085079545073,
+ "tokens_seen": 1662648320
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040102708124373123,
+ "loss": 1.3457,
+ "theoretical_loss": 3.513073207454534,
+ "tokens_seen": 1662713856
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004010110330992979,
+ "loss": 1.36,
+ "theoretical_loss": 3.5130613359629415,
+ "tokens_seen": 1662779392
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040099498495486463,
+ "loss": 1.2871,
+ "theoretical_loss": 3.5130494650702433,
+ "tokens_seen": 1662844928
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004009789368104313,
+ "loss": 1.3447,
+ "theoretical_loss": 3.5130375947763843,
+ "tokens_seen": 1662910464
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040096288866599804,
+ "loss": 1.3518,
+ "theoretical_loss": 3.513025725081312,
+ "tokens_seen": 1662976000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004009468405215647,
+ "loss": 1.3244,
+ "theoretical_loss": 3.513013855984971,
+ "tokens_seen": 1663041536
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040093079237713144,
+ "loss": 1.2671,
+ "theoretical_loss": 3.513001987487309,
+ "tokens_seen": 1663107072
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004009147442326981,
+ "loss": 1.2597,
+ "theoretical_loss": 3.5129901195882707,
+ "tokens_seen": 1663172608
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040089869608826484,
+ "loss": 1.3389,
+ "theoretical_loss": 3.5129782522878035,
+ "tokens_seen": 1663238144
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004008826479438315,
+ "loss": 1.4299,
+ "theoretical_loss": 3.512966385585853,
+ "tokens_seen": 1663303680
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040086659979939824,
+ "loss": 1.2636,
+ "theoretical_loss": 3.512954519482366,
+ "tokens_seen": 1663369216
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004008505516549649,
+ "loss": 1.3397,
+ "theoretical_loss": 3.512942653977288,
+ "tokens_seen": 1663434752
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040083450351053164,
+ "loss": 1.2753,
+ "theoretical_loss": 3.512930789070566,
+ "tokens_seen": 1663500288
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004008184553660983,
+ "loss": 1.3264,
+ "theoretical_loss": 3.512918924762146,
+ "tokens_seen": 1663565824
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040080240722166504,
+ "loss": 1.2388,
+ "theoretical_loss": 3.5129070610519744,
+ "tokens_seen": 1663631360
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004007863590772317,
+ "loss": 1.2611,
+ "theoretical_loss": 3.5128951979399967,
+ "tokens_seen": 1663696896
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040077031093279844,
+ "loss": 1.3229,
+ "theoretical_loss": 3.51288333542616,
+ "tokens_seen": 1663762432
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004007542627883651,
+ "loss": 1.3436,
+ "theoretical_loss": 3.51287147351041,
+ "tokens_seen": 1663827968
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040073821464393184,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5128596121926936,
+ "tokens_seen": 1663893504
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004007221664994985,
+ "loss": 1.3475,
+ "theoretical_loss": 3.512847751472957,
+ "tokens_seen": 1663959040
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040070611835506525,
+ "loss": 1.3899,
+ "theoretical_loss": 3.5128358913511457,
+ "tokens_seen": 1664024576
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004006900702106319,
+ "loss": 1.2378,
+ "theoretical_loss": 3.5128240318272073,
+ "tokens_seen": 1664090112
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040067402206619865,
+ "loss": 1.2986,
+ "theoretical_loss": 3.512812172901087,
+ "tokens_seen": 1664155648
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004006579739217653,
+ "loss": 1.2335,
+ "theoretical_loss": 3.512800314572732,
+ "tokens_seen": 1664221184
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040064192577733205,
+ "loss": 1.2466,
+ "theoretical_loss": 3.5127884568420873,
+ "tokens_seen": 1664286720
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004006258776328987,
+ "loss": 1.3123,
+ "theoretical_loss": 3.5127765997091007,
+ "tokens_seen": 1664352256
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040060982948846545,
+ "loss": 1.3587,
+ "theoretical_loss": 3.512764743173718,
+ "tokens_seen": 1664417792
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004005937813440321,
+ "loss": 1.2871,
+ "theoretical_loss": 3.5127528872358855,
+ "tokens_seen": 1664483328
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040057773319959885,
+ "loss": 1.2303,
+ "theoretical_loss": 3.5127410318955494,
+ "tokens_seen": 1664548864
+ },
+ {
+ "epoch": 0.5,
+ "objective/train/docs_used": 558020,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1752697229385376,
+ "objective/train/theoretical_loss": 3.5127291771526563,
+ "objective/train/tokens_used": 1685074400,
+ "theoretical_loss": 3.5127291771526563,
+ "tokens_seen": 1664614400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004005616850551655,
+ "loss": 1.1725,
+ "theoretical_loss": 3.5127291771526563,
+ "tokens_seen": 1664614400
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040054563691073225,
+ "loss": 1.2851,
+ "theoretical_loss": 3.512717323007153,
+ "tokens_seen": 1664679936
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004005295887662989,
+ "loss": 1.2361,
+ "theoretical_loss": 3.512705469458985,
+ "tokens_seen": 1664745472
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040051354062186565,
+ "loss": 1.2552,
+ "theoretical_loss": 3.512693616508099,
+ "tokens_seen": 1664811008
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004004974924774323,
+ "loss": 1.366,
+ "theoretical_loss": 3.512681764154441,
+ "tokens_seen": 1664876544
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040048144433299905,
+ "loss": 1.2966,
+ "theoretical_loss": 3.512669912397959,
+ "tokens_seen": 1664942080
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004004653961885657,
+ "loss": 1.2526,
+ "theoretical_loss": 3.5126580612385974,
+ "tokens_seen": 1665007616
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040044934804413246,
+ "loss": 1.2695,
+ "theoretical_loss": 3.5126462106763037,
+ "tokens_seen": 1665073152
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004004332998996991,
+ "loss": 1.224,
+ "theoretical_loss": 3.512634360711024,
+ "tokens_seen": 1665138688
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040041725175526586,
+ "loss": 1.3352,
+ "theoretical_loss": 3.5126225113427054,
+ "tokens_seen": 1665204224
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004004012036108325,
+ "loss": 1.3257,
+ "theoretical_loss": 3.5126106625712934,
+ "tokens_seen": 1665269760
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004003851554663992,
+ "loss": 1.2553,
+ "theoretical_loss": 3.5125988143967346,
+ "tokens_seen": 1665335296
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004003691073219659,
+ "loss": 1.279,
+ "theoretical_loss": 3.5125869668189758,
+ "tokens_seen": 1665400832
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004003530591775326,
+ "loss": 1.2295,
+ "theoretical_loss": 3.5125751198379636,
+ "tokens_seen": 1665466368
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004003370110330993,
+ "loss": 1.2436,
+ "theoretical_loss": 3.512563273453644,
+ "tokens_seen": 1665531904
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000400320962888666,
+ "loss": 1.169,
+ "theoretical_loss": 3.5125514276659637,
+ "tokens_seen": 1665597440
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004003049147442327,
+ "loss": 1.2532,
+ "theoretical_loss": 3.512539582474869,
+ "tokens_seen": 1665662976
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004002888665997994,
+ "loss": 1.3368,
+ "theoretical_loss": 3.5125277378803066,
+ "tokens_seen": 1665728512
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004002728184553661,
+ "loss": 1.3694,
+ "theoretical_loss": 3.512515893882223,
+ "tokens_seen": 1665794048
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004002567703109328,
+ "loss": 1.251,
+ "theoretical_loss": 3.5125040504805645,
+ "tokens_seen": 1665859584
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040024072216649946,
+ "loss": 1.3366,
+ "theoretical_loss": 3.512492207675278,
+ "tokens_seen": 1665925120
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004002246740220662,
+ "loss": 1.2784,
+ "theoretical_loss": 3.5124803654663093,
+ "tokens_seen": 1665990656
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040020862587763286,
+ "loss": 1.3781,
+ "theoretical_loss": 3.5124685238536055,
+ "tokens_seen": 1666056192
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004001925777331996,
+ "loss": 1.3585,
+ "theoretical_loss": 3.512456682837113,
+ "tokens_seen": 1666121728
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040017652958876626,
+ "loss": 1.2931,
+ "theoretical_loss": 3.5124448424167785,
+ "tokens_seen": 1666187264
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.000400160481444333,
+ "loss": 1.259,
+ "theoretical_loss": 3.5124330025925476,
+ "tokens_seen": 1666252800
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040014443329989966,
+ "loss": 1.2293,
+ "theoretical_loss": 3.512421163364368,
+ "tokens_seen": 1666318336
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.0004001283851554664,
+ "loss": 1.2357,
+ "theoretical_loss": 3.512409324732186,
+ "tokens_seen": 1666383872
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 0.00040011233701103306,
+ "loss": 1.3073,
+ "theoretical_loss": 3.512397486695948,
+ "tokens_seen": 1666449408
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0004000962888665998,
+ "loss": 1.2175,
+ "theoretical_loss": 3.5123856492556005,
+ "tokens_seen": 1666514944
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00040008024072216646,
+ "loss": 1.246,
+ "theoretical_loss": 3.5123738124110897,
+ "tokens_seen": 1666580480
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0004000641925777332,
+ "loss": 1.2875,
+ "theoretical_loss": 3.5123619761623632,
+ "tokens_seen": 1666646016
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00040004814443329986,
+ "loss": 1.2551,
+ "theoretical_loss": 3.5123501405093664,
+ "tokens_seen": 1666711552
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0004000320962888666,
+ "loss": 1.3639,
+ "theoretical_loss": 3.5123383054520474,
+ "tokens_seen": 1666777088
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00040001604814443326,
+ "loss": 1.2718,
+ "theoretical_loss": 3.5123264709903514,
+ "tokens_seen": 1666842624
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0004,
+ "loss": 1.317,
+ "theoretical_loss": 3.5123146371242253,
+ "tokens_seen": 1666908160
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999839518555667,
+ "loss": 1.2167,
+ "theoretical_loss": 3.5123028038536157,
+ "tokens_seen": 1666973696
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999679037111334,
+ "loss": 1.2887,
+ "theoretical_loss": 3.51229097117847,
+ "tokens_seen": 1667039232
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999518555667001,
+ "loss": 1.3884,
+ "theoretical_loss": 3.512279139098734,
+ "tokens_seen": 1667104768
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999358074222668,
+ "loss": 1.247,
+ "theoretical_loss": 3.512267307614355,
+ "tokens_seen": 1667170304
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999197592778335,
+ "loss": 1.2982,
+ "theoretical_loss": 3.5122554767252785,
+ "tokens_seen": 1667235840
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003999037111334002,
+ "loss": 1.3163,
+ "theoretical_loss": 3.512243646431452,
+ "tokens_seen": 1667301376
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003998876629889669,
+ "loss": 1.2875,
+ "theoretical_loss": 3.5122318167328226,
+ "tokens_seen": 1667366912
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003998716148445336,
+ "loss": 1.2666,
+ "theoretical_loss": 3.512219987629336,
+ "tokens_seen": 1667432448
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003998555667001003,
+ "loss": 1.2264,
+ "theoretical_loss": 3.5122081591209393,
+ "tokens_seen": 1667497984
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000399839518555667,
+ "loss": 1.2592,
+ "theoretical_loss": 3.512196331207579,
+ "tokens_seen": 1667563520
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003998234704112337,
+ "loss": 1.3014,
+ "theoretical_loss": 3.5121845038892023,
+ "tokens_seen": 1667629056
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003998074222668004,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5121726771657555,
+ "tokens_seen": 1667694592
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039979137412236713,
+ "loss": 1.2329,
+ "theoretical_loss": 3.512160851037185,
+ "tokens_seen": 1667760128
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039977532597793383,
+ "loss": 1.3256,
+ "theoretical_loss": 3.512149025503438,
+ "tokens_seen": 1667825664
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 558924,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3271944522857666,
+ "objective/train/theoretical_loss": 3.512137200564461,
+ "objective/train/tokens_used": 1688351200,
+ "theoretical_loss": 3.512137200564461,
+ "tokens_seen": 1667891200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039975927783350053,
+ "loss": 1.3634,
+ "theoretical_loss": 3.512137200564461,
+ "tokens_seen": 1667891200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039974322968906723,
+ "loss": 1.3186,
+ "theoretical_loss": 3.512125376220201,
+ "tokens_seen": 1667956736
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039972718154463393,
+ "loss": 1.2646,
+ "theoretical_loss": 3.5121135524706038,
+ "tokens_seen": 1668022272
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039971113340020063,
+ "loss": 1.3485,
+ "theoretical_loss": 3.5121017293156176,
+ "tokens_seen": 1668087808
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039969508525576733,
+ "loss": 1.3102,
+ "theoretical_loss": 3.512089906755188,
+ "tokens_seen": 1668153344
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039967903711133403,
+ "loss": 1.328,
+ "theoretical_loss": 3.512078084789262,
+ "tokens_seen": 1668218880
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039966298896690073,
+ "loss": 1.2124,
+ "theoretical_loss": 3.5120662634177866,
+ "tokens_seen": 1668284416
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039964694082246743,
+ "loss": 1.2866,
+ "theoretical_loss": 3.512054442640708,
+ "tokens_seen": 1668349952
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039963089267803413,
+ "loss": 1.2512,
+ "theoretical_loss": 3.512042622457974,
+ "tokens_seen": 1668415488
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039961484453360083,
+ "loss": 1.3261,
+ "theoretical_loss": 3.51203080286953,
+ "tokens_seen": 1668481024
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995987963891675,
+ "loss": 1.2457,
+ "theoretical_loss": 3.512018983875324,
+ "tokens_seen": 1668546560
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995827482447342,
+ "loss": 1.3881,
+ "theoretical_loss": 3.5120071654753025,
+ "tokens_seen": 1668612096
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995667001003009,
+ "loss": 1.3635,
+ "theoretical_loss": 3.511995347669412,
+ "tokens_seen": 1668677632
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995506519558676,
+ "loss": 1.3337,
+ "theoretical_loss": 3.511983530457599,
+ "tokens_seen": 1668743168
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995346038114343,
+ "loss": 1.3834,
+ "theoretical_loss": 3.511971713839811,
+ "tokens_seen": 1668808704
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000399518555667001,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5119598978159945,
+ "tokens_seen": 1668874240
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003995025075225677,
+ "loss": 1.2736,
+ "theoretical_loss": 3.5119480823860965,
+ "tokens_seen": 1668939776
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994864593781344,
+ "loss": 1.2634,
+ "theoretical_loss": 3.5119362675500634,
+ "tokens_seen": 1669005312
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994704112337011,
+ "loss": 1.2757,
+ "theoretical_loss": 3.5119244533078424,
+ "tokens_seen": 1669070848
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994543630892678,
+ "loss": 1.2338,
+ "theoretical_loss": 3.51191263965938,
+ "tokens_seen": 1669136384
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994383149448345,
+ "loss": 1.1499,
+ "theoretical_loss": 3.511900826604623,
+ "tokens_seen": 1669201920
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994222668004012,
+ "loss": 1.3141,
+ "theoretical_loss": 3.5118890141435193,
+ "tokens_seen": 1669267456
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003994062186559679,
+ "loss": 1.3507,
+ "theoretical_loss": 3.5118772022760147,
+ "tokens_seen": 1669332992
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039939017051153464,
+ "loss": 1.252,
+ "theoretical_loss": 3.5118653910020563,
+ "tokens_seen": 1669398528
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039937412236710134,
+ "loss": 1.2746,
+ "theoretical_loss": 3.511853580321591,
+ "tokens_seen": 1669464064
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039935807422266804,
+ "loss": 1.3705,
+ "theoretical_loss": 3.511841770234566,
+ "tokens_seen": 1669529600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039934202607823475,
+ "loss": 1.2305,
+ "theoretical_loss": 3.511829960740928,
+ "tokens_seen": 1669595136
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039932597793380145,
+ "loss": 1.266,
+ "theoretical_loss": 3.5118181518406235,
+ "tokens_seen": 1669660672
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039930992978936815,
+ "loss": 1.3566,
+ "theoretical_loss": 3.5118063435335998,
+ "tokens_seen": 1669726208
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039929388164493485,
+ "loss": 1.3382,
+ "theoretical_loss": 3.5117945358198037,
+ "tokens_seen": 1669791744
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039927783350050155,
+ "loss": 1.321,
+ "theoretical_loss": 3.511782728699182,
+ "tokens_seen": 1669857280
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039926178535606825,
+ "loss": 1.2712,
+ "theoretical_loss": 3.5117709221716815,
+ "tokens_seen": 1669922816
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039924573721163495,
+ "loss": 1.2721,
+ "theoretical_loss": 3.5117591162372497,
+ "tokens_seen": 1669988352
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039922968906720165,
+ "loss": 1.3125,
+ "theoretical_loss": 3.5117473108958337,
+ "tokens_seen": 1670053888
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039921364092276835,
+ "loss": 1.3768,
+ "theoretical_loss": 3.5117355061473794,
+ "tokens_seen": 1670119424
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039919759277833505,
+ "loss": 1.2725,
+ "theoretical_loss": 3.5117237019918344,
+ "tokens_seen": 1670184960
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039918154463390175,
+ "loss": 1.2311,
+ "theoretical_loss": 3.5117118984291458,
+ "tokens_seen": 1670250496
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039916549648946845,
+ "loss": 1.2508,
+ "theoretical_loss": 3.5117000954592603,
+ "tokens_seen": 1670316032
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039914944834503515,
+ "loss": 1.2023,
+ "theoretical_loss": 3.5116882930821247,
+ "tokens_seen": 1670381568
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039913340020060185,
+ "loss": 1.2607,
+ "theoretical_loss": 3.5116764912976866,
+ "tokens_seen": 1670447104
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039911735205616855,
+ "loss": 1.4079,
+ "theoretical_loss": 3.511664690105892,
+ "tokens_seen": 1670512640
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039910130391173525,
+ "loss": 1.2866,
+ "theoretical_loss": 3.5116528895066885,
+ "tokens_seen": 1670578176
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039908525576730196,
+ "loss": 1.3169,
+ "theoretical_loss": 3.5116410895000234,
+ "tokens_seen": 1670643712
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039906920762286866,
+ "loss": 1.3179,
+ "theoretical_loss": 3.5116292900858435,
+ "tokens_seen": 1670709248
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039905315947843536,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5116174912640954,
+ "tokens_seen": 1670774784
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039903711133400206,
+ "loss": 1.3305,
+ "theoretical_loss": 3.5116056930347264,
+ "tokens_seen": 1670840320
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039902106318956876,
+ "loss": 1.2188,
+ "theoretical_loss": 3.5115938953976835,
+ "tokens_seen": 1670905856
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039900501504513546,
+ "loss": 1.2628,
+ "theoretical_loss": 3.511582098352914,
+ "tokens_seen": 1670971392
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039898896690070216,
+ "loss": 1.2478,
+ "theoretical_loss": 3.5115703019003646,
+ "tokens_seen": 1671036928
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003989729187562688,
+ "loss": 1.3259,
+ "theoretical_loss": 3.5115585060399823,
+ "tokens_seen": 1671102464
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 560108,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.264222502708435,
+ "objective/train/theoretical_loss": 3.5115467107717144,
+ "objective/train/tokens_used": 1691628000,
+ "theoretical_loss": 3.5115467107717144,
+ "tokens_seen": 1671168000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003989568706118355,
+ "loss": 1.215,
+ "theoretical_loss": 3.5115467107717144,
+ "tokens_seen": 1671168000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003989408224674022,
+ "loss": 1.2237,
+ "theoretical_loss": 3.5115349160955076,
+ "tokens_seen": 1671233536
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003989247743229689,
+ "loss": 1.2811,
+ "theoretical_loss": 3.5115231220113095,
+ "tokens_seen": 1671299072
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003989087261785356,
+ "loss": 1.2688,
+ "theoretical_loss": 3.5115113285190667,
+ "tokens_seen": 1671364608
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003988926780341023,
+ "loss": 1.3173,
+ "theoretical_loss": 3.511499535618727,
+ "tokens_seen": 1671430144
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000398876629889669,
+ "loss": 1.3643,
+ "theoretical_loss": 3.5114877433102363,
+ "tokens_seen": 1671495680
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003988605817452357,
+ "loss": 1.3867,
+ "theoretical_loss": 3.5114759515935425,
+ "tokens_seen": 1671561216
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003988445336008024,
+ "loss": 1.4443,
+ "theoretical_loss": 3.5114641604685928,
+ "tokens_seen": 1671626752
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003988284854563691,
+ "loss": 1.2469,
+ "theoretical_loss": 3.5114523699353337,
+ "tokens_seen": 1671692288
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003988124373119358,
+ "loss": 1.299,
+ "theoretical_loss": 3.511440579993713,
+ "tokens_seen": 1671757824
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987963891675025,
+ "loss": 1.3172,
+ "theoretical_loss": 3.511428790643677,
+ "tokens_seen": 1671823360
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987803410230692,
+ "loss": 1.4323,
+ "theoretical_loss": 3.5114170018851736,
+ "tokens_seen": 1671888896
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987642928786359,
+ "loss": 1.3876,
+ "theoretical_loss": 3.51140521371815,
+ "tokens_seen": 1671954432
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987482447342026,
+ "loss": 1.2385,
+ "theoretical_loss": 3.511393426142553,
+ "tokens_seen": 1672019968
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987321965897693,
+ "loss": 1.3177,
+ "theoretical_loss": 3.511381639158329,
+ "tokens_seen": 1672085504
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000398716148445336,
+ "loss": 1.3065,
+ "theoretical_loss": 3.5113698527654265,
+ "tokens_seen": 1672151040
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003987001003009027,
+ "loss": 1.29,
+ "theoretical_loss": 3.5113580669637923,
+ "tokens_seen": 1672216576
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986840521564694,
+ "loss": 1.2656,
+ "theoretical_loss": 3.5113462817533727,
+ "tokens_seen": 1672282112
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986680040120361,
+ "loss": 1.2402,
+ "theoretical_loss": 3.511334497134116,
+ "tokens_seen": 1672347648
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986519558676028,
+ "loss": 1.3029,
+ "theoretical_loss": 3.5113227131059688,
+ "tokens_seen": 1672413184
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986359077231695,
+ "loss": 1.2444,
+ "theoretical_loss": 3.511310929668878,
+ "tokens_seen": 1672478720
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986198595787362,
+ "loss": 1.435,
+ "theoretical_loss": 3.5112991468227914,
+ "tokens_seen": 1672544256
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003986038114343029,
+ "loss": 1.2414,
+ "theoretical_loss": 3.5112873645676554,
+ "tokens_seen": 1672609792
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003985877632898696,
+ "loss": 1.314,
+ "theoretical_loss": 3.511275582903419,
+ "tokens_seen": 1672675328
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003985717151454363,
+ "loss": 1.3716,
+ "theoretical_loss": 3.5112638018300273,
+ "tokens_seen": 1672740864
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000398555667001003,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5112520213474285,
+ "tokens_seen": 1672806400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003985396188565697,
+ "loss": 1.2805,
+ "theoretical_loss": 3.51124024145557,
+ "tokens_seen": 1672871936
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003985235707121364,
+ "loss": 1.2475,
+ "theoretical_loss": 3.5112284621543983,
+ "tokens_seen": 1672937472
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003985075225677031,
+ "loss": 1.3827,
+ "theoretical_loss": 3.5112166834438616,
+ "tokens_seen": 1673003008
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003984914744232698,
+ "loss": 1.3495,
+ "theoretical_loss": 3.5112049053239067,
+ "tokens_seen": 1673068544
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003984754262788365,
+ "loss": 1.2697,
+ "theoretical_loss": 3.511193127794481,
+ "tokens_seen": 1673134080
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003984593781344032,
+ "loss": 1.1476,
+ "theoretical_loss": 3.511181350855531,
+ "tokens_seen": 1673199616
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003984433299899699,
+ "loss": 1.2646,
+ "theoretical_loss": 3.511169574507005,
+ "tokens_seen": 1673265152
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039842728184553663,
+ "loss": 1.2672,
+ "theoretical_loss": 3.5111577987488496,
+ "tokens_seen": 1673330688
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039841123370110333,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5111460235810124,
+ "tokens_seen": 1673396224
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039839518555667003,
+ "loss": 1.305,
+ "theoretical_loss": 3.5111342490034407,
+ "tokens_seen": 1673461760
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039837913741223673,
+ "loss": 1.3364,
+ "theoretical_loss": 3.511122475016082,
+ "tokens_seen": 1673527296
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039836308926780343,
+ "loss": 1.2294,
+ "theoretical_loss": 3.5111107016188825,
+ "tokens_seen": 1673592832
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039834704112337013,
+ "loss": 1.3126,
+ "theoretical_loss": 3.5110989288117906,
+ "tokens_seen": 1673658368
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039833099297893683,
+ "loss": 1.315,
+ "theoretical_loss": 3.511087156594754,
+ "tokens_seen": 1673723904
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039831494483450353,
+ "loss": 1.3515,
+ "theoretical_loss": 3.5110753849677185,
+ "tokens_seen": 1673789440
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039829889669007023,
+ "loss": 1.3381,
+ "theoretical_loss": 3.5110636139306326,
+ "tokens_seen": 1673854976
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039828284854563693,
+ "loss": 1.4816,
+ "theoretical_loss": 3.5110518434834432,
+ "tokens_seen": 1673920512
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039826680040120363,
+ "loss": 1.2884,
+ "theoretical_loss": 3.5110400736260985,
+ "tokens_seen": 1673986048
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039825075225677033,
+ "loss": 1.1816,
+ "theoretical_loss": 3.511028304358544,
+ "tokens_seen": 1674051584
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039823470411233703,
+ "loss": 1.4133,
+ "theoretical_loss": 3.5110165356807292,
+ "tokens_seen": 1674117120
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039821865596790374,
+ "loss": 1.3351,
+ "theoretical_loss": 3.5110047675926,
+ "tokens_seen": 1674182656
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003982026078234704,
+ "loss": 1.3084,
+ "theoretical_loss": 3.5109930000941043,
+ "tokens_seen": 1674248192
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039818655967903714,
+ "loss": 1.3028,
+ "theoretical_loss": 3.5109812331851895,
+ "tokens_seen": 1674313728
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039817051153460384,
+ "loss": 1.2348,
+ "theoretical_loss": 3.5109694668658027,
+ "tokens_seen": 1674379264
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 561113,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4016382694244385,
+ "objective/train/theoretical_loss": 3.5109577011358915,
+ "objective/train/tokens_used": 1694904800,
+ "theoretical_loss": 3.5109577011358915,
+ "tokens_seen": 1674444800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039815446339017054,
+ "loss": 1.3322,
+ "theoretical_loss": 3.5109577011358915,
+ "tokens_seen": 1674444800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039813841524573724,
+ "loss": 1.4314,
+ "theoretical_loss": 3.5109459359954034,
+ "tokens_seen": 1674510336
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039812236710130394,
+ "loss": 1.3244,
+ "theoretical_loss": 3.5109341714442857,
+ "tokens_seen": 1674575872
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039810631895687064,
+ "loss": 1.3995,
+ "theoretical_loss": 3.510922407482486,
+ "tokens_seen": 1674641408
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039809027081243734,
+ "loss": 1.2781,
+ "theoretical_loss": 3.510910644109951,
+ "tokens_seen": 1674706944
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039807422266800404,
+ "loss": 1.5037,
+ "theoretical_loss": 3.5108988813266286,
+ "tokens_seen": 1674772480
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039805817452357074,
+ "loss": 1.2534,
+ "theoretical_loss": 3.510887119132467,
+ "tokens_seen": 1674838016
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039804212637913744,
+ "loss": 1.4516,
+ "theoretical_loss": 3.510875357527412,
+ "tokens_seen": 1674903552
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039802607823470414,
+ "loss": 1.2991,
+ "theoretical_loss": 3.5108635965114123,
+ "tokens_seen": 1674969088
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039801003009027084,
+ "loss": 1.2832,
+ "theoretical_loss": 3.510851836084415,
+ "tokens_seen": 1675034624
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039799398194583754,
+ "loss": 1.2692,
+ "theoretical_loss": 3.510840076246368,
+ "tokens_seen": 1675100160
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039797793380140424,
+ "loss": 1.3077,
+ "theoretical_loss": 3.510828316997218,
+ "tokens_seen": 1675165696
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039796188565697095,
+ "loss": 1.3315,
+ "theoretical_loss": 3.510816558336913,
+ "tokens_seen": 1675231232
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039794583751253765,
+ "loss": 1.3603,
+ "theoretical_loss": 3.5108048002654,
+ "tokens_seen": 1675296768
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039792978936810435,
+ "loss": 1.4642,
+ "theoretical_loss": 3.510793042782627,
+ "tokens_seen": 1675362304
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039791374122367105,
+ "loss": 1.2901,
+ "theoretical_loss": 3.510781285888541,
+ "tokens_seen": 1675427840
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039789769307923775,
+ "loss": 1.2233,
+ "theoretical_loss": 3.5107695295830896,
+ "tokens_seen": 1675493376
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039788164493480445,
+ "loss": 1.2658,
+ "theoretical_loss": 3.510757773866221,
+ "tokens_seen": 1675558912
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039786559679037115,
+ "loss": 1.2616,
+ "theoretical_loss": 3.5107460187378816,
+ "tokens_seen": 1675624448
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039784954864593785,
+ "loss": 1.3663,
+ "theoretical_loss": 3.51073426419802,
+ "tokens_seen": 1675689984
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039783350050150455,
+ "loss": 1.3221,
+ "theoretical_loss": 3.510722510246583,
+ "tokens_seen": 1675755520
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039781745235707125,
+ "loss": 1.4027,
+ "theoretical_loss": 3.5107107568835185,
+ "tokens_seen": 1675821056
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039780140421263795,
+ "loss": 1.333,
+ "theoretical_loss": 3.5106990041087736,
+ "tokens_seen": 1675886592
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039778535606820465,
+ "loss": 1.286,
+ "theoretical_loss": 3.5106872519222962,
+ "tokens_seen": 1675952128
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039776930792377135,
+ "loss": 1.3235,
+ "theoretical_loss": 3.510675500324034,
+ "tokens_seen": 1676017664
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039775325977933805,
+ "loss": 1.2732,
+ "theoretical_loss": 3.5106637493139345,
+ "tokens_seen": 1676083200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039773721163490475,
+ "loss": 1.3502,
+ "theoretical_loss": 3.5106519988919453,
+ "tokens_seen": 1676148736
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039772116349047146,
+ "loss": 1.2631,
+ "theoretical_loss": 3.510640249058013,
+ "tokens_seen": 1676214272
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039770511534603816,
+ "loss": 1.4787,
+ "theoretical_loss": 3.5106284998120865,
+ "tokens_seen": 1676279808
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039768906720160486,
+ "loss": 1.3185,
+ "theoretical_loss": 3.510616751154113,
+ "tokens_seen": 1676345344
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039767301905717156,
+ "loss": 1.3165,
+ "theoretical_loss": 3.5106050030840397,
+ "tokens_seen": 1676410880
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039765697091273826,
+ "loss": 1.2874,
+ "theoretical_loss": 3.5105932556018145,
+ "tokens_seen": 1676476416
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039764092276830496,
+ "loss": 1.3481,
+ "theoretical_loss": 3.510581508707385,
+ "tokens_seen": 1676541952
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039762487462387166,
+ "loss": 1.1868,
+ "theoretical_loss": 3.510569762400699,
+ "tokens_seen": 1676607488
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039760882647943836,
+ "loss": 1.4,
+ "theoretical_loss": 3.5105580166817036,
+ "tokens_seen": 1676673024
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039759277833500506,
+ "loss": 1.374,
+ "theoretical_loss": 3.510546271550347,
+ "tokens_seen": 1676738560
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003975767301905717,
+ "loss": 1.3006,
+ "theoretical_loss": 3.510534527006576,
+ "tokens_seen": 1676804096
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003975606820461384,
+ "loss": 1.2967,
+ "theoretical_loss": 3.510522783050339,
+ "tokens_seen": 1676869632
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003975446339017051,
+ "loss": 1.3701,
+ "theoretical_loss": 3.5105110396815844,
+ "tokens_seen": 1676935168
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003975285857572718,
+ "loss": 1.5421,
+ "theoretical_loss": 3.5104992969002575,
+ "tokens_seen": 1677000704
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003975125376128385,
+ "loss": 1.4495,
+ "theoretical_loss": 3.510487554706309,
+ "tokens_seen": 1677066240
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974964894684052,
+ "loss": 1.33,
+ "theoretical_loss": 3.5104758130996836,
+ "tokens_seen": 1677131776
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974804413239719,
+ "loss": 1.2582,
+ "theoretical_loss": 3.5104640720803304,
+ "tokens_seen": 1677197312
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974643931795386,
+ "loss": 1.3215,
+ "theoretical_loss": 3.5104523316481973,
+ "tokens_seen": 1677262848
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974483450351053,
+ "loss": 1.2341,
+ "theoretical_loss": 3.5104405918032318,
+ "tokens_seen": 1677328384
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000397432296890672,
+ "loss": 1.3402,
+ "theoretical_loss": 3.510428852545381,
+ "tokens_seen": 1677393920
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974162487462387,
+ "loss": 1.3536,
+ "theoretical_loss": 3.5104171138745937,
+ "tokens_seen": 1677459456
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003974002006018054,
+ "loss": 1.2893,
+ "theoretical_loss": 3.5104053757908167,
+ "tokens_seen": 1677524992
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973841524573721,
+ "loss": 1.3752,
+ "theoretical_loss": 3.510393638293998,
+ "tokens_seen": 1677590528
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973681043129388,
+ "loss": 1.4102,
+ "theoretical_loss": 3.5103819013840853,
+ "tokens_seen": 1677656064
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 562346,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.505894422531128,
+ "objective/train/theoretical_loss": 3.5103701650610266,
+ "objective/train/tokens_used": 1698181600,
+ "theoretical_loss": 3.5103701650610266,
+ "tokens_seen": 1677721600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973520561685055,
+ "loss": 1.391,
+ "theoretical_loss": 3.5103701650610266,
+ "tokens_seen": 1677721600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973360080240722,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5103584293247687,
+ "tokens_seen": 1677787136
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973199598796389,
+ "loss": 1.3318,
+ "theoretical_loss": 3.5103466941752606,
+ "tokens_seen": 1677852672
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003973039117352056,
+ "loss": 1.4684,
+ "theoretical_loss": 3.5103349596124493,
+ "tokens_seen": 1677918208
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003972878635907723,
+ "loss": 1.3585,
+ "theoretical_loss": 3.510323225636283,
+ "tokens_seen": 1677983744
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000397271815446339,
+ "loss": 1.2919,
+ "theoretical_loss": 3.510311492246709,
+ "tokens_seen": 1678049280
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003972557673019057,
+ "loss": 1.4674,
+ "theoretical_loss": 3.5102997594436753,
+ "tokens_seen": 1678114816
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003972397191574724,
+ "loss": 1.2753,
+ "theoretical_loss": 3.5102880272271295,
+ "tokens_seen": 1678180352
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003972236710130391,
+ "loss": 1.2766,
+ "theoretical_loss": 3.51027629559702,
+ "tokens_seen": 1678245888
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003972076228686058,
+ "loss": 1.266,
+ "theoretical_loss": 3.5102645645532933,
+ "tokens_seen": 1678311424
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003971915747241725,
+ "loss": 1.2545,
+ "theoretical_loss": 3.5102528340958985,
+ "tokens_seen": 1678376960
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003971755265797392,
+ "loss": 1.3106,
+ "theoretical_loss": 3.5102411042247823,
+ "tokens_seen": 1678442496
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003971594784353059,
+ "loss": 1.2323,
+ "theoretical_loss": 3.510229374939894,
+ "tokens_seen": 1678508032
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003971434302908726,
+ "loss": 1.3667,
+ "theoretical_loss": 3.51021764624118,
+ "tokens_seen": 1678573568
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003971273821464393,
+ "loss": 1.3376,
+ "theoretical_loss": 3.5102059181285887,
+ "tokens_seen": 1678639104
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000397111334002006,
+ "loss": 1.3192,
+ "theoretical_loss": 3.5101941906020677,
+ "tokens_seen": 1678704640
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003970952858575727,
+ "loss": 1.3216,
+ "theoretical_loss": 3.5101824636615655,
+ "tokens_seen": 1678770176
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003970792377131394,
+ "loss": 1.3051,
+ "theoretical_loss": 3.510170737307029,
+ "tokens_seen": 1678835712
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003970631895687061,
+ "loss": 1.3591,
+ "theoretical_loss": 3.510159011538407,
+ "tokens_seen": 1678901248
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039704714142427283,
+ "loss": 1.2935,
+ "theoretical_loss": 3.510147286355646,
+ "tokens_seen": 1678966784
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039703109327983953,
+ "loss": 1.395,
+ "theoretical_loss": 3.5101355617586956,
+ "tokens_seen": 1679032320
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039701504513540623,
+ "loss": 1.348,
+ "theoretical_loss": 3.5101238377475026,
+ "tokens_seen": 1679097856
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039699899699097293,
+ "loss": 1.2281,
+ "theoretical_loss": 3.5101121143220144,
+ "tokens_seen": 1679163392
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003969829488465397,
+ "loss": 1.4958,
+ "theoretical_loss": 3.5101003914821804,
+ "tokens_seen": 1679228928
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003969669007021064,
+ "loss": 1.2386,
+ "theoretical_loss": 3.5100886692279474,
+ "tokens_seen": 1679294464
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003969508525576731,
+ "loss": 1.3555,
+ "theoretical_loss": 3.510076947559263,
+ "tokens_seen": 1679360000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039693480441323973,
+ "loss": 1.3538,
+ "theoretical_loss": 3.510065226476076,
+ "tokens_seen": 1679425536
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039691875626880643,
+ "loss": 1.3413,
+ "theoretical_loss": 3.510053505978334,
+ "tokens_seen": 1679491072
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039690270812437313,
+ "loss": 1.1577,
+ "theoretical_loss": 3.5100417860659845,
+ "tokens_seen": 1679556608
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039688665997993983,
+ "loss": 1.3217,
+ "theoretical_loss": 3.510030066738976,
+ "tokens_seen": 1679622144
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039687061183550653,
+ "loss": 1.3729,
+ "theoretical_loss": 3.5100183479972564,
+ "tokens_seen": 1679687680
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039685456369107324,
+ "loss": 1.4063,
+ "theoretical_loss": 3.510006629840773,
+ "tokens_seen": 1679753216
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039683851554663994,
+ "loss": 1.3822,
+ "theoretical_loss": 3.5099949122694745,
+ "tokens_seen": 1679818752
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039682246740220664,
+ "loss": 1.4432,
+ "theoretical_loss": 3.5099831952833087,
+ "tokens_seen": 1679884288
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039680641925777334,
+ "loss": 1.3221,
+ "theoretical_loss": 3.509971478882223,
+ "tokens_seen": 1679949824
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039679037111334004,
+ "loss": 1.27,
+ "theoretical_loss": 3.5099597630661656,
+ "tokens_seen": 1680015360
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039677432296890674,
+ "loss": 1.3166,
+ "theoretical_loss": 3.509948047835085,
+ "tokens_seen": 1680080896
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039675827482447344,
+ "loss": 1.3011,
+ "theoretical_loss": 3.5099363331889286,
+ "tokens_seen": 1680146432
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039674222668004014,
+ "loss": 1.3736,
+ "theoretical_loss": 3.5099246191276445,
+ "tokens_seen": 1680211968
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039672617853560684,
+ "loss": 1.3727,
+ "theoretical_loss": 3.5099129056511806,
+ "tokens_seen": 1680277504
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039671013039117354,
+ "loss": 1.3683,
+ "theoretical_loss": 3.509901192759486,
+ "tokens_seen": 1680343040
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039669408224674024,
+ "loss": 1.3507,
+ "theoretical_loss": 3.5098894804525065,
+ "tokens_seen": 1680408576
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039667803410230694,
+ "loss": 1.3361,
+ "theoretical_loss": 3.509877768730192,
+ "tokens_seen": 1680474112
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039666198595787364,
+ "loss": 1.3515,
+ "theoretical_loss": 3.5098660575924896,
+ "tokens_seen": 1680539648
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039664593781344034,
+ "loss": 1.3914,
+ "theoretical_loss": 3.509854347039348,
+ "tokens_seen": 1680605184
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039662988966900704,
+ "loss": 1.3152,
+ "theoretical_loss": 3.5098426370707143,
+ "tokens_seen": 1680670720
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039661384152457374,
+ "loss": 1.3061,
+ "theoretical_loss": 3.5098309276865374,
+ "tokens_seen": 1680736256
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039659779338014045,
+ "loss": 1.3654,
+ "theoretical_loss": 3.509819218886765,
+ "tokens_seen": 1680801792
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039658174523570715,
+ "loss": 1.2333,
+ "theoretical_loss": 3.509807510671345,
+ "tokens_seen": 1680867328
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039656569709127385,
+ "loss": 1.3129,
+ "theoretical_loss": 3.509795803040226,
+ "tokens_seen": 1680932864
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 563295,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3253703117370605,
+ "objective/train/theoretical_loss": 3.5097840959933553,
+ "objective/train/tokens_used": 1701458400,
+ "theoretical_loss": 3.5097840959933553,
+ "tokens_seen": 1680998400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039654964894684055,
+ "loss": 1.2798,
+ "theoretical_loss": 3.5097840959933553,
+ "tokens_seen": 1680998400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039653360080240725,
+ "loss": 1.3603,
+ "theoretical_loss": 3.5097723895306814,
+ "tokens_seen": 1681063936
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039651755265797395,
+ "loss": 1.3603,
+ "theoretical_loss": 3.5097606836521518,
+ "tokens_seen": 1681129472
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039650150451354065,
+ "loss": 1.2203,
+ "theoretical_loss": 3.509748978357716,
+ "tokens_seen": 1681195008
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039648545636910735,
+ "loss": 1.3285,
+ "theoretical_loss": 3.5097372736473207,
+ "tokens_seen": 1681260544
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039646940822467405,
+ "loss": 1.274,
+ "theoretical_loss": 3.5097255695209144,
+ "tokens_seen": 1681326080
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039645336008024075,
+ "loss": 1.335,
+ "theoretical_loss": 3.509713865978445,
+ "tokens_seen": 1681391616
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039643731193580745,
+ "loss": 1.2972,
+ "theoretical_loss": 3.5097021630198615,
+ "tokens_seen": 1681457152
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039642126379137415,
+ "loss": 1.3608,
+ "theoretical_loss": 3.5096904606451114,
+ "tokens_seen": 1681522688
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039640521564694085,
+ "loss": 1.3304,
+ "theoretical_loss": 3.5096787588541423,
+ "tokens_seen": 1681588224
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039638916750250755,
+ "loss": 1.3501,
+ "theoretical_loss": 3.509667057646903,
+ "tokens_seen": 1681653760
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039637311935807425,
+ "loss": 1.2682,
+ "theoretical_loss": 3.509655357023342,
+ "tokens_seen": 1681719296
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039635707121364095,
+ "loss": 1.4212,
+ "theoretical_loss": 3.509643656983406,
+ "tokens_seen": 1681784832
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039634102306920766,
+ "loss": 1.2548,
+ "theoretical_loss": 3.509631957527045,
+ "tokens_seen": 1681850368
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039632497492477436,
+ "loss": 1.3027,
+ "theoretical_loss": 3.5096202586542056,
+ "tokens_seen": 1681915904
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039630892678034106,
+ "loss": 1.3055,
+ "theoretical_loss": 3.5096085603648364,
+ "tokens_seen": 1681981440
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039629287863590776,
+ "loss": 1.3005,
+ "theoretical_loss": 3.5095968626588867,
+ "tokens_seen": 1682046976
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039627683049147446,
+ "loss": 1.2608,
+ "theoretical_loss": 3.509585165536303,
+ "tokens_seen": 1682112512
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039626078234704116,
+ "loss": 1.3951,
+ "theoretical_loss": 3.509573468997034,
+ "tokens_seen": 1682178048
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039624473420260786,
+ "loss": 1.2975,
+ "theoretical_loss": 3.5095617730410282,
+ "tokens_seen": 1682243584
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039622868605817456,
+ "loss": 1.3445,
+ "theoretical_loss": 3.5095500776682345,
+ "tokens_seen": 1682309120
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039621263791374126,
+ "loss": 1.3088,
+ "theoretical_loss": 3.509538382878599,
+ "tokens_seen": 1682374656
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039619658976930796,
+ "loss": 1.1954,
+ "theoretical_loss": 3.5095266886720724,
+ "tokens_seen": 1682440192
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039618054162487466,
+ "loss": 1.2492,
+ "theoretical_loss": 3.509514995048601,
+ "tokens_seen": 1682505728
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003961644934804413,
+ "loss": 1.4055,
+ "theoretical_loss": 3.5095033020081337,
+ "tokens_seen": 1682571264
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000396148445336008,
+ "loss": 1.4714,
+ "theoretical_loss": 3.5094916095506195,
+ "tokens_seen": 1682636800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003961323971915747,
+ "loss": 1.3832,
+ "theoretical_loss": 3.509479917676005,
+ "tokens_seen": 1682702336
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003961163490471414,
+ "loss": 1.395,
+ "theoretical_loss": 3.5094682263842394,
+ "tokens_seen": 1682767872
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003961003009027081,
+ "loss": 1.2958,
+ "theoretical_loss": 3.5094565356752714,
+ "tokens_seen": 1682833408
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960842527582748,
+ "loss": 1.3726,
+ "theoretical_loss": 3.509444845549049,
+ "tokens_seen": 1682898944
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960682046138415,
+ "loss": 1.3026,
+ "theoretical_loss": 3.509433156005519,
+ "tokens_seen": 1682964480
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960521564694082,
+ "loss": 1.358,
+ "theoretical_loss": 3.5094214670446315,
+ "tokens_seen": 1683030016
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960361083249749,
+ "loss": 1.4805,
+ "theoretical_loss": 3.5094097786663343,
+ "tokens_seen": 1683095552
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960200601805416,
+ "loss": 1.2693,
+ "theoretical_loss": 3.509398090870575,
+ "tokens_seen": 1683161088
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003960040120361083,
+ "loss": 1.3693,
+ "theoretical_loss": 3.5093864036573033,
+ "tokens_seen": 1683226624
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000395987963891675,
+ "loss": 1.2932,
+ "theoretical_loss": 3.509374717026466,
+ "tokens_seen": 1683292160
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003959719157472417,
+ "loss": 1.2604,
+ "theoretical_loss": 3.5093630309780117,
+ "tokens_seen": 1683357696
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003959558676028084,
+ "loss": 1.3242,
+ "theoretical_loss": 3.509351345511889,
+ "tokens_seen": 1683423232
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003959398194583751,
+ "loss": 1.4339,
+ "theoretical_loss": 3.509339660628047,
+ "tokens_seen": 1683488768
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003959237713139418,
+ "loss": 1.5202,
+ "theoretical_loss": 3.509327976326433,
+ "tokens_seen": 1683554304
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003959077231695085,
+ "loss": 1.3499,
+ "theoretical_loss": 3.5093162926069947,
+ "tokens_seen": 1683619840
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003958916750250752,
+ "loss": 1.3759,
+ "theoretical_loss": 3.509304609469682,
+ "tokens_seen": 1683685376
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003958756268806419,
+ "loss": 1.4145,
+ "theoretical_loss": 3.5092929269144424,
+ "tokens_seen": 1683750912
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003958595787362086,
+ "loss": 1.2288,
+ "theoretical_loss": 3.5092812449412243,
+ "tokens_seen": 1683816448
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003958435305917753,
+ "loss": 1.3088,
+ "theoretical_loss": 3.509269563549976,
+ "tokens_seen": 1683881984
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000395827482447342,
+ "loss": 1.3354,
+ "theoretical_loss": 3.5092578827406467,
+ "tokens_seen": 1683947520
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003958114343029087,
+ "loss": 1.4618,
+ "theoretical_loss": 3.509246202513183,
+ "tokens_seen": 1684013056
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003957953861584755,
+ "loss": 1.4285,
+ "theoretical_loss": 3.5092345228675352,
+ "tokens_seen": 1684078592
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003957793380140422,
+ "loss": 1.3331,
+ "theoretical_loss": 3.5092228438036503,
+ "tokens_seen": 1684144128
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003957632898696089,
+ "loss": 1.395,
+ "theoretical_loss": 3.5092111653214775,
+ "tokens_seen": 1684209664
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 564537,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2556380033493042,
+ "objective/train/theoretical_loss": 3.5091994874209647,
+ "objective/train/tokens_used": 1704735200,
+ "theoretical_loss": 3.5091994874209647,
+ "tokens_seen": 1684275200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003957472417251756,
+ "loss": 1.2579,
+ "theoretical_loss": 3.5091994874209647,
+ "tokens_seen": 1684275200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003957311935807423,
+ "loss": 1.328,
+ "theoretical_loss": 3.509187810102061,
+ "tokens_seen": 1684340736
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000395715145436309,
+ "loss": 1.3016,
+ "theoretical_loss": 3.509176133364713,
+ "tokens_seen": 1684406272
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956990972918757,
+ "loss": 1.3205,
+ "theoretical_loss": 3.509164457208872,
+ "tokens_seen": 1684471808
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956830491474424,
+ "loss": 1.3491,
+ "theoretical_loss": 3.5091527816344836,
+ "tokens_seen": 1684537344
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956670010030091,
+ "loss": 1.294,
+ "theoretical_loss": 3.509141106641498,
+ "tokens_seen": 1684602880
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956509528585758,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5091294322298623,
+ "tokens_seen": 1684668416
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956349047141425,
+ "loss": 1.4362,
+ "theoretical_loss": 3.5091177583995266,
+ "tokens_seen": 1684733952
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956188565697092,
+ "loss": 1.3623,
+ "theoretical_loss": 3.509106085150438,
+ "tokens_seen": 1684799488
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003956028084252759,
+ "loss": 1.3761,
+ "theoretical_loss": 3.5090944124825456,
+ "tokens_seen": 1684865024
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003955867602808426,
+ "loss": 1.3485,
+ "theoretical_loss": 3.509082740395798,
+ "tokens_seen": 1684930560
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003955707121364093,
+ "loss": 1.3722,
+ "theoretical_loss": 3.509071068890143,
+ "tokens_seen": 1684996096
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000395554663991976,
+ "loss": 1.2775,
+ "theoretical_loss": 3.5090593979655287,
+ "tokens_seen": 1685061632
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039553861584754263,
+ "loss": 1.3034,
+ "theoretical_loss": 3.509047727621905,
+ "tokens_seen": 1685127168
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039552256770310933,
+ "loss": 1.2756,
+ "theoretical_loss": 3.5090360578592197,
+ "tokens_seen": 1685192704
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039550651955867603,
+ "loss": 1.3566,
+ "theoretical_loss": 3.509024388677421,
+ "tokens_seen": 1685258240
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039549047141424273,
+ "loss": 1.3254,
+ "theoretical_loss": 3.5090127200764583,
+ "tokens_seen": 1685323776
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039547442326980944,
+ "loss": 1.3804,
+ "theoretical_loss": 3.509001052056279,
+ "tokens_seen": 1685389312
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039545837512537614,
+ "loss": 1.2246,
+ "theoretical_loss": 3.5089893846168314,
+ "tokens_seen": 1685454848
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039544232698094284,
+ "loss": 1.2429,
+ "theoretical_loss": 3.508977717758065,
+ "tokens_seen": 1685520384
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039542627883650954,
+ "loss": 1.2984,
+ "theoretical_loss": 3.508966051479929,
+ "tokens_seen": 1685585920
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039541023069207624,
+ "loss": 1.2696,
+ "theoretical_loss": 3.5089543857823697,
+ "tokens_seen": 1685651456
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039539418254764294,
+ "loss": 1.3574,
+ "theoretical_loss": 3.5089427206653374,
+ "tokens_seen": 1685716992
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039537813440320964,
+ "loss": 1.2712,
+ "theoretical_loss": 3.5089310561287803,
+ "tokens_seen": 1685782528
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039536208625877634,
+ "loss": 1.3783,
+ "theoretical_loss": 3.5089193921726465,
+ "tokens_seen": 1685848064
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039534603811434304,
+ "loss": 1.2313,
+ "theoretical_loss": 3.508907728796885,
+ "tokens_seen": 1685913600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039532998996990974,
+ "loss": 1.3936,
+ "theoretical_loss": 3.5088960660014443,
+ "tokens_seen": 1685979136
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039531394182547644,
+ "loss": 1.4377,
+ "theoretical_loss": 3.5088844037862725,
+ "tokens_seen": 1686044672
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039529789368104314,
+ "loss": 1.2427,
+ "theoretical_loss": 3.5088727421513184,
+ "tokens_seen": 1686110208
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039528184553660984,
+ "loss": 1.3432,
+ "theoretical_loss": 3.508861081096531,
+ "tokens_seen": 1686175744
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039526579739217654,
+ "loss": 1.2598,
+ "theoretical_loss": 3.508849420621859,
+ "tokens_seen": 1686241280
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039524974924774324,
+ "loss": 1.337,
+ "theoretical_loss": 3.50883776072725,
+ "tokens_seen": 1686306816
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039523370110330995,
+ "loss": 1.2391,
+ "theoretical_loss": 3.5088261014126534,
+ "tokens_seen": 1686372352
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039521765295887665,
+ "loss": 1.2831,
+ "theoretical_loss": 3.508814442678017,
+ "tokens_seen": 1686437888
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039520160481444335,
+ "loss": 1.2825,
+ "theoretical_loss": 3.508802784523291,
+ "tokens_seen": 1686503424
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039518555667001005,
+ "loss": 1.303,
+ "theoretical_loss": 3.508791126948423,
+ "tokens_seen": 1686568960
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039516950852557675,
+ "loss": 1.4422,
+ "theoretical_loss": 3.5087794699533608,
+ "tokens_seen": 1686634496
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039515346038114345,
+ "loss": 1.3851,
+ "theoretical_loss": 3.5087678135380544,
+ "tokens_seen": 1686700032
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039513741223671015,
+ "loss": 1.2958,
+ "theoretical_loss": 3.508756157702452,
+ "tokens_seen": 1686765568
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039512136409227685,
+ "loss": 1.22,
+ "theoretical_loss": 3.5087445024465023,
+ "tokens_seen": 1686831104
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039510531594784355,
+ "loss": 1.2511,
+ "theoretical_loss": 3.508732847770154,
+ "tokens_seen": 1686896640
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039508926780341025,
+ "loss": 1.2633,
+ "theoretical_loss": 3.508721193673355,
+ "tokens_seen": 1686962176
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039507321965897695,
+ "loss": 1.3662,
+ "theoretical_loss": 3.5087095401560546,
+ "tokens_seen": 1687027712
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039505717151454365,
+ "loss": 1.3562,
+ "theoretical_loss": 3.5086978872182017,
+ "tokens_seen": 1687093248
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039504112337011035,
+ "loss": 1.3871,
+ "theoretical_loss": 3.508686234859745,
+ "tokens_seen": 1687158784
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039502507522567705,
+ "loss": 1.2493,
+ "theoretical_loss": 3.5086745830806327,
+ "tokens_seen": 1687224320
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039500902708124375,
+ "loss": 1.3574,
+ "theoretical_loss": 3.5086629318808136,
+ "tokens_seen": 1687289856
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039499297893681045,
+ "loss": 1.3345,
+ "theoretical_loss": 3.508651281260237,
+ "tokens_seen": 1687355392
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039497693079237716,
+ "loss": 1.439,
+ "theoretical_loss": 3.5086396312188506,
+ "tokens_seen": 1687420928
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039496088264794386,
+ "loss": 1.3993,
+ "theoretical_loss": 3.5086279817566037,
+ "tokens_seen": 1687486464
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 565497,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2637591361999512,
+ "objective/train/theoretical_loss": 3.508616332873445,
+ "objective/train/tokens_used": 1708012000,
+ "theoretical_loss": 3.508616332873445,
+ "tokens_seen": 1687552000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039494483450351056,
+ "loss": 1.2331,
+ "theoretical_loss": 3.508616332873445,
+ "tokens_seen": 1687552000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039492878635907726,
+ "loss": 1.3628,
+ "theoretical_loss": 3.508604684569324,
+ "tokens_seen": 1687617536
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039491273821464396,
+ "loss": 1.2634,
+ "theoretical_loss": 3.5085930368441876,
+ "tokens_seen": 1687683072
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039489669007021066,
+ "loss": 1.3334,
+ "theoretical_loss": 3.508581389697986,
+ "tokens_seen": 1687748608
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039488064192577736,
+ "loss": 1.275,
+ "theoretical_loss": 3.5085697431306673,
+ "tokens_seen": 1687814144
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039486459378134406,
+ "loss": 1.21,
+ "theoretical_loss": 3.5085580971421804,
+ "tokens_seen": 1687879680
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039484854563691076,
+ "loss": 1.2891,
+ "theoretical_loss": 3.5085464517324745,
+ "tokens_seen": 1687945216
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039483249749247746,
+ "loss": 1.392,
+ "theoretical_loss": 3.5085348069014977,
+ "tokens_seen": 1688010752
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039481644934804416,
+ "loss": 1.3625,
+ "theoretical_loss": 3.508523162649199,
+ "tokens_seen": 1688076288
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039480040120361086,
+ "loss": 1.3794,
+ "theoretical_loss": 3.5085115189755274,
+ "tokens_seen": 1688141824
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039478435305917756,
+ "loss": 1.3969,
+ "theoretical_loss": 3.5084998758804313,
+ "tokens_seen": 1688207360
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003947683049147442,
+ "loss": 1.3288,
+ "theoretical_loss": 3.5084882333638596,
+ "tokens_seen": 1688272896
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003947522567703109,
+ "loss": 1.3619,
+ "theoretical_loss": 3.5084765914257616,
+ "tokens_seen": 1688338432
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003947362086258776,
+ "loss": 1.4312,
+ "theoretical_loss": 3.508464950066086,
+ "tokens_seen": 1688403968
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003947201604814443,
+ "loss": 1.2946,
+ "theoretical_loss": 3.5084533092847807,
+ "tokens_seen": 1688469504
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000394704112337011,
+ "loss": 1.3366,
+ "theoretical_loss": 3.5084416690817957,
+ "tokens_seen": 1688535040
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946880641925777,
+ "loss": 1.2482,
+ "theoretical_loss": 3.5084300294570787,
+ "tokens_seen": 1688600576
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946720160481444,
+ "loss": 1.3815,
+ "theoretical_loss": 3.5084183904105792,
+ "tokens_seen": 1688666112
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946559679037111,
+ "loss": 1.4278,
+ "theoretical_loss": 3.508406751942246,
+ "tokens_seen": 1688731648
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946399197592778,
+ "loss": 1.3412,
+ "theoretical_loss": 3.5083951140520284,
+ "tokens_seen": 1688797184
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946238716148445,
+ "loss": 1.4112,
+ "theoretical_loss": 3.508383476739874,
+ "tokens_seen": 1688862720
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003946078234704112,
+ "loss": 1.2653,
+ "theoretical_loss": 3.508371840005733,
+ "tokens_seen": 1688928256
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039459177532597797,
+ "loss": 1.3189,
+ "theoretical_loss": 3.508360203849553,
+ "tokens_seen": 1688993792
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039457572718154467,
+ "loss": 1.331,
+ "theoretical_loss": 3.508348568271284,
+ "tokens_seen": 1689059328
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039455967903711137,
+ "loss": 1.2246,
+ "theoretical_loss": 3.5083369332708747,
+ "tokens_seen": 1689124864
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039454363089267807,
+ "loss": 1.3353,
+ "theoretical_loss": 3.508325298848273,
+ "tokens_seen": 1689190400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003945275827482448,
+ "loss": 1.3473,
+ "theoretical_loss": 3.5083136650034286,
+ "tokens_seen": 1689255936
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003945115346038115,
+ "loss": 1.2427,
+ "theoretical_loss": 3.5083020317362905,
+ "tokens_seen": 1689321472
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003944954864593782,
+ "loss": 1.3452,
+ "theoretical_loss": 3.5082903990468073,
+ "tokens_seen": 1689387008
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003944794383149449,
+ "loss": 1.3949,
+ "theoretical_loss": 3.508278766934928,
+ "tokens_seen": 1689452544
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003944633901705116,
+ "loss": 1.2256,
+ "theoretical_loss": 3.5082671354006014,
+ "tokens_seen": 1689518080
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003944473420260783,
+ "loss": 1.3816,
+ "theoretical_loss": 3.5082555044437767,
+ "tokens_seen": 1689583616
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000394431293881645,
+ "loss": 1.2993,
+ "theoretical_loss": 3.5082438740644024,
+ "tokens_seen": 1689649152
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003944152457372117,
+ "loss": 1.3085,
+ "theoretical_loss": 3.5082322442624276,
+ "tokens_seen": 1689714688
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943991975927784,
+ "loss": 1.3416,
+ "theoretical_loss": 3.5082206150378017,
+ "tokens_seen": 1689780224
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943831494483451,
+ "loss": 1.2564,
+ "theoretical_loss": 3.508208986390473,
+ "tokens_seen": 1689845760
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943671013039118,
+ "loss": 1.2665,
+ "theoretical_loss": 3.508197358320391,
+ "tokens_seen": 1689911296
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943510531594785,
+ "loss": 1.3752,
+ "theoretical_loss": 3.5081857308275044,
+ "tokens_seen": 1689976832
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943350050150452,
+ "loss": 1.3116,
+ "theoretical_loss": 3.5081741039117618,
+ "tokens_seen": 1690042368
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943189568706119,
+ "loss": 1.324,
+ "theoretical_loss": 3.5081624775731126,
+ "tokens_seen": 1690107904
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003943029087261786,
+ "loss": 1.2645,
+ "theoretical_loss": 3.508150851811506,
+ "tokens_seen": 1690173440
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003942868605817453,
+ "loss": 1.3925,
+ "theoretical_loss": 3.5081392266268905,
+ "tokens_seen": 1690238976
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000394270812437312,
+ "loss": 1.2388,
+ "theoretical_loss": 3.508127602019215,
+ "tokens_seen": 1690304512
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003942547642928787,
+ "loss": 1.3048,
+ "theoretical_loss": 3.508115977988429,
+ "tokens_seen": 1690370048
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003942387161484454,
+ "loss": 1.2891,
+ "theoretical_loss": 3.5081043545344817,
+ "tokens_seen": 1690435584
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003942226680040121,
+ "loss": 1.1727,
+ "theoretical_loss": 3.5080927316573214,
+ "tokens_seen": 1690501120
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003942066198595788,
+ "loss": 1.3343,
+ "theoretical_loss": 3.5080811093568975,
+ "tokens_seen": 1690566656
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003941905717151455,
+ "loss": 1.3054,
+ "theoretical_loss": 3.508069487633159,
+ "tokens_seen": 1690632192
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003941745235707122,
+ "loss": 1.2212,
+ "theoretical_loss": 3.5080578664860544,
+ "tokens_seen": 1690697728
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003941584754262789,
+ "loss": 1.3057,
+ "theoretical_loss": 3.508046245915534,
+ "tokens_seen": 1690763264
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 566850,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2949119806289673,
+ "objective/train/theoretical_loss": 3.5080346259215456,
+ "objective/train/tokens_used": 1711288800,
+ "theoretical_loss": 3.5080346259215456,
+ "tokens_seen": 1690828800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039414242728184553,
+ "loss": 1.263,
+ "theoretical_loss": 3.5080346259215456,
+ "tokens_seen": 1690828800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039412637913741223,
+ "loss": 1.2198,
+ "theoretical_loss": 3.5080230065040388,
+ "tokens_seen": 1690894336
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039411033099297894,
+ "loss": 1.3229,
+ "theoretical_loss": 3.508011387662963,
+ "tokens_seen": 1690959872
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039409428284854564,
+ "loss": 1.301,
+ "theoretical_loss": 3.5079997693982663,
+ "tokens_seen": 1691025408
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039407823470411234,
+ "loss": 1.2693,
+ "theoretical_loss": 3.5079881517098985,
+ "tokens_seen": 1691090944
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039406218655967904,
+ "loss": 1.2663,
+ "theoretical_loss": 3.507976534597808,
+ "tokens_seen": 1691156480
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039404613841524574,
+ "loss": 1.2388,
+ "theoretical_loss": 3.507964918061945,
+ "tokens_seen": 1691222016
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039403009027081244,
+ "loss": 1.3611,
+ "theoretical_loss": 3.507953302102258,
+ "tokens_seen": 1691287552
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039401404212637914,
+ "loss": 1.3171,
+ "theoretical_loss": 3.507941686718696,
+ "tokens_seen": 1691353088
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039399799398194584,
+ "loss": 1.2994,
+ "theoretical_loss": 3.507930071911208,
+ "tokens_seen": 1691418624
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039398194583751254,
+ "loss": 1.28,
+ "theoretical_loss": 3.5079184576797435,
+ "tokens_seen": 1691484160
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039396589769307924,
+ "loss": 1.3806,
+ "theoretical_loss": 3.507906844024251,
+ "tokens_seen": 1691549696
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039394984954864594,
+ "loss": 1.2292,
+ "theoretical_loss": 3.5078952309446807,
+ "tokens_seen": 1691615232
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039393380140421264,
+ "loss": 1.3192,
+ "theoretical_loss": 3.5078836184409803,
+ "tokens_seen": 1691680768
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039391775325977934,
+ "loss": 1.3907,
+ "theoretical_loss": 3.5078720065131,
+ "tokens_seen": 1691746304
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039390170511534604,
+ "loss": 1.3506,
+ "theoretical_loss": 3.507860395160989,
+ "tokens_seen": 1691811840
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039388565697091274,
+ "loss": 1.2145,
+ "theoretical_loss": 3.5078487843845956,
+ "tokens_seen": 1691877376
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039386960882647944,
+ "loss": 1.316,
+ "theoretical_loss": 3.5078371741838694,
+ "tokens_seen": 1691942912
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039385356068204615,
+ "loss": 1.417,
+ "theoretical_loss": 3.5078255645587597,
+ "tokens_seen": 1692008448
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039383751253761285,
+ "loss": 1.2959,
+ "theoretical_loss": 3.507813955509216,
+ "tokens_seen": 1692073984
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039382146439317955,
+ "loss": 1.2502,
+ "theoretical_loss": 3.5078023470351862,
+ "tokens_seen": 1692139520
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039380541624874625,
+ "loss": 1.3025,
+ "theoretical_loss": 3.507790739136621,
+ "tokens_seen": 1692205056
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039378936810431295,
+ "loss": 1.3568,
+ "theoretical_loss": 3.507779131813469,
+ "tokens_seen": 1692270592
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039377331995987965,
+ "loss": 1.3236,
+ "theoretical_loss": 3.507767525065679,
+ "tokens_seen": 1692336128
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039375727181544635,
+ "loss": 1.3854,
+ "theoretical_loss": 3.5077559188932,
+ "tokens_seen": 1692401664
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039374122367101305,
+ "loss": 1.3598,
+ "theoretical_loss": 3.5077443132959827,
+ "tokens_seen": 1692467200
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039372517552657975,
+ "loss": 1.2831,
+ "theoretical_loss": 3.5077327082739744,
+ "tokens_seen": 1692532736
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039370912738214645,
+ "loss": 1.3579,
+ "theoretical_loss": 3.5077211038271257,
+ "tokens_seen": 1692598272
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039369307923771315,
+ "loss": 1.3286,
+ "theoretical_loss": 3.5077094999553857,
+ "tokens_seen": 1692663808
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039367703109327985,
+ "loss": 1.4154,
+ "theoretical_loss": 3.507697896658703,
+ "tokens_seen": 1692729344
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039366098294884655,
+ "loss": 1.2917,
+ "theoretical_loss": 3.5076862939370272,
+ "tokens_seen": 1692794880
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039364493480441325,
+ "loss": 1.3786,
+ "theoretical_loss": 3.5076746917903074,
+ "tokens_seen": 1692860416
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039362888665997995,
+ "loss": 1.3908,
+ "theoretical_loss": 3.507663090218493,
+ "tokens_seen": 1692925952
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039361283851554665,
+ "loss": 1.3052,
+ "theoretical_loss": 3.5076514892215327,
+ "tokens_seen": 1692991488
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039359679037111336,
+ "loss": 1.3047,
+ "theoretical_loss": 3.507639888799377,
+ "tokens_seen": 1693057024
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039358074222668006,
+ "loss": 1.2796,
+ "theoretical_loss": 3.507628288951974,
+ "tokens_seen": 1693122560
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039356469408224676,
+ "loss": 1.2555,
+ "theoretical_loss": 3.5076166896792733,
+ "tokens_seen": 1693188096
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039354864593781346,
+ "loss": 1.3208,
+ "theoretical_loss": 3.507605090981225,
+ "tokens_seen": 1693253632
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039353259779338016,
+ "loss": 1.3149,
+ "theoretical_loss": 3.507593492857777,
+ "tokens_seen": 1693319168
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039351654964894686,
+ "loss": 1.2571,
+ "theoretical_loss": 3.5075818953088795,
+ "tokens_seen": 1693384704
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039350050150451356,
+ "loss": 1.2692,
+ "theoretical_loss": 3.507570298334482,
+ "tokens_seen": 1693450240
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039348445336008026,
+ "loss": 1.3489,
+ "theoretical_loss": 3.507558701934533,
+ "tokens_seen": 1693515776
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039346840521564696,
+ "loss": 1.3062,
+ "theoretical_loss": 3.507547106108982,
+ "tokens_seen": 1693581312
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039345235707121366,
+ "loss": 1.184,
+ "theoretical_loss": 3.507535510857779,
+ "tokens_seen": 1693646848
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039343630892678036,
+ "loss": 1.2422,
+ "theoretical_loss": 3.5075239161808724,
+ "tokens_seen": 1693712384
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039342026078234706,
+ "loss": 1.342,
+ "theoretical_loss": 3.5075123220782123,
+ "tokens_seen": 1693777920
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039340421263791376,
+ "loss": 1.3779,
+ "theoretical_loss": 3.507500728549748,
+ "tokens_seen": 1693843456
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039338816449348046,
+ "loss": 1.2936,
+ "theoretical_loss": 3.507489135595428,
+ "tokens_seen": 1693908992
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039337211634904716,
+ "loss": 1.2804,
+ "theoretical_loss": 3.5074775432152023,
+ "tokens_seen": 1693974528
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039335606820461387,
+ "loss": 1.3327,
+ "theoretical_loss": 3.5074659514090207,
+ "tokens_seen": 1694040064
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 567680,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2184815406799316,
+ "objective/train/theoretical_loss": 3.507454360176832,
+ "objective/train/tokens_used": 1714565600,
+ "theoretical_loss": 3.507454360176832,
+ "tokens_seen": 1694105600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039334002006018057,
+ "loss": 1.3059,
+ "theoretical_loss": 3.507454360176832,
+ "tokens_seen": 1694105600
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039332397191574727,
+ "loss": 1.4208,
+ "theoretical_loss": 3.5074427695185855,
+ "tokens_seen": 1694171136
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039330792377131397,
+ "loss": 1.3137,
+ "theoretical_loss": 3.5074311794342306,
+ "tokens_seen": 1694236672
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039329187562688067,
+ "loss": 1.103,
+ "theoretical_loss": 3.507419589923717,
+ "tokens_seen": 1694302208
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039327582748244737,
+ "loss": 1.2381,
+ "theoretical_loss": 3.507408000986994,
+ "tokens_seen": 1694367744
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039325977933801407,
+ "loss": 1.3529,
+ "theoretical_loss": 3.5073964126240105,
+ "tokens_seen": 1694433280
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039324373119358077,
+ "loss": 1.2204,
+ "theoretical_loss": 3.507384824834717,
+ "tokens_seen": 1694498816
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039322768304914747,
+ "loss": 1.2338,
+ "theoretical_loss": 3.5073732376190616,
+ "tokens_seen": 1694564352
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039321163490471417,
+ "loss": 1.3662,
+ "theoretical_loss": 3.5073616509769945,
+ "tokens_seen": 1694629888
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039319558676028087,
+ "loss": 1.3137,
+ "theoretical_loss": 3.507350064908465,
+ "tokens_seen": 1694695424
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039317953861584757,
+ "loss": 1.3784,
+ "theoretical_loss": 3.5073384794134226,
+ "tokens_seen": 1694760960
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039316349047141427,
+ "loss": 1.2652,
+ "theoretical_loss": 3.507326894491817,
+ "tokens_seen": 1694826496
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000393147442326981,
+ "loss": 1.2707,
+ "theoretical_loss": 3.507315310143597,
+ "tokens_seen": 1694892032
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003931313941825477,
+ "loss": 1.3725,
+ "theoretical_loss": 3.507303726368712,
+ "tokens_seen": 1694957568
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003931153460381144,
+ "loss": 1.234,
+ "theoretical_loss": 3.5072921431671125,
+ "tokens_seen": 1695023104
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930992978936811,
+ "loss": 1.2371,
+ "theoretical_loss": 3.5072805605387467,
+ "tokens_seen": 1695088640
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930832497492478,
+ "loss": 1.3322,
+ "theoretical_loss": 3.507268978483565,
+ "tokens_seen": 1695154176
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930672016048145,
+ "loss": 1.2113,
+ "theoretical_loss": 3.507257397001516,
+ "tokens_seen": 1695219712
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930511534603812,
+ "loss": 1.2818,
+ "theoretical_loss": 3.50724581609255,
+ "tokens_seen": 1695285248
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930351053159479,
+ "loss": 1.2946,
+ "theoretical_loss": 3.5072342357566164,
+ "tokens_seen": 1695350784
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930190571715146,
+ "loss": 1.2973,
+ "theoretical_loss": 3.5072226559936643,
+ "tokens_seen": 1695416320
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003930030090270813,
+ "loss": 1.3219,
+ "theoretical_loss": 3.5072110768036433,
+ "tokens_seen": 1695481856
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000392986960882648,
+ "loss": 1.3231,
+ "theoretical_loss": 3.5071994981865036,
+ "tokens_seen": 1695547392
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003929709127382147,
+ "loss": 1.2811,
+ "theoretical_loss": 3.507187920142193,
+ "tokens_seen": 1695612928
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003929548645937814,
+ "loss": 1.2201,
+ "theoretical_loss": 3.507176342670663,
+ "tokens_seen": 1695678464
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003929388164493481,
+ "loss": 1.2676,
+ "theoretical_loss": 3.5071647657718623,
+ "tokens_seen": 1695744000
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003929227683049148,
+ "loss": 1.2043,
+ "theoretical_loss": 3.5071531894457397,
+ "tokens_seen": 1695809536
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003929067201604815,
+ "loss": 1.2442,
+ "theoretical_loss": 3.507141613692246,
+ "tokens_seen": 1695875072
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003928906720160482,
+ "loss": 1.4047,
+ "theoretical_loss": 3.50713003851133,
+ "tokens_seen": 1695940608
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003928746238716149,
+ "loss": 1.3448,
+ "theoretical_loss": 3.507118463902942,
+ "tokens_seen": 1696006144
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003928585757271816,
+ "loss": 1.3224,
+ "theoretical_loss": 3.50710688986703,
+ "tokens_seen": 1696071680
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003928425275827483,
+ "loss": 1.3006,
+ "theoretical_loss": 3.507095316403545,
+ "tokens_seen": 1696137216
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000392826479438315,
+ "loss": 1.4223,
+ "theoretical_loss": 3.5070837435124362,
+ "tokens_seen": 1696202752
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003928104312938817,
+ "loss": 1.3044,
+ "theoretical_loss": 3.5070721711936534,
+ "tokens_seen": 1696268288
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003927943831494484,
+ "loss": 1.2839,
+ "theoretical_loss": 3.5070605994471453,
+ "tokens_seen": 1696333824
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003927783350050151,
+ "loss": 1.2511,
+ "theoretical_loss": 3.5070490282728626,
+ "tokens_seen": 1696399360
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003927622868605818,
+ "loss": 1.2513,
+ "theoretical_loss": 3.507037457670754,
+ "tokens_seen": 1696464896
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003927462387161485,
+ "loss": 1.3928,
+ "theoretical_loss": 3.5070258876407703,
+ "tokens_seen": 1696530432
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039273019057171514,
+ "loss": 1.2776,
+ "theoretical_loss": 3.5070143181828595,
+ "tokens_seen": 1696595968
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039271414242728184,
+ "loss": 1.3406,
+ "theoretical_loss": 3.507002749296973,
+ "tokens_seen": 1696661504
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039269809428284854,
+ "loss": 1.3934,
+ "theoretical_loss": 3.506991180983058,
+ "tokens_seen": 1696727040
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039268204613841524,
+ "loss": 1.3359,
+ "theoretical_loss": 3.5069796132410667,
+ "tokens_seen": 1696792576
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039266599799398194,
+ "loss": 1.3247,
+ "theoretical_loss": 3.506968046070947,
+ "tokens_seen": 1696858112
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039264994984954864,
+ "loss": 1.2309,
+ "theoretical_loss": 3.5069564794726498,
+ "tokens_seen": 1696923648
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039263390170511534,
+ "loss": 1.416,
+ "theoretical_loss": 3.5069449134461235,
+ "tokens_seen": 1696989184
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039261785356068204,
+ "loss": 1.285,
+ "theoretical_loss": 3.5069333479913185,
+ "tokens_seen": 1697054720
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039260180541624874,
+ "loss": 1.2694,
+ "theoretical_loss": 3.5069217831081847,
+ "tokens_seen": 1697120256
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039258575727181544,
+ "loss": 1.39,
+ "theoretical_loss": 3.506910218796671,
+ "tokens_seen": 1697185792
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039256970912738214,
+ "loss": 1.2365,
+ "theoretical_loss": 3.5068986550567276,
+ "tokens_seen": 1697251328
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039255366098294884,
+ "loss": 1.2729,
+ "theoretical_loss": 3.506887091888304,
+ "tokens_seen": 1697316864
+ },
+ {
+ "epoch": 0.51,
+ "objective/train/docs_used": 569072,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3679840564727783,
+ "objective/train/theoretical_loss": 3.5068755292913503,
+ "objective/train/tokens_used": 1717842400,
+ "theoretical_loss": 3.5068755292913503,
+ "tokens_seen": 1697382400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039253761283851554,
+ "loss": 1.2892,
+ "theoretical_loss": 3.5068755292913503,
+ "tokens_seen": 1697382400
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039252156469408224,
+ "loss": 1.3705,
+ "theoretical_loss": 3.5068639672658155,
+ "tokens_seen": 1697447936
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039250551654964894,
+ "loss": 1.2891,
+ "theoretical_loss": 3.50685240581165,
+ "tokens_seen": 1697513472
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039248946840521565,
+ "loss": 1.1874,
+ "theoretical_loss": 3.506840844928803,
+ "tokens_seen": 1697579008
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039247342026078235,
+ "loss": 1.366,
+ "theoretical_loss": 3.5068292846172238,
+ "tokens_seen": 1697644544
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039245737211634905,
+ "loss": 1.4041,
+ "theoretical_loss": 3.5068177248768633,
+ "tokens_seen": 1697710080
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039244132397191575,
+ "loss": 1.3315,
+ "theoretical_loss": 3.506806165707671,
+ "tokens_seen": 1697775616
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039242527582748245,
+ "loss": 1.2505,
+ "theoretical_loss": 3.506794607109595,
+ "tokens_seen": 1697841152
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039240922768304915,
+ "loss": 1.2652,
+ "theoretical_loss": 3.5067830490825873,
+ "tokens_seen": 1697906688
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039239317953861585,
+ "loss": 1.2623,
+ "theoretical_loss": 3.506771491626596,
+ "tokens_seen": 1697972224
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039237713139418255,
+ "loss": 1.2927,
+ "theoretical_loss": 3.506759934741572,
+ "tokens_seen": 1698037760
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039236108324974925,
+ "loss": 1.2347,
+ "theoretical_loss": 3.5067483784274645,
+ "tokens_seen": 1698103296
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039234503510531595,
+ "loss": 1.2806,
+ "theoretical_loss": 3.506736822684223,
+ "tokens_seen": 1698168832
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039232898696088265,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5067252675117975,
+ "tokens_seen": 1698234368
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039231293881644935,
+ "loss": 1.3127,
+ "theoretical_loss": 3.506713712910138,
+ "tokens_seen": 1698299904
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039229689067201605,
+ "loss": 1.3687,
+ "theoretical_loss": 3.506702158879194,
+ "tokens_seen": 1698365440
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039228084252758275,
+ "loss": 1.294,
+ "theoretical_loss": 3.5066906054189158,
+ "tokens_seen": 1698430976
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039226479438314945,
+ "loss": 1.3141,
+ "theoretical_loss": 3.5066790525292526,
+ "tokens_seen": 1698496512
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039224874623871615,
+ "loss": 1.2798,
+ "theoretical_loss": 3.5066675002101544,
+ "tokens_seen": 1698562048
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039223269809428286,
+ "loss": 1.2686,
+ "theoretical_loss": 3.506655948461571,
+ "tokens_seen": 1698627584
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039221664994984956,
+ "loss": 1.2018,
+ "theoretical_loss": 3.506644397283452,
+ "tokens_seen": 1698693120
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039220060180541626,
+ "loss": 1.2765,
+ "theoretical_loss": 3.506632846675748,
+ "tokens_seen": 1698758656
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.000392184553660983,
+ "loss": 1.2743,
+ "theoretical_loss": 3.506621296638408,
+ "tokens_seen": 1698824192
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003921685055165497,
+ "loss": 1.2716,
+ "theoretical_loss": 3.5066097471713826,
+ "tokens_seen": 1698889728
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003921524573721164,
+ "loss": 1.3958,
+ "theoretical_loss": 3.50659819827462,
+ "tokens_seen": 1698955264
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003921364092276831,
+ "loss": 1.3092,
+ "theoretical_loss": 3.5065866499480727,
+ "tokens_seen": 1699020800
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.0003921203610832498,
+ "loss": 1.2211,
+ "theoretical_loss": 3.5065751021916878,
+ "tokens_seen": 1699086336
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039210431293881646,
+ "loss": 1.3246,
+ "theoretical_loss": 3.5065635550054175,
+ "tokens_seen": 1699151872
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039208826479438316,
+ "loss": 1.2878,
+ "theoretical_loss": 3.50655200838921,
+ "tokens_seen": 1699217408
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039207221664994986,
+ "loss": 1.305,
+ "theoretical_loss": 3.5065404623430156,
+ "tokens_seen": 1699282944
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039205616850551656,
+ "loss": 1.3833,
+ "theoretical_loss": 3.5065289168667846,
+ "tokens_seen": 1699348480
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039204012036108326,
+ "loss": 1.2665,
+ "theoretical_loss": 3.5065173719604665,
+ "tokens_seen": 1699414016
+ },
+ {
+ "epoch": 0.51,
+ "learning_rate": 0.00039202407221664996,
+ "loss": 1.2285,
+ "theoretical_loss": 3.5065058276240113,
+ "tokens_seen": 1699479552
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039200802407221666,
+ "loss": 1.3226,
+ "theoretical_loss": 3.506494283857369,
+ "tokens_seen": 1699545088
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039199197592778336,
+ "loss": 1.2431,
+ "theoretical_loss": 3.5064827406604895,
+ "tokens_seen": 1699610624
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039197592778335007,
+ "loss": 1.1,
+ "theoretical_loss": 3.5064711980333225,
+ "tokens_seen": 1699676160
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039195987963891677,
+ "loss": 1.2415,
+ "theoretical_loss": 3.506459655975818,
+ "tokens_seen": 1699741696
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039194383149448347,
+ "loss": 1.2683,
+ "theoretical_loss": 3.5064481144879265,
+ "tokens_seen": 1699807232
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039192778335005017,
+ "loss": 1.3517,
+ "theoretical_loss": 3.506436573569597,
+ "tokens_seen": 1699872768
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039191173520561687,
+ "loss": 1.2321,
+ "theoretical_loss": 3.5064250332207796,
+ "tokens_seen": 1699938304
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039189568706118357,
+ "loss": 1.3446,
+ "theoretical_loss": 3.506413493441425,
+ "tokens_seen": 1700003840
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039187963891675027,
+ "loss": 1.3141,
+ "theoretical_loss": 3.506401954231482,
+ "tokens_seen": 1700069376
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039186359077231697,
+ "loss": 1.3589,
+ "theoretical_loss": 3.506390415590902,
+ "tokens_seen": 1700134912
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039184754262788367,
+ "loss": 1.2161,
+ "theoretical_loss": 3.5063788775196336,
+ "tokens_seen": 1700200448
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039183149448345037,
+ "loss": 1.2506,
+ "theoretical_loss": 3.506367340017627,
+ "tokens_seen": 1700265984
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039181544633901707,
+ "loss": 1.3,
+ "theoretical_loss": 3.5063558030848325,
+ "tokens_seen": 1700331520
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039179939819458377,
+ "loss": 1.2518,
+ "theoretical_loss": 3.506344266721201,
+ "tokens_seen": 1700397056
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003917833500501505,
+ "loss": 1.342,
+ "theoretical_loss": 3.5063327309266805,
+ "tokens_seen": 1700462592
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003917673019057172,
+ "loss": 1.3103,
+ "theoretical_loss": 3.5063211957012226,
+ "tokens_seen": 1700528128
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003917512537612839,
+ "loss": 1.3478,
+ "theoretical_loss": 3.506309661044777,
+ "tokens_seen": 1700593664
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 569913,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1845080852508545,
+ "objective/train/theoretical_loss": 3.506298126957293,
+ "objective/train/tokens_used": 1721119200,
+ "theoretical_loss": 3.506298126957293,
+ "tokens_seen": 1700659200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003917352056168506,
+ "loss": 1.2229,
+ "theoretical_loss": 3.506298126957293,
+ "tokens_seen": 1700659200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003917191574724173,
+ "loss": 1.3233,
+ "theoretical_loss": 3.506286593438721,
+ "tokens_seen": 1700724736
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000391703109327984,
+ "loss": 1.3259,
+ "theoretical_loss": 3.5062750604890116,
+ "tokens_seen": 1700790272
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916870611835507,
+ "loss": 1.4017,
+ "theoretical_loss": 3.5062635281081134,
+ "tokens_seen": 1700855808
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916710130391174,
+ "loss": 1.4651,
+ "theoretical_loss": 3.506251996295978,
+ "tokens_seen": 1700921344
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916549648946841,
+ "loss": 1.3008,
+ "theoretical_loss": 3.5062404650525547,
+ "tokens_seen": 1700986880
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916389167502508,
+ "loss": 1.3168,
+ "theoretical_loss": 3.5062289343777935,
+ "tokens_seen": 1701052416
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916228686058175,
+ "loss": 1.2484,
+ "theoretical_loss": 3.5062174042716445,
+ "tokens_seen": 1701117952
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003916068204613842,
+ "loss": 1.3394,
+ "theoretical_loss": 3.506205874734058,
+ "tokens_seen": 1701183488
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003915907723169509,
+ "loss": 1.3757,
+ "theoretical_loss": 3.5061943457649836,
+ "tokens_seen": 1701249024
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003915747241725176,
+ "loss": 1.4251,
+ "theoretical_loss": 3.5061828173643717,
+ "tokens_seen": 1701314560
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003915586760280843,
+ "loss": 1.2473,
+ "theoretical_loss": 3.5061712895321726,
+ "tokens_seen": 1701380096
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000391542627883651,
+ "loss": 1.2597,
+ "theoretical_loss": 3.506159762268336,
+ "tokens_seen": 1701445632
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003915265797392177,
+ "loss": 1.2608,
+ "theoretical_loss": 3.506148235572812,
+ "tokens_seen": 1701511168
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003915105315947844,
+ "loss": 1.1998,
+ "theoretical_loss": 3.506136709445551,
+ "tokens_seen": 1701576704
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914944834503511,
+ "loss": 1.3601,
+ "theoretical_loss": 3.5061251838865024,
+ "tokens_seen": 1701642240
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914784353059178,
+ "loss": 1.3088,
+ "theoretical_loss": 3.506113658895617,
+ "tokens_seen": 1701707776
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914623871614845,
+ "loss": 1.2491,
+ "theoretical_loss": 3.506102134472845,
+ "tokens_seen": 1701773312
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914463390170512,
+ "loss": 1.2246,
+ "theoretical_loss": 3.506090610618136,
+ "tokens_seen": 1701838848
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914302908726179,
+ "loss": 1.2195,
+ "theoretical_loss": 3.5060790873314405,
+ "tokens_seen": 1701904384
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003914142427281846,
+ "loss": 1.2233,
+ "theoretical_loss": 3.5060675646127084,
+ "tokens_seen": 1701969920
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003913981945837513,
+ "loss": 1.2162,
+ "theoretical_loss": 3.506056042461889,
+ "tokens_seen": 1702035456
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000391382146439318,
+ "loss": 1.2986,
+ "theoretical_loss": 3.506044520878935,
+ "tokens_seen": 1702100992
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003913660982948847,
+ "loss": 1.3525,
+ "theoretical_loss": 3.506032999863794,
+ "tokens_seen": 1702166528
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003913500501504514,
+ "loss": 1.2059,
+ "theoretical_loss": 3.506021479416417,
+ "tokens_seen": 1702232064
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039133400200601804,
+ "loss": 1.3543,
+ "theoretical_loss": 3.506009959536754,
+ "tokens_seen": 1702297600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039131795386158474,
+ "loss": 1.2637,
+ "theoretical_loss": 3.505998440224756,
+ "tokens_seen": 1702363136
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039130190571715144,
+ "loss": 1.3314,
+ "theoretical_loss": 3.5059869214803725,
+ "tokens_seen": 1702428672
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039128585757271814,
+ "loss": 1.2534,
+ "theoretical_loss": 3.5059754033035535,
+ "tokens_seen": 1702494208
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039126980942828484,
+ "loss": 1.2815,
+ "theoretical_loss": 3.50596388569425,
+ "tokens_seen": 1702559744
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039125376128385154,
+ "loss": 1.2362,
+ "theoretical_loss": 3.5059523686524106,
+ "tokens_seen": 1702625280
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039123771313941824,
+ "loss": 1.3542,
+ "theoretical_loss": 3.5059408521779876,
+ "tokens_seen": 1702690816
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039122166499498494,
+ "loss": 1.2336,
+ "theoretical_loss": 3.5059293362709294,
+ "tokens_seen": 1702756352
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039120561685055164,
+ "loss": 1.4013,
+ "theoretical_loss": 3.505917820931187,
+ "tokens_seen": 1702821888
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039118956870611834,
+ "loss": 1.2276,
+ "theoretical_loss": 3.5059063061587112,
+ "tokens_seen": 1702887424
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039117352056168504,
+ "loss": 1.1717,
+ "theoretical_loss": 3.505894791953451,
+ "tokens_seen": 1702952960
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039115747241725174,
+ "loss": 1.2855,
+ "theoretical_loss": 3.5058832783153573,
+ "tokens_seen": 1703018496
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039114142427281844,
+ "loss": 1.3067,
+ "theoretical_loss": 3.5058717652443803,
+ "tokens_seen": 1703084032
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039112537612838514,
+ "loss": 1.2898,
+ "theoretical_loss": 3.5058602527404705,
+ "tokens_seen": 1703149568
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039110932798395185,
+ "loss": 1.2778,
+ "theoretical_loss": 3.5058487408035774,
+ "tokens_seen": 1703215104
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039109327983951855,
+ "loss": 1.201,
+ "theoretical_loss": 3.505837229433652,
+ "tokens_seen": 1703280640
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039107723169508525,
+ "loss": 1.2255,
+ "theoretical_loss": 3.5058257186306445,
+ "tokens_seen": 1703346176
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039106118355065195,
+ "loss": 1.3764,
+ "theoretical_loss": 3.5058142083945043,
+ "tokens_seen": 1703411712
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039104513540621865,
+ "loss": 1.3237,
+ "theoretical_loss": 3.505802698725183,
+ "tokens_seen": 1703477248
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039102908726178535,
+ "loss": 1.3422,
+ "theoretical_loss": 3.50579118962263,
+ "tokens_seen": 1703542784
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039101303911735205,
+ "loss": 1.2957,
+ "theoretical_loss": 3.505779681086796,
+ "tokens_seen": 1703608320
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039099699097291875,
+ "loss": 1.221,
+ "theoretical_loss": 3.5057681731176307,
+ "tokens_seen": 1703673856
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003909809428284855,
+ "loss": 1.2782,
+ "theoretical_loss": 3.5057566657150847,
+ "tokens_seen": 1703739392
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003909648946840522,
+ "loss": 1.2679,
+ "theoretical_loss": 3.505745158879109,
+ "tokens_seen": 1703804928
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003909488465396189,
+ "loss": 1.3089,
+ "theoretical_loss": 3.5057336526096528,
+ "tokens_seen": 1703870464
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 571215,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2172174453735352,
+ "objective/train/theoretical_loss": 3.5057221469066673,
+ "objective/train/tokens_used": 1724396000,
+ "theoretical_loss": 3.5057221469066673,
+ "tokens_seen": 1703936000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003909327983951856,
+ "loss": 1.1634,
+ "theoretical_loss": 3.5057221469066673,
+ "tokens_seen": 1703936000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003909167502507523,
+ "loss": 1.2943,
+ "theoretical_loss": 3.5057106417701025,
+ "tokens_seen": 1704001536
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000390900702106319,
+ "loss": 1.3314,
+ "theoretical_loss": 3.505699137199908,
+ "tokens_seen": 1704067072
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908846539618857,
+ "loss": 1.272,
+ "theoretical_loss": 3.5056876331960356,
+ "tokens_seen": 1704132608
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908686058174524,
+ "loss": 1.3367,
+ "theoretical_loss": 3.5056761297584345,
+ "tokens_seen": 1704198144
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908525576730191,
+ "loss": 1.349,
+ "theoretical_loss": 3.5056646268870555,
+ "tokens_seen": 1704263680
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908365095285858,
+ "loss": 1.1898,
+ "theoretical_loss": 3.5056531245818494,
+ "tokens_seen": 1704329216
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908204613841525,
+ "loss": 1.3588,
+ "theoretical_loss": 3.5056416228427656,
+ "tokens_seen": 1704394752
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003908044132397192,
+ "loss": 1.4023,
+ "theoretical_loss": 3.5056301216697547,
+ "tokens_seen": 1704460288
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003907883650952859,
+ "loss": 1.275,
+ "theoretical_loss": 3.5056186210627676,
+ "tokens_seen": 1704525824
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003907723169508526,
+ "loss": 1.2603,
+ "theoretical_loss": 3.5056071210217543,
+ "tokens_seen": 1704591360
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003907562688064193,
+ "loss": 1.3336,
+ "theoretical_loss": 3.5055956215466653,
+ "tokens_seen": 1704656896
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000390740220661986,
+ "loss": 1.2997,
+ "theoretical_loss": 3.505584122637451,
+ "tokens_seen": 1704722432
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003907241725175527,
+ "loss": 1.3568,
+ "theoretical_loss": 3.5055726242940617,
+ "tokens_seen": 1704787968
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003907081243731194,
+ "loss": 1.2819,
+ "theoretical_loss": 3.5055611265164477,
+ "tokens_seen": 1704853504
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039069207622868606,
+ "loss": 1.3309,
+ "theoretical_loss": 3.50554962930456,
+ "tokens_seen": 1704919040
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039067602808425276,
+ "loss": 1.2358,
+ "theoretical_loss": 3.5055381326583483,
+ "tokens_seen": 1704984576
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039065997993981946,
+ "loss": 1.3174,
+ "theoretical_loss": 3.5055266365777635,
+ "tokens_seen": 1705050112
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039064393179538616,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5055151410627556,
+ "tokens_seen": 1705115648
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039062788365095286,
+ "loss": 1.3895,
+ "theoretical_loss": 3.505503646113276,
+ "tokens_seen": 1705181184
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039061183550651957,
+ "loss": 1.2752,
+ "theoretical_loss": 3.505492151729274,
+ "tokens_seen": 1705246720
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039059578736208627,
+ "loss": 1.3243,
+ "theoretical_loss": 3.5054806579107,
+ "tokens_seen": 1705312256
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039057973921765297,
+ "loss": 1.3128,
+ "theoretical_loss": 3.5054691646575056,
+ "tokens_seen": 1705377792
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039056369107321967,
+ "loss": 1.2306,
+ "theoretical_loss": 3.50545767196964,
+ "tokens_seen": 1705443328
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039054764292878637,
+ "loss": 1.3383,
+ "theoretical_loss": 3.505446179847055,
+ "tokens_seen": 1705508864
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039053159478435307,
+ "loss": 1.2969,
+ "theoretical_loss": 3.5054346882897,
+ "tokens_seen": 1705574400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039051554663991977,
+ "loss": 1.2773,
+ "theoretical_loss": 3.5054231972975263,
+ "tokens_seen": 1705639936
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039049949849548647,
+ "loss": 1.3803,
+ "theoretical_loss": 3.5054117068704835,
+ "tokens_seen": 1705705472
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039048345035105317,
+ "loss": 1.2678,
+ "theoretical_loss": 3.5054002170085226,
+ "tokens_seen": 1705771008
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039046740220661987,
+ "loss": 1.3976,
+ "theoretical_loss": 3.505388727711594,
+ "tokens_seen": 1705836544
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039045135406218657,
+ "loss": 1.3176,
+ "theoretical_loss": 3.5053772389796487,
+ "tokens_seen": 1705902080
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039043530591775327,
+ "loss": 1.2333,
+ "theoretical_loss": 3.505365750812636,
+ "tokens_seen": 1705967616
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00039041925777331997,
+ "loss": 1.271,
+ "theoretical_loss": 3.5053542632105077,
+ "tokens_seen": 1706033152
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003904032096288867,
+ "loss": 1.196,
+ "theoretical_loss": 3.5053427761732134,
+ "tokens_seen": 1706098688
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903871614844534,
+ "loss": 1.25,
+ "theoretical_loss": 3.5053312897007043,
+ "tokens_seen": 1706164224
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903711133400201,
+ "loss": 1.2192,
+ "theoretical_loss": 3.505319803792931,
+ "tokens_seen": 1706229760
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903550651955868,
+ "loss": 1.3528,
+ "theoretical_loss": 3.5053083184498433,
+ "tokens_seen": 1706295296
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903390170511535,
+ "loss": 1.3391,
+ "theoretical_loss": 3.5052968336713923,
+ "tokens_seen": 1706360832
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903229689067202,
+ "loss": 1.317,
+ "theoretical_loss": 3.5052853494575285,
+ "tokens_seen": 1706426368
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003903069207622869,
+ "loss": 1.2589,
+ "theoretical_loss": 3.505273865808202,
+ "tokens_seen": 1706491904
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003902908726178536,
+ "loss": 1.2602,
+ "theoretical_loss": 3.505262382723364,
+ "tokens_seen": 1706557440
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003902748244734203,
+ "loss": 1.2766,
+ "theoretical_loss": 3.505250900202965,
+ "tokens_seen": 1706622976
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000390258776328987,
+ "loss": 1.2648,
+ "theoretical_loss": 3.5052394182469553,
+ "tokens_seen": 1706688512
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003902427281845537,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5052279368552854,
+ "tokens_seen": 1706754048
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003902266800401204,
+ "loss": 1.2978,
+ "theoretical_loss": 3.505216456027907,
+ "tokens_seen": 1706819584
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003902106318956871,
+ "loss": 1.22,
+ "theoretical_loss": 3.5052049757647685,
+ "tokens_seen": 1706885120
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901945837512538,
+ "loss": 1.2294,
+ "theoretical_loss": 3.5051934960658224,
+ "tokens_seen": 1706950656
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901785356068205,
+ "loss": 1.2286,
+ "theoretical_loss": 3.505182016931019,
+ "tokens_seen": 1707016192
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901624874623872,
+ "loss": 1.3281,
+ "theoretical_loss": 3.5051705383603085,
+ "tokens_seen": 1707081728
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901464393179539,
+ "loss": 1.2442,
+ "theoretical_loss": 3.5051590603536416,
+ "tokens_seen": 1707147264
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 572081,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.278428077697754,
+ "objective/train/theoretical_loss": 3.505147582910969,
+ "objective/train/tokens_used": 1727672800,
+ "theoretical_loss": 3.505147582910969,
+ "tokens_seen": 1707212800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901303911735206,
+ "loss": 1.2695,
+ "theoretical_loss": 3.505147582910969,
+ "tokens_seen": 1707212800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003901143430290873,
+ "loss": 1.2812,
+ "theoretical_loss": 3.505136106032241,
+ "tokens_seen": 1707278336
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000390098294884654,
+ "loss": 1.3015,
+ "theoretical_loss": 3.5051246297174092,
+ "tokens_seen": 1707343872
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900822467402207,
+ "loss": 1.254,
+ "theoretical_loss": 3.5051131539664233,
+ "tokens_seen": 1707409408
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900661985957874,
+ "loss": 1.2437,
+ "theoretical_loss": 3.5051016787792344,
+ "tokens_seen": 1707474944
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900501504513541,
+ "loss": 1.3657,
+ "theoretical_loss": 3.5050902041557928,
+ "tokens_seen": 1707540480
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900341023069208,
+ "loss": 1.2889,
+ "theoretical_loss": 3.5050787300960495,
+ "tokens_seen": 1707606016
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900180541624875,
+ "loss": 1.2741,
+ "theoretical_loss": 3.5050672565999554,
+ "tokens_seen": 1707671552
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003900020060180542,
+ "loss": 1.224,
+ "theoretical_loss": 3.5050557836674607,
+ "tokens_seen": 1707737088
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003899859578736209,
+ "loss": 1.2895,
+ "theoretical_loss": 3.5050443112985166,
+ "tokens_seen": 1707802624
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003899699097291876,
+ "loss": 1.3661,
+ "theoretical_loss": 3.505032839493073,
+ "tokens_seen": 1707868160
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003899538615847543,
+ "loss": 1.2699,
+ "theoretical_loss": 3.5050213682510813,
+ "tokens_seen": 1707933696
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000389937813440321,
+ "loss": 1.3541,
+ "theoretical_loss": 3.5050098975724917,
+ "tokens_seen": 1707999232
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038992176529588764,
+ "loss": 1.1876,
+ "theoretical_loss": 3.504998427457255,
+ "tokens_seen": 1708064768
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038990571715145434,
+ "loss": 1.3763,
+ "theoretical_loss": 3.5049869579053228,
+ "tokens_seen": 1708130304
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038988966900702104,
+ "loss": 1.3298,
+ "theoretical_loss": 3.504975488916645,
+ "tokens_seen": 1708195840
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038987362086258774,
+ "loss": 1.2195,
+ "theoretical_loss": 3.504964020491172,
+ "tokens_seen": 1708261376
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038985757271815444,
+ "loss": 1.2675,
+ "theoretical_loss": 3.504952552628855,
+ "tokens_seen": 1708326912
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038984152457372114,
+ "loss": 1.3184,
+ "theoretical_loss": 3.504941085329645,
+ "tokens_seen": 1708392448
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038982547642928784,
+ "loss": 1.2692,
+ "theoretical_loss": 3.5049296185934926,
+ "tokens_seen": 1708457984
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038980942828485454,
+ "loss": 1.1954,
+ "theoretical_loss": 3.5049181524203483,
+ "tokens_seen": 1708523520
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038979338014042124,
+ "loss": 1.2956,
+ "theoretical_loss": 3.504906686810163,
+ "tokens_seen": 1708589056
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000389777331995988,
+ "loss": 1.2719,
+ "theoretical_loss": 3.5048952217628875,
+ "tokens_seen": 1708654592
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003897612838515547,
+ "loss": 1.2865,
+ "theoretical_loss": 3.504883757278473,
+ "tokens_seen": 1708720128
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003897452357071214,
+ "loss": 1.3534,
+ "theoretical_loss": 3.5048722933568692,
+ "tokens_seen": 1708785664
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003897291875626881,
+ "loss": 1.411,
+ "theoretical_loss": 3.5048608299980275,
+ "tokens_seen": 1708851200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003897131394182548,
+ "loss": 1.3395,
+ "theoretical_loss": 3.5048493672018988,
+ "tokens_seen": 1708916736
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896970912738215,
+ "loss": 1.2185,
+ "theoretical_loss": 3.5048379049684337,
+ "tokens_seen": 1708982272
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896810431293882,
+ "loss": 1.1767,
+ "theoretical_loss": 3.504826443297584,
+ "tokens_seen": 1709047808
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896649949849549,
+ "loss": 1.2424,
+ "theoretical_loss": 3.5048149821892984,
+ "tokens_seen": 1709113344
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896489468405216,
+ "loss": 1.2404,
+ "theoretical_loss": 3.50480352164353,
+ "tokens_seen": 1709178880
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896328986960883,
+ "loss": 1.192,
+ "theoretical_loss": 3.504792061660228,
+ "tokens_seen": 1709244416
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000389616850551655,
+ "loss": 1.3171,
+ "theoretical_loss": 3.504780602239344,
+ "tokens_seen": 1709309952
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003896008024072217,
+ "loss": 1.1987,
+ "theoretical_loss": 3.5047691433808286,
+ "tokens_seen": 1709375488
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895847542627884,
+ "loss": 1.4319,
+ "theoretical_loss": 3.5047576850846323,
+ "tokens_seen": 1709441024
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895687061183551,
+ "loss": 1.2345,
+ "theoretical_loss": 3.5047462273507066,
+ "tokens_seen": 1709506560
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895526579739218,
+ "loss": 1.3256,
+ "theoretical_loss": 3.5047347701790024,
+ "tokens_seen": 1709572096
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895366098294885,
+ "loss": 1.3368,
+ "theoretical_loss": 3.5047233135694698,
+ "tokens_seen": 1709637632
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895205616850552,
+ "loss": 1.3262,
+ "theoretical_loss": 3.50471185752206,
+ "tokens_seen": 1709703168
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003895045135406219,
+ "loss": 1.3029,
+ "theoretical_loss": 3.5047004020367245,
+ "tokens_seen": 1709768704
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003894884653961886,
+ "loss": 1.2998,
+ "theoretical_loss": 3.5046889471134133,
+ "tokens_seen": 1709834240
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003894724172517553,
+ "loss": 1.222,
+ "theoretical_loss": 3.504677492752078,
+ "tokens_seen": 1709899776
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000389456369107322,
+ "loss": 1.3711,
+ "theoretical_loss": 3.504666038952669,
+ "tokens_seen": 1709965312
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003894403209628887,
+ "loss": 1.2662,
+ "theoretical_loss": 3.504654585715137,
+ "tokens_seen": 1710030848
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003894242728184554,
+ "loss": 1.3747,
+ "theoretical_loss": 3.5046431330394334,
+ "tokens_seen": 1710096384
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003894082246740221,
+ "loss": 1.1992,
+ "theoretical_loss": 3.504631680925509,
+ "tokens_seen": 1710161920
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893921765295888,
+ "loss": 1.2252,
+ "theoretical_loss": 3.504620229373315,
+ "tokens_seen": 1710227456
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893761283851555,
+ "loss": 1.2379,
+ "theoretical_loss": 3.504608778382802,
+ "tokens_seen": 1710292992
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893600802407222,
+ "loss": 1.2257,
+ "theoretical_loss": 3.5045973279539204,
+ "tokens_seen": 1710358528
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893440320962889,
+ "loss": 1.2381,
+ "theoretical_loss": 3.5045858780866217,
+ "tokens_seen": 1710424064
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 573219,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2893404960632324,
+ "objective/train/theoretical_loss": 3.5045744287808573,
+ "objective/train/tokens_used": 1730949600,
+ "theoretical_loss": 3.5045744287808573,
+ "tokens_seen": 1710489600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893279839518556,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5045744287808573,
+ "tokens_seen": 1710489600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003893119358074223,
+ "loss": 1.2419,
+ "theoretical_loss": 3.504562980036577,
+ "tokens_seen": 1710555136
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038929588766298896,
+ "loss": 1.3192,
+ "theoretical_loss": 3.5045515318537332,
+ "tokens_seen": 1710620672
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038927983951855566,
+ "loss": 1.3724,
+ "theoretical_loss": 3.504540084232276,
+ "tokens_seen": 1710686208
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038926379137412236,
+ "loss": 1.281,
+ "theoretical_loss": 3.5045286371721556,
+ "tokens_seen": 1710751744
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038924774322968906,
+ "loss": 1.1835,
+ "theoretical_loss": 3.5045171906733246,
+ "tokens_seen": 1710817280
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038923169508525577,
+ "loss": 1.1356,
+ "theoretical_loss": 3.5045057447357326,
+ "tokens_seen": 1710882816
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038921564694082247,
+ "loss": 1.3201,
+ "theoretical_loss": 3.5044942993593318,
+ "tokens_seen": 1710948352
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038919959879638917,
+ "loss": 1.2701,
+ "theoretical_loss": 3.5044828545440723,
+ "tokens_seen": 1711013888
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038918355065195587,
+ "loss": 1.2679,
+ "theoretical_loss": 3.504471410289905,
+ "tokens_seen": 1711079424
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038916750250752257,
+ "loss": 1.2908,
+ "theoretical_loss": 3.504459966596782,
+ "tokens_seen": 1711144960
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038915145436308927,
+ "loss": 1.1841,
+ "theoretical_loss": 3.504448523464653,
+ "tokens_seen": 1711210496
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038913540621865597,
+ "loss": 1.2049,
+ "theoretical_loss": 3.5044370808934695,
+ "tokens_seen": 1711276032
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038911935807422267,
+ "loss": 1.2412,
+ "theoretical_loss": 3.5044256388831827,
+ "tokens_seen": 1711341568
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038910330992978937,
+ "loss": 1.2963,
+ "theoretical_loss": 3.5044141974337437,
+ "tokens_seen": 1711407104
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038908726178535607,
+ "loss": 1.2257,
+ "theoretical_loss": 3.5044027565451032,
+ "tokens_seen": 1711472640
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038907121364092277,
+ "loss": 1.3079,
+ "theoretical_loss": 3.5043913162172124,
+ "tokens_seen": 1711538176
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038905516549648947,
+ "loss": 1.222,
+ "theoretical_loss": 3.5043798764500225,
+ "tokens_seen": 1711603712
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003890391173520562,
+ "loss": 1.241,
+ "theoretical_loss": 3.5043684372434845,
+ "tokens_seen": 1711669248
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003890230692076229,
+ "loss": 1.3066,
+ "theoretical_loss": 3.504356998597549,
+ "tokens_seen": 1711734784
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003890070210631896,
+ "loss": 1.1662,
+ "theoretical_loss": 3.504345560512168,
+ "tokens_seen": 1711800320
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003889909729187563,
+ "loss": 1.3688,
+ "theoretical_loss": 3.5043341229872915,
+ "tokens_seen": 1711865856
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388974924774323,
+ "loss": 1.3321,
+ "theoretical_loss": 3.504322686022871,
+ "tokens_seen": 1711931392
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003889588766298897,
+ "loss": 1.3109,
+ "theoretical_loss": 3.504311249618858,
+ "tokens_seen": 1711996928
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003889428284854564,
+ "loss": 1.3121,
+ "theoretical_loss": 3.5042998137752033,
+ "tokens_seen": 1712062464
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003889267803410231,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5042883784918577,
+ "tokens_seen": 1712128000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003889107321965898,
+ "loss": 1.2557,
+ "theoretical_loss": 3.5042769437687724,
+ "tokens_seen": 1712193536
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003888946840521565,
+ "loss": 1.344,
+ "theoretical_loss": 3.504265509605899,
+ "tokens_seen": 1712259072
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003888786359077232,
+ "loss": 1.1765,
+ "theoretical_loss": 3.504254076003188,
+ "tokens_seen": 1712324608
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003888625877632899,
+ "loss": 1.3774,
+ "theoretical_loss": 3.504242642960591,
+ "tokens_seen": 1712390144
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003888465396188566,
+ "loss": 1.3669,
+ "theoretical_loss": 3.504231210478059,
+ "tokens_seen": 1712455680
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003888304914744233,
+ "loss": 1.1514,
+ "theoretical_loss": 3.5042197785555427,
+ "tokens_seen": 1712521216
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038881444332999,
+ "loss": 1.286,
+ "theoretical_loss": 3.5042083471929937,
+ "tokens_seen": 1712586752
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887983951855567,
+ "loss": 1.2841,
+ "theoretical_loss": 3.504196916390363,
+ "tokens_seen": 1712652288
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887823470411234,
+ "loss": 1.3139,
+ "theoretical_loss": 3.504185486147602,
+ "tokens_seen": 1712717824
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887662988966901,
+ "loss": 1.262,
+ "theoretical_loss": 3.5041740564646613,
+ "tokens_seen": 1712783360
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887502507522568,
+ "loss": 1.2651,
+ "theoretical_loss": 3.5041626273414925,
+ "tokens_seen": 1712848896
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887342026078235,
+ "loss": 1.2129,
+ "theoretical_loss": 3.504151198778047,
+ "tokens_seen": 1712914432
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887181544633902,
+ "loss": 1.2808,
+ "theoretical_loss": 3.504139770774276,
+ "tokens_seen": 1712979968
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003887021063189569,
+ "loss": 1.1952,
+ "theoretical_loss": 3.5041283433301293,
+ "tokens_seen": 1713045504
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003886860581745236,
+ "loss": 1.3395,
+ "theoretical_loss": 3.5041169164455592,
+ "tokens_seen": 1713111040
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003886700100300903,
+ "loss": 1.2485,
+ "theoretical_loss": 3.5041054901205175,
+ "tokens_seen": 1713176576
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388653961885657,
+ "loss": 1.3806,
+ "theoretical_loss": 3.504094064354954,
+ "tokens_seen": 1713242112
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003886379137412237,
+ "loss": 1.2293,
+ "theoretical_loss": 3.5040826391488213,
+ "tokens_seen": 1713307648
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003886218655967904,
+ "loss": 1.2129,
+ "theoretical_loss": 3.504071214502069,
+ "tokens_seen": 1713373184
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003886058174523571,
+ "loss": 1.3016,
+ "theoretical_loss": 3.50405979041465,
+ "tokens_seen": 1713438720
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885897693079238,
+ "loss": 1.2437,
+ "theoretical_loss": 3.504048366886515,
+ "tokens_seen": 1713504256
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885737211634905,
+ "loss": 1.3001,
+ "theoretical_loss": 3.5040369439176144,
+ "tokens_seen": 1713569792
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885576730190572,
+ "loss": 1.2468,
+ "theoretical_loss": 3.5040255215079004,
+ "tokens_seen": 1713635328
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885416248746239,
+ "loss": 1.3741,
+ "theoretical_loss": 3.504014099657323,
+ "tokens_seen": 1713700864
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 574441,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.194473147392273,
+ "objective/train/theoretical_loss": 3.5040026783658353,
+ "objective/train/tokens_used": 1734226400,
+ "theoretical_loss": 3.5040026783658353,
+ "tokens_seen": 1713766400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885255767301906,
+ "loss": 1.274,
+ "theoretical_loss": 3.5040026783658353,
+ "tokens_seen": 1713766400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003885095285857573,
+ "loss": 1.1716,
+ "theoretical_loss": 3.5039912576333876,
+ "tokens_seen": 1713831936
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388493480441324,
+ "loss": 1.3378,
+ "theoretical_loss": 3.5039798374599305,
+ "tokens_seen": 1713897472
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003884774322968907,
+ "loss": 1.2533,
+ "theoretical_loss": 3.5039684178454165,
+ "tokens_seen": 1713963008
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003884613841524574,
+ "loss": 1.2971,
+ "theoretical_loss": 3.5039569987897954,
+ "tokens_seen": 1714028544
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003884453360080241,
+ "loss": 1.2658,
+ "theoretical_loss": 3.50394558029302,
+ "tokens_seen": 1714094080
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003884292878635908,
+ "loss": 1.2133,
+ "theoretical_loss": 3.5039341623550406,
+ "tokens_seen": 1714159616
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003884132397191575,
+ "loss": 1.3748,
+ "theoretical_loss": 3.5039227449758092,
+ "tokens_seen": 1714225152
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883971915747242,
+ "loss": 1.2349,
+ "theoretical_loss": 3.5039113281552767,
+ "tokens_seen": 1714290688
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883811434302909,
+ "loss": 1.1814,
+ "theoretical_loss": 3.5038999118933942,
+ "tokens_seen": 1714356224
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883650952858576,
+ "loss": 1.1674,
+ "theoretical_loss": 3.5038884961901138,
+ "tokens_seen": 1714421760
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883490471414243,
+ "loss": 1.2864,
+ "theoretical_loss": 3.5038770810453856,
+ "tokens_seen": 1714487296
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388332998996991,
+ "loss": 1.2061,
+ "theoretical_loss": 3.503865666459162,
+ "tokens_seen": 1714552832
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883169508525577,
+ "loss": 1.2857,
+ "theoretical_loss": 3.5038542524313936,
+ "tokens_seen": 1714618368
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003883009027081244,
+ "loss": 1.248,
+ "theoretical_loss": 3.503842838962032,
+ "tokens_seen": 1714683904
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882848545636911,
+ "loss": 1.291,
+ "theoretical_loss": 3.503831426051029,
+ "tokens_seen": 1714749440
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882688064192578,
+ "loss": 1.2367,
+ "theoretical_loss": 3.503820013698335,
+ "tokens_seen": 1714814976
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882527582748245,
+ "loss": 1.3266,
+ "theoretical_loss": 3.5038086019039025,
+ "tokens_seen": 1714880512
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882367101303912,
+ "loss": 1.2368,
+ "theoretical_loss": 3.503797190667682,
+ "tokens_seen": 1714946048
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882206619859579,
+ "loss": 1.3412,
+ "theoretical_loss": 3.5037857799896246,
+ "tokens_seen": 1715011584
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003882046138415246,
+ "loss": 1.2556,
+ "theoretical_loss": 3.503774369869683,
+ "tokens_seen": 1715077120
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003881885656970913,
+ "loss": 1.2661,
+ "theoretical_loss": 3.503762960307807,
+ "tokens_seen": 1715142656
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388172517552658,
+ "loss": 1.1965,
+ "theoretical_loss": 3.503751551303949,
+ "tokens_seen": 1715208192
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003881564694082247,
+ "loss": 1.2574,
+ "theoretical_loss": 3.50374014285806,
+ "tokens_seen": 1715273728
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003881404212637914,
+ "loss": 1.1964,
+ "theoretical_loss": 3.503728734970092,
+ "tokens_seen": 1715339264
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003881243731193581,
+ "loss": 1.3806,
+ "theoretical_loss": 3.5037173276399955,
+ "tokens_seen": 1715404800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003881083249749248,
+ "loss": 1.2613,
+ "theoretical_loss": 3.5037059208677226,
+ "tokens_seen": 1715470336
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003880922768304915,
+ "loss": 1.2779,
+ "theoretical_loss": 3.5036945146532243,
+ "tokens_seen": 1715535872
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003880762286860582,
+ "loss": 1.2052,
+ "theoretical_loss": 3.503683108996452,
+ "tokens_seen": 1715601408
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003880601805416249,
+ "loss": 1.2645,
+ "theoretical_loss": 3.503671703897357,
+ "tokens_seen": 1715666944
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003880441323971916,
+ "loss": 1.3068,
+ "theoretical_loss": 3.503660299355892,
+ "tokens_seen": 1715732480
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003880280842527583,
+ "loss": 1.2085,
+ "theoretical_loss": 3.503648895372007,
+ "tokens_seen": 1715798016
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000388012036108325,
+ "loss": 1.2277,
+ "theoretical_loss": 3.5036374919456534,
+ "tokens_seen": 1715863552
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879959879638917,
+ "loss": 1.2956,
+ "theoretical_loss": 3.503626089076784,
+ "tokens_seen": 1715929088
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879799398194584,
+ "loss": 1.2481,
+ "theoretical_loss": 3.5036146867653484,
+ "tokens_seen": 1715994624
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879638916750251,
+ "loss": 1.3012,
+ "theoretical_loss": 3.5036032850113,
+ "tokens_seen": 1716060160
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879478435305918,
+ "loss": 1.3273,
+ "theoretical_loss": 3.5035918838145887,
+ "tokens_seen": 1716125696
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879317953861585,
+ "loss": 1.2891,
+ "theoretical_loss": 3.503580483175167,
+ "tokens_seen": 1716191232
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003879157472417252,
+ "loss": 1.2986,
+ "theoretical_loss": 3.5035690830929855,
+ "tokens_seen": 1716256768
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038789969909729186,
+ "loss": 1.1123,
+ "theoretical_loss": 3.5035576835679967,
+ "tokens_seen": 1716322304
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038788365095285856,
+ "loss": 1.346,
+ "theoretical_loss": 3.503546284600151,
+ "tokens_seen": 1716387840
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038786760280842527,
+ "loss": 1.233,
+ "theoretical_loss": 3.503534886189401,
+ "tokens_seen": 1716453376
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038785155466399197,
+ "loss": 1.2708,
+ "theoretical_loss": 3.5035234883356976,
+ "tokens_seen": 1716518912
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038783550651955867,
+ "loss": 1.2776,
+ "theoretical_loss": 3.503512091038992,
+ "tokens_seen": 1716584448
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038781945837512537,
+ "loss": 1.3255,
+ "theoretical_loss": 3.5035006942992366,
+ "tokens_seen": 1716649984
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038780341023069207,
+ "loss": 1.3007,
+ "theoretical_loss": 3.503489298116382,
+ "tokens_seen": 1716715520
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038778736208625877,
+ "loss": 1.2798,
+ "theoretical_loss": 3.5034779024903804,
+ "tokens_seen": 1716781056
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038777131394182547,
+ "loss": 1.3119,
+ "theoretical_loss": 3.503466507421183,
+ "tokens_seen": 1716846592
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038775526579739217,
+ "loss": 1.2688,
+ "theoretical_loss": 3.503455112908741,
+ "tokens_seen": 1716912128
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038773921765295887,
+ "loss": 1.1861,
+ "theoretical_loss": 3.5034437189530068,
+ "tokens_seen": 1716977664
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 575414,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4047168493270874,
+ "objective/train/theoretical_loss": 3.5034323255539315,
+ "objective/train/tokens_used": 1737503200,
+ "theoretical_loss": 3.5034323255539315,
+ "tokens_seen": 1717043200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038772316950852557,
+ "loss": 1.3359,
+ "theoretical_loss": 3.5034323255539315,
+ "tokens_seen": 1717043200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038770712136409227,
+ "loss": 1.1252,
+ "theoretical_loss": 3.5034209327114665,
+ "tokens_seen": 1717108736
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038769107321965897,
+ "loss": 1.2256,
+ "theoretical_loss": 3.503409540425564,
+ "tokens_seen": 1717174272
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038767502507522567,
+ "loss": 1.3175,
+ "theoretical_loss": 3.5033981486961747,
+ "tokens_seen": 1717239808
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003876589769307924,
+ "loss": 1.1878,
+ "theoretical_loss": 3.503386757523251,
+ "tokens_seen": 1717305344
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003876429287863591,
+ "loss": 1.2558,
+ "theoretical_loss": 3.5033753669067433,
+ "tokens_seen": 1717370880
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003876268806419258,
+ "loss": 1.2643,
+ "theoretical_loss": 3.5033639768466047,
+ "tokens_seen": 1717436416
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003876108324974925,
+ "loss": 1.3296,
+ "theoretical_loss": 3.5033525873427855,
+ "tokens_seen": 1717501952
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003875947843530592,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5033411983952387,
+ "tokens_seen": 1717567488
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003875787362086259,
+ "loss": 1.2233,
+ "theoretical_loss": 3.5033298100039145,
+ "tokens_seen": 1717633024
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003875626880641926,
+ "loss": 1.2723,
+ "theoretical_loss": 3.503318422168765,
+ "tokens_seen": 1717698560
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003875466399197593,
+ "loss": 1.2527,
+ "theoretical_loss": 3.503307034889742,
+ "tokens_seen": 1717764096
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000387530591775326,
+ "loss": 1.2452,
+ "theoretical_loss": 3.503295648166797,
+ "tokens_seen": 1717829632
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003875145436308927,
+ "loss": 1.2603,
+ "theoretical_loss": 3.503284261999882,
+ "tokens_seen": 1717895168
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874984954864594,
+ "loss": 1.284,
+ "theoretical_loss": 3.503272876388948,
+ "tokens_seen": 1717960704
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874824473420261,
+ "loss": 1.3345,
+ "theoretical_loss": 3.503261491333947,
+ "tokens_seen": 1718026240
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874663991975928,
+ "loss": 1.3111,
+ "theoretical_loss": 3.5032501068348307,
+ "tokens_seen": 1718091776
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874503510531595,
+ "loss": 1.2987,
+ "theoretical_loss": 3.5032387228915507,
+ "tokens_seen": 1718157312
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874343029087262,
+ "loss": 1.1232,
+ "theoretical_loss": 3.503227339504058,
+ "tokens_seen": 1718222848
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874182547642929,
+ "loss": 1.2476,
+ "theoretical_loss": 3.503215956672306,
+ "tokens_seen": 1718288384
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003874022066198596,
+ "loss": 1.2676,
+ "theoretical_loss": 3.503204574396244,
+ "tokens_seen": 1718353920
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038738615847542634,
+ "loss": 1.3392,
+ "theoretical_loss": 3.5031931926758255,
+ "tokens_seen": 1718419456
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038737011033099304,
+ "loss": 1.0821,
+ "theoretical_loss": 3.5031818115110016,
+ "tokens_seen": 1718484992
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038735406218655974,
+ "loss": 1.2518,
+ "theoretical_loss": 3.503170430901724,
+ "tokens_seen": 1718550528
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038733801404212644,
+ "loss": 1.3108,
+ "theoretical_loss": 3.5031590508479447,
+ "tokens_seen": 1718616064
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038732196589769314,
+ "loss": 1.3032,
+ "theoretical_loss": 3.503147671349615,
+ "tokens_seen": 1718681600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038730591775325984,
+ "loss": 1.2388,
+ "theoretical_loss": 3.5031362924066864,
+ "tokens_seen": 1718747136
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038728986960882654,
+ "loss": 1.2785,
+ "theoretical_loss": 3.503124914019111,
+ "tokens_seen": 1718812672
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038727382146439324,
+ "loss": 1.2557,
+ "theoretical_loss": 3.5031135361868406,
+ "tokens_seen": 1718878208
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003872577733199599,
+ "loss": 1.295,
+ "theoretical_loss": 3.503102158909827,
+ "tokens_seen": 1718943744
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003872417251755266,
+ "loss": 1.2851,
+ "theoretical_loss": 3.5030907821880213,
+ "tokens_seen": 1719009280
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003872256770310933,
+ "loss": 1.2851,
+ "theoretical_loss": 3.503079406021376,
+ "tokens_seen": 1719074816
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038720962888666,
+ "loss": 1.2895,
+ "theoretical_loss": 3.5030680304098425,
+ "tokens_seen": 1719140352
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871935807422267,
+ "loss": 1.3034,
+ "theoretical_loss": 3.5030566553533724,
+ "tokens_seen": 1719205888
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871775325977934,
+ "loss": 1.3101,
+ "theoretical_loss": 3.503045280851918,
+ "tokens_seen": 1719271424
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871614844533601,
+ "loss": 1.3319,
+ "theoretical_loss": 3.50303390690543,
+ "tokens_seen": 1719336960
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871454363089268,
+ "loss": 1.2764,
+ "theoretical_loss": 3.5030225335138616,
+ "tokens_seen": 1719402496
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871293881644935,
+ "loss": 1.2565,
+ "theoretical_loss": 3.503011160677164,
+ "tokens_seen": 1719468032
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003871133400200602,
+ "loss": 1.2869,
+ "theoretical_loss": 3.5029997883952877,
+ "tokens_seen": 1719533568
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870972918756269,
+ "loss": 1.3573,
+ "theoretical_loss": 3.5029884166681864,
+ "tokens_seen": 1719599104
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870812437311936,
+ "loss": 1.307,
+ "theoretical_loss": 3.502977045495811,
+ "tokens_seen": 1719664640
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870651955867603,
+ "loss": 1.2413,
+ "theoretical_loss": 3.502965674878113,
+ "tokens_seen": 1719730176
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000387049147442327,
+ "loss": 1.393,
+ "theoretical_loss": 3.502954304815045,
+ "tokens_seen": 1719795712
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870330992978937,
+ "loss": 1.2674,
+ "theoretical_loss": 3.5029429353065584,
+ "tokens_seen": 1719861248
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870170511534604,
+ "loss": 1.2347,
+ "theoretical_loss": 3.502931566352605,
+ "tokens_seen": 1719926784
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003870010030090271,
+ "loss": 1.3603,
+ "theoretical_loss": 3.502920197953137,
+ "tokens_seen": 1719992320
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869849548645938,
+ "loss": 1.3203,
+ "theoretical_loss": 3.5029088301081055,
+ "tokens_seen": 1720057856
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869689067201605,
+ "loss": 1.2947,
+ "theoretical_loss": 3.502897462817463,
+ "tokens_seen": 1720123392
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869528585757272,
+ "loss": 1.2552,
+ "theoretical_loss": 3.502886096081161,
+ "tokens_seen": 1720188928
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869368104312939,
+ "loss": 1.3952,
+ "theoretical_loss": 3.502874729899151,
+ "tokens_seen": 1720254464
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 576723,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3283004760742188,
+ "objective/train/theoretical_loss": 3.502863364271386,
+ "objective/train/tokens_used": 1740780000,
+ "theoretical_loss": 3.502863364271386,
+ "tokens_seen": 1720320000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869207622868606,
+ "loss": 1.2018,
+ "theoretical_loss": 3.502863364271386,
+ "tokens_seen": 1720320000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003869047141424273,
+ "loss": 1.1755,
+ "theoretical_loss": 3.5028519991978166,
+ "tokens_seen": 1720385536
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000386888665997994,
+ "loss": 1.2339,
+ "theoretical_loss": 3.5028406346783956,
+ "tokens_seen": 1720451072
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003868726178535607,
+ "loss": 1.2661,
+ "theoretical_loss": 3.5028292707130744,
+ "tokens_seen": 1720516608
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003868565697091274,
+ "loss": 1.263,
+ "theoretical_loss": 3.5028179073018046,
+ "tokens_seen": 1720582144
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003868405215646941,
+ "loss": 1.2141,
+ "theoretical_loss": 3.502806544444539,
+ "tokens_seen": 1720647680
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003868244734202608,
+ "loss": 1.3295,
+ "theoretical_loss": 3.5027951821412286,
+ "tokens_seen": 1720713216
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003868084252758275,
+ "loss": 1.2162,
+ "theoretical_loss": 3.5027838203918256,
+ "tokens_seen": 1720778752
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003867923771313942,
+ "loss": 1.1668,
+ "theoretical_loss": 3.502772459196282,
+ "tokens_seen": 1720844288
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003867763289869609,
+ "loss": 1.2905,
+ "theoretical_loss": 3.50276109855455,
+ "tokens_seen": 1720909824
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003867602808425276,
+ "loss": 1.2931,
+ "theoretical_loss": 3.502749738466581,
+ "tokens_seen": 1720975360
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003867442326980943,
+ "loss": 1.263,
+ "theoretical_loss": 3.5027383789323263,
+ "tokens_seen": 1721040896
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000386728184553661,
+ "loss": 1.2885,
+ "theoretical_loss": 3.5027270199517395,
+ "tokens_seen": 1721106432
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003867121364092277,
+ "loss": 1.2198,
+ "theoretical_loss": 3.502715661524771,
+ "tokens_seen": 1721171968
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866960882647944,
+ "loss": 1.2981,
+ "theoretical_loss": 3.5027043036513743,
+ "tokens_seen": 1721237504
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866800401203611,
+ "loss": 1.2177,
+ "theoretical_loss": 3.5026929463315,
+ "tokens_seen": 1721303040
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866639919759278,
+ "loss": 1.3437,
+ "theoretical_loss": 3.5026815895651002,
+ "tokens_seen": 1721368576
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866479438314945,
+ "loss": 1.36,
+ "theoretical_loss": 3.5026702333521276,
+ "tokens_seen": 1721434112
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866318956870612,
+ "loss": 1.2333,
+ "theoretical_loss": 3.502658877692533,
+ "tokens_seen": 1721499648
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003866158475426279,
+ "loss": 1.2444,
+ "theoretical_loss": 3.50264752258627,
+ "tokens_seen": 1721565184
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865997993981946,
+ "loss": 1.1374,
+ "theoretical_loss": 3.5026361680332894,
+ "tokens_seen": 1721630720
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865837512537613,
+ "loss": 1.2729,
+ "theoretical_loss": 3.5026248140335428,
+ "tokens_seen": 1721696256
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000386567703109328,
+ "loss": 1.3754,
+ "theoretical_loss": 3.5026134605869834,
+ "tokens_seen": 1721761792
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865516549648947,
+ "loss": 1.2265,
+ "theoretical_loss": 3.5026021076935625,
+ "tokens_seen": 1721827328
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865356068204614,
+ "loss": 1.4701,
+ "theoretical_loss": 3.502590755353232,
+ "tokens_seen": 1721892864
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865195586760281,
+ "loss": 1.3204,
+ "theoretical_loss": 3.502579403565944,
+ "tokens_seen": 1721958400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003865035105315948,
+ "loss": 1.3213,
+ "theoretical_loss": 3.502568052331651,
+ "tokens_seen": 1722023936
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038648746238716147,
+ "loss": 1.2029,
+ "theoretical_loss": 3.502556701650305,
+ "tokens_seen": 1722089472
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038647141424272817,
+ "loss": 1.3054,
+ "theoretical_loss": 3.502545351521857,
+ "tokens_seen": 1722155008
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038645536609829487,
+ "loss": 1.2125,
+ "theoretical_loss": 3.50253400194626,
+ "tokens_seen": 1722220544
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038643931795386157,
+ "loss": 1.2697,
+ "theoretical_loss": 3.5025226529234654,
+ "tokens_seen": 1722286080
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038642326980942827,
+ "loss": 1.2667,
+ "theoretical_loss": 3.502511304453426,
+ "tokens_seen": 1722351616
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038640722166499497,
+ "loss": 1.3068,
+ "theoretical_loss": 3.5024999565360933,
+ "tokens_seen": 1722417152
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038639117352056167,
+ "loss": 1.2916,
+ "theoretical_loss": 3.5024886091714196,
+ "tokens_seen": 1722482688
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038637512537612837,
+ "loss": 1.3041,
+ "theoretical_loss": 3.5024772623593563,
+ "tokens_seen": 1722548224
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038635907723169507,
+ "loss": 1.2427,
+ "theoretical_loss": 3.5024659160998564,
+ "tokens_seen": 1722613760
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038634302908726177,
+ "loss": 1.2501,
+ "theoretical_loss": 3.502454570392872,
+ "tokens_seen": 1722679296
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038632698094282847,
+ "loss": 1.3302,
+ "theoretical_loss": 3.5024432252383537,
+ "tokens_seen": 1722744832
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038631093279839517,
+ "loss": 1.3459,
+ "theoretical_loss": 3.502431880636255,
+ "tokens_seen": 1722810368
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003862948846539619,
+ "loss": 1.2801,
+ "theoretical_loss": 3.502420536586528,
+ "tokens_seen": 1722875904
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003862788365095286,
+ "loss": 1.3317,
+ "theoretical_loss": 3.5024091930891244,
+ "tokens_seen": 1722941440
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003862627883650953,
+ "loss": 1.2373,
+ "theoretical_loss": 3.5023978501439963,
+ "tokens_seen": 1723006976
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000386246740220662,
+ "loss": 1.3406,
+ "theoretical_loss": 3.5023865077510954,
+ "tokens_seen": 1723072512
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003862306920762287,
+ "loss": 1.2907,
+ "theoretical_loss": 3.502375165910374,
+ "tokens_seen": 1723138048
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003862146439317954,
+ "loss": 1.2452,
+ "theoretical_loss": 3.502363824621785,
+ "tokens_seen": 1723203584
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003861985957873621,
+ "loss": 1.2664,
+ "theoretical_loss": 3.5023524838852804,
+ "tokens_seen": 1723269120
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038618254764292883,
+ "loss": 1.2562,
+ "theoretical_loss": 3.5023411437008116,
+ "tokens_seen": 1723334656
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038616649949849553,
+ "loss": 1.2671,
+ "theoretical_loss": 3.5023298040683306,
+ "tokens_seen": 1723400192
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038615045135406223,
+ "loss": 1.2465,
+ "theoretical_loss": 3.5023184649877903,
+ "tokens_seen": 1723465728
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038613440320962893,
+ "loss": 1.2813,
+ "theoretical_loss": 3.5023071264591423,
+ "tokens_seen": 1723531264
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 577498,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2661242485046387,
+ "objective/train/theoretical_loss": 3.502295788482339,
+ "objective/train/tokens_used": 1744056800,
+ "theoretical_loss": 3.502295788482339,
+ "tokens_seen": 1723596800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038611835506519563,
+ "loss": 1.2756,
+ "theoretical_loss": 3.502295788482339,
+ "tokens_seen": 1723596800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038610230692076234,
+ "loss": 1.2662,
+ "theoretical_loss": 3.5022844510573328,
+ "tokens_seen": 1723662336
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038608625877632904,
+ "loss": 1.2964,
+ "theoretical_loss": 3.5022731141840753,
+ "tokens_seen": 1723727872
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038607021063189574,
+ "loss": 1.32,
+ "theoretical_loss": 3.502261777862519,
+ "tokens_seen": 1723793408
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038605416248746244,
+ "loss": 1.3984,
+ "theoretical_loss": 3.502250442092617,
+ "tokens_seen": 1723858944
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038603811434302914,
+ "loss": 1.2503,
+ "theoretical_loss": 3.5022391068743195,
+ "tokens_seen": 1723924480
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038602206619859584,
+ "loss": 1.4484,
+ "theoretical_loss": 3.50222777220758,
+ "tokens_seen": 1723990016
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038600601805416254,
+ "loss": 1.2121,
+ "theoretical_loss": 3.5022164380923506,
+ "tokens_seen": 1724055552
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038598996990972924,
+ "loss": 1.3436,
+ "theoretical_loss": 3.502205104528583,
+ "tokens_seen": 1724121088
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038597392176529594,
+ "loss": 1.2334,
+ "theoretical_loss": 3.5021937715162306,
+ "tokens_seen": 1724186624
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038595787362086264,
+ "loss": 1.2394,
+ "theoretical_loss": 3.502182439055244,
+ "tokens_seen": 1724252160
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038594182547642934,
+ "loss": 1.3184,
+ "theoretical_loss": 3.5021711071455766,
+ "tokens_seen": 1724317696
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038592577733199604,
+ "loss": 1.351,
+ "theoretical_loss": 3.5021597757871796,
+ "tokens_seen": 1724383232
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038590972918756274,
+ "loss": 1.2574,
+ "theoretical_loss": 3.502148444980006,
+ "tokens_seen": 1724448768
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038589368104312944,
+ "loss": 1.3638,
+ "theoretical_loss": 3.5021371147240083,
+ "tokens_seen": 1724514304
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038587763289869614,
+ "loss": 1.2326,
+ "theoretical_loss": 3.502125785019138,
+ "tokens_seen": 1724579840
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003858615847542628,
+ "loss": 1.3309,
+ "theoretical_loss": 3.502114455865348,
+ "tokens_seen": 1724645376
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003858455366098295,
+ "loss": 1.2836,
+ "theoretical_loss": 3.5021031272625898,
+ "tokens_seen": 1724710912
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003858294884653962,
+ "loss": 1.2931,
+ "theoretical_loss": 3.5020917992108167,
+ "tokens_seen": 1724776448
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003858134403209629,
+ "loss": 1.2563,
+ "theoretical_loss": 3.50208047170998,
+ "tokens_seen": 1724841984
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857973921765296,
+ "loss": 1.2966,
+ "theoretical_loss": 3.502069144760032,
+ "tokens_seen": 1724907520
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857813440320963,
+ "loss": 1.2582,
+ "theoretical_loss": 3.502057818360926,
+ "tokens_seen": 1724973056
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000385765295887663,
+ "loss": 1.1705,
+ "theoretical_loss": 3.502046492512613,
+ "tokens_seen": 1725038592
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857492477432297,
+ "loss": 1.383,
+ "theoretical_loss": 3.502035167215046,
+ "tokens_seen": 1725104128
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857331995987964,
+ "loss": 1.3285,
+ "theoretical_loss": 3.5020238424681773,
+ "tokens_seen": 1725169664
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857171514543631,
+ "loss": 1.2179,
+ "theoretical_loss": 3.5020125182719593,
+ "tokens_seen": 1725235200
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003857011033099298,
+ "loss": 1.2932,
+ "theoretical_loss": 3.502001194626344,
+ "tokens_seen": 1725300736
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003856850551654965,
+ "loss": 1.3432,
+ "theoretical_loss": 3.5019898715312836,
+ "tokens_seen": 1725366272
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003856690070210632,
+ "loss": 1.2372,
+ "theoretical_loss": 3.5019785489867306,
+ "tokens_seen": 1725431808
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003856529588766299,
+ "loss": 1.2005,
+ "theoretical_loss": 3.5019672269926376,
+ "tokens_seen": 1725497344
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003856369107321966,
+ "loss": 1.2437,
+ "theoretical_loss": 3.5019559055489564,
+ "tokens_seen": 1725562880
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003856208625877633,
+ "loss": 1.3594,
+ "theoretical_loss": 3.5019445846556403,
+ "tokens_seen": 1725628416
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038560481444333,
+ "loss": 1.3102,
+ "theoretical_loss": 3.5019332643126404,
+ "tokens_seen": 1725693952
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855887662988967,
+ "loss": 1.377,
+ "theoretical_loss": 3.5019219445199097,
+ "tokens_seen": 1725759488
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855727181544634,
+ "loss": 1.2909,
+ "theoretical_loss": 3.5019106252774006,
+ "tokens_seen": 1725825024
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855566700100301,
+ "loss": 1.2693,
+ "theoretical_loss": 3.5018993065850657,
+ "tokens_seen": 1725890560
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855406218655968,
+ "loss": 1.2746,
+ "theoretical_loss": 3.5018879884428564,
+ "tokens_seen": 1725956096
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855245737211635,
+ "loss": 1.282,
+ "theoretical_loss": 3.501876670850726,
+ "tokens_seen": 1726021632
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003855085255767302,
+ "loss": 1.3583,
+ "theoretical_loss": 3.5018653538086264,
+ "tokens_seen": 1726087168
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003854924774322969,
+ "loss": 1.3132,
+ "theoretical_loss": 3.50185403731651,
+ "tokens_seen": 1726152704
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003854764292878636,
+ "loss": 1.3157,
+ "theoretical_loss": 3.50184272137433,
+ "tokens_seen": 1726218240
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003854603811434303,
+ "loss": 1.3115,
+ "theoretical_loss": 3.501831405982038,
+ "tokens_seen": 1726283776
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000385444332998997,
+ "loss": 1.1602,
+ "theoretical_loss": 3.501820091139586,
+ "tokens_seen": 1726349312
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003854282848545637,
+ "loss": 1.2871,
+ "theoretical_loss": 3.5018087768469277,
+ "tokens_seen": 1726414848
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003854122367101304,
+ "loss": 1.3348,
+ "theoretical_loss": 3.5017974631040145,
+ "tokens_seen": 1726480384
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853961885656971,
+ "loss": 1.3088,
+ "theoretical_loss": 3.5017861499107985,
+ "tokens_seen": 1726545920
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853801404212638,
+ "loss": 1.2737,
+ "theoretical_loss": 3.5017748372672335,
+ "tokens_seen": 1726611456
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853640922768305,
+ "loss": 1.2174,
+ "theoretical_loss": 3.501763525173271,
+ "tokens_seen": 1726676992
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853480441323972,
+ "loss": 1.2302,
+ "theoretical_loss": 3.5017522136288637,
+ "tokens_seen": 1726742528
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853319959879639,
+ "loss": 1.2866,
+ "theoretical_loss": 3.501740902633964,
+ "tokens_seen": 1726808064
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 578865,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2408570051193237,
+ "objective/train/theoretical_loss": 3.501729592188524,
+ "objective/train/tokens_used": 1747333600,
+ "theoretical_loss": 3.501729592188524,
+ "tokens_seen": 1726873600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003853159478435306,
+ "loss": 1.2528,
+ "theoretical_loss": 3.501729592188524,
+ "tokens_seen": 1726873600
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852998996990973,
+ "loss": 1.2242,
+ "theoretical_loss": 3.5017182822924964,
+ "tokens_seen": 1726939136
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000385283851554664,
+ "loss": 1.3313,
+ "theoretical_loss": 3.5017069729458337,
+ "tokens_seen": 1727004672
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852678034102307,
+ "loss": 1.3475,
+ "theoretical_loss": 3.501695664148489,
+ "tokens_seen": 1727070208
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852517552657974,
+ "loss": 1.2876,
+ "theoretical_loss": 3.501684355900414,
+ "tokens_seen": 1727135744
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852357071213641,
+ "loss": 1.2614,
+ "theoretical_loss": 3.5016730482015612,
+ "tokens_seen": 1727201280
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852196589769308,
+ "loss": 1.303,
+ "theoretical_loss": 3.5016617410518833,
+ "tokens_seen": 1727266816
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003852036108324975,
+ "loss": 1.2216,
+ "theoretical_loss": 3.5016504344513324,
+ "tokens_seen": 1727332352
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003851875626880642,
+ "loss": 1.3225,
+ "theoretical_loss": 3.5016391283998614,
+ "tokens_seen": 1727397888
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003851715145436309,
+ "loss": 1.3322,
+ "theoretical_loss": 3.501627822897423,
+ "tokens_seen": 1727463424
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003851554663991976,
+ "loss": 1.3411,
+ "theoretical_loss": 3.5016165179439698,
+ "tokens_seen": 1727528960
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003851394182547643,
+ "loss": 1.3269,
+ "theoretical_loss": 3.5016052135394533,
+ "tokens_seen": 1727594496
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000385123370110331,
+ "loss": 1.2784,
+ "theoretical_loss": 3.5015939096838267,
+ "tokens_seen": 1727660032
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003851073219658977,
+ "loss": 1.1286,
+ "theoretical_loss": 3.501582606377043,
+ "tokens_seen": 1727725568
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038509127382146437,
+ "loss": 1.1784,
+ "theoretical_loss": 3.501571303619054,
+ "tokens_seen": 1727791104
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038507522567703107,
+ "loss": 1.3183,
+ "theoretical_loss": 3.5015600014098127,
+ "tokens_seen": 1727856640
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038505917753259777,
+ "loss": 1.1559,
+ "theoretical_loss": 3.5015486997492715,
+ "tokens_seen": 1727922176
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038504312938816447,
+ "loss": 1.2435,
+ "theoretical_loss": 3.5015373986373826,
+ "tokens_seen": 1727987712
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038502708124373117,
+ "loss": 1.2908,
+ "theoretical_loss": 3.5015260980740988,
+ "tokens_seen": 1728053248
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038501103309929787,
+ "loss": 1.2526,
+ "theoretical_loss": 3.5015147980593726,
+ "tokens_seen": 1728118784
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038499498495486457,
+ "loss": 1.2501,
+ "theoretical_loss": 3.5015034985931575,
+ "tokens_seen": 1728184320
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003849789368104313,
+ "loss": 1.1863,
+ "theoretical_loss": 3.5014921996754045,
+ "tokens_seen": 1728249856
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000384962888665998,
+ "loss": 1.2358,
+ "theoretical_loss": 3.501480901306067,
+ "tokens_seen": 1728315392
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003849468405215647,
+ "loss": 1.2084,
+ "theoretical_loss": 3.5014696034850976,
+ "tokens_seen": 1728380928
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038493079237713143,
+ "loss": 1.255,
+ "theoretical_loss": 3.501458306212449,
+ "tokens_seen": 1728446464
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038491474423269813,
+ "loss": 1.3228,
+ "theoretical_loss": 3.5014470094880736,
+ "tokens_seen": 1728512000
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038489869608826483,
+ "loss": 1.214,
+ "theoretical_loss": 3.501435713311924,
+ "tokens_seen": 1728577536
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038488264794383153,
+ "loss": 1.3664,
+ "theoretical_loss": 3.501424417683953,
+ "tokens_seen": 1728643072
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038486659979939823,
+ "loss": 1.4088,
+ "theoretical_loss": 3.501413122604113,
+ "tokens_seen": 1728708608
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038485055165496493,
+ "loss": 1.2379,
+ "theoretical_loss": 3.501401828072356,
+ "tokens_seen": 1728774144
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038483450351053163,
+ "loss": 1.3197,
+ "theoretical_loss": 3.501390534088636,
+ "tokens_seen": 1728839680
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038481845536609833,
+ "loss": 1.3221,
+ "theoretical_loss": 3.5013792406529047,
+ "tokens_seen": 1728905216
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038480240722166503,
+ "loss": 1.3249,
+ "theoretical_loss": 3.5013679477651154,
+ "tokens_seen": 1728970752
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038478635907723173,
+ "loss": 1.3494,
+ "theoretical_loss": 3.5013566554252202,
+ "tokens_seen": 1729036288
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038477031093279843,
+ "loss": 1.2786,
+ "theoretical_loss": 3.5013453636331717,
+ "tokens_seen": 1729101824
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038475426278836513,
+ "loss": 1.3348,
+ "theoretical_loss": 3.501334072388923,
+ "tokens_seen": 1729167360
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038473821464393183,
+ "loss": 1.2591,
+ "theoretical_loss": 3.501322781692426,
+ "tokens_seen": 1729232896
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038472216649949854,
+ "loss": 1.4162,
+ "theoretical_loss": 3.5013114915436345,
+ "tokens_seen": 1729298432
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038470611835506524,
+ "loss": 1.3602,
+ "theoretical_loss": 3.5013002019424997,
+ "tokens_seen": 1729363968
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038469007021063194,
+ "loss": 1.2954,
+ "theoretical_loss": 3.5012889128889757,
+ "tokens_seen": 1729429504
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038467402206619864,
+ "loss": 1.2467,
+ "theoretical_loss": 3.5012776243830146,
+ "tokens_seen": 1729495040
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038465797392176534,
+ "loss": 1.3075,
+ "theoretical_loss": 3.5012663364245693,
+ "tokens_seen": 1729560576
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038464192577733204,
+ "loss": 1.2362,
+ "theoretical_loss": 3.5012550490135923,
+ "tokens_seen": 1729626112
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038462587763289874,
+ "loss": 1.351,
+ "theoretical_loss": 3.501243762150036,
+ "tokens_seen": 1729691648
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038460982948846544,
+ "loss": 1.2804,
+ "theoretical_loss": 3.501232475833854,
+ "tokens_seen": 1729757184
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038459378134403214,
+ "loss": 1.3178,
+ "theoretical_loss": 3.5012211900649977,
+ "tokens_seen": 1729822720
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038457773319959884,
+ "loss": 1.2581,
+ "theoretical_loss": 3.501209904843421,
+ "tokens_seen": 1729888256
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038456168505516554,
+ "loss": 1.2255,
+ "theoretical_loss": 3.501198620169076,
+ "tokens_seen": 1729953792
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038454563691073224,
+ "loss": 1.2716,
+ "theoretical_loss": 3.501187336041916,
+ "tokens_seen": 1730019328
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038452958876629894,
+ "loss": 1.332,
+ "theoretical_loss": 3.501176052461893,
+ "tokens_seen": 1730084864
+ },
+ {
+ "epoch": 0.52,
+ "objective/train/docs_used": 579798,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.325954556465149,
+ "objective/train/theoretical_loss": 3.5011647694289607,
+ "objective/train/tokens_used": 1750610400,
+ "theoretical_loss": 3.5011647694289607,
+ "tokens_seen": 1730150400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038451354062186564,
+ "loss": 1.3125,
+ "theoretical_loss": 3.5011647694289607,
+ "tokens_seen": 1730150400
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038449749247743234,
+ "loss": 1.2824,
+ "theoretical_loss": 3.5011534869430703,
+ "tokens_seen": 1730215936
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038448144433299904,
+ "loss": 1.3664,
+ "theoretical_loss": 3.501142205004176,
+ "tokens_seen": 1730281472
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.00038446539618856575,
+ "loss": 1.2695,
+ "theoretical_loss": 3.5011309236122305,
+ "tokens_seen": 1730347008
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003844493480441324,
+ "loss": 1.2399,
+ "theoretical_loss": 3.5011196427671862,
+ "tokens_seen": 1730412544
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003844332998996991,
+ "loss": 1.185,
+ "theoretical_loss": 3.5011083624689956,
+ "tokens_seen": 1730478080
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003844172517552658,
+ "loss": 1.1577,
+ "theoretical_loss": 3.501097082717611,
+ "tokens_seen": 1730543616
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003844012036108325,
+ "loss": 1.3516,
+ "theoretical_loss": 3.501085803512987,
+ "tokens_seen": 1730609152
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003843851554663992,
+ "loss": 1.2924,
+ "theoretical_loss": 3.501074524855075,
+ "tokens_seen": 1730674688
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003843691073219659,
+ "loss": 1.3261,
+ "theoretical_loss": 3.5010632467438283,
+ "tokens_seen": 1730740224
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003843530591775326,
+ "loss": 1.191,
+ "theoretical_loss": 3.501051969179199,
+ "tokens_seen": 1730805760
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003843370110330993,
+ "loss": 1.2456,
+ "theoretical_loss": 3.5010406921611406,
+ "tokens_seen": 1730871296
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000384320962888666,
+ "loss": 1.2673,
+ "theoretical_loss": 3.501029415689606,
+ "tokens_seen": 1730936832
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003843049147442327,
+ "loss": 1.2671,
+ "theoretical_loss": 3.5010181397645472,
+ "tokens_seen": 1731002368
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842888665997994,
+ "loss": 1.2298,
+ "theoretical_loss": 3.5010068643859182,
+ "tokens_seen": 1731067904
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842728184553661,
+ "loss": 1.3162,
+ "theoretical_loss": 3.500995589553671,
+ "tokens_seen": 1731133440
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842567703109328,
+ "loss": 1.2446,
+ "theoretical_loss": 3.5009843152677584,
+ "tokens_seen": 1731198976
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842407221664995,
+ "loss": 1.2342,
+ "theoretical_loss": 3.500973041528134,
+ "tokens_seen": 1731264512
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842246740220662,
+ "loss": 1.2531,
+ "theoretical_loss": 3.50096176833475,
+ "tokens_seen": 1731330048
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003842086258776329,
+ "loss": 1.255,
+ "theoretical_loss": 3.5009504956875594,
+ "tokens_seen": 1731395584
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003841925777331996,
+ "loss": 1.2469,
+ "theoretical_loss": 3.500939223586515,
+ "tokens_seen": 1731461120
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003841765295887663,
+ "loss": 1.27,
+ "theoretical_loss": 3.50092795203157,
+ "tokens_seen": 1731526656
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000384160481444333,
+ "loss": 1.2564,
+ "theoretical_loss": 3.5009166810226766,
+ "tokens_seen": 1731592192
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003841444332998997,
+ "loss": 1.2429,
+ "theoretical_loss": 3.5009054105597883,
+ "tokens_seen": 1731657728
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003841283851554664,
+ "loss": 1.2264,
+ "theoretical_loss": 3.500894140642858,
+ "tokens_seen": 1731723264
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003841123370110331,
+ "loss": 1.1761,
+ "theoretical_loss": 3.5008828712718385,
+ "tokens_seen": 1731788800
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840962888665998,
+ "loss": 1.2289,
+ "theoretical_loss": 3.5008716024466824,
+ "tokens_seen": 1731854336
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840802407221665,
+ "loss": 1.3899,
+ "theoretical_loss": 3.500860334167343,
+ "tokens_seen": 1731919872
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840641925777332,
+ "loss": 1.2143,
+ "theoretical_loss": 3.500849066433773,
+ "tokens_seen": 1731985408
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840481444332999,
+ "loss": 1.2238,
+ "theoretical_loss": 3.5008377992459248,
+ "tokens_seen": 1732050944
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840320962888666,
+ "loss": 1.2567,
+ "theoretical_loss": 3.500826532603752,
+ "tokens_seen": 1732116480
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003840160481444333,
+ "loss": 1.2777,
+ "theoretical_loss": 3.5008152665072076,
+ "tokens_seen": 1732182016
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.000384,
+ "loss": 1.3072,
+ "theoretical_loss": 3.5008040009562444,
+ "tokens_seen": 1732247552
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003839839518555667,
+ "loss": 1.162,
+ "theoretical_loss": 3.500792735950815,
+ "tokens_seen": 1732313088
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003839679037111334,
+ "loss": 1.3281,
+ "theoretical_loss": 3.500781471490873,
+ "tokens_seen": 1732378624
+ },
+ {
+ "epoch": 0.52,
+ "learning_rate": 0.0003839518555667001,
+ "loss": 1.2472,
+ "theoretical_loss": 3.5007702075763705,
+ "tokens_seen": 1732444160
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003839358074222668,
+ "loss": 1.344,
+ "theoretical_loss": 3.500758944207261,
+ "tokens_seen": 1732509696
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003839197592778335,
+ "loss": 1.2549,
+ "theoretical_loss": 3.5007476813834977,
+ "tokens_seen": 1732575232
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003839037111334002,
+ "loss": 1.2809,
+ "theoretical_loss": 3.5007364191050327,
+ "tokens_seen": 1732640768
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003838876629889669,
+ "loss": 1.2261,
+ "theoretical_loss": 3.5007251573718197,
+ "tokens_seen": 1732706304
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003838716148445336,
+ "loss": 1.2648,
+ "theoretical_loss": 3.500713896183812,
+ "tokens_seen": 1732771840
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003838555667001003,
+ "loss": 1.2785,
+ "theoretical_loss": 3.5007026355409616,
+ "tokens_seen": 1732837376
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000383839518555667,
+ "loss": 1.339,
+ "theoretical_loss": 3.500691375443222,
+ "tokens_seen": 1732902912
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003838234704112337,
+ "loss": 1.1722,
+ "theoretical_loss": 3.500680115890546,
+ "tokens_seen": 1732968448
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003838074222668004,
+ "loss": 1.282,
+ "theoretical_loss": 3.5006688568828874,
+ "tokens_seen": 1733033984
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837913741223671,
+ "loss": 1.2174,
+ "theoretical_loss": 3.500657598420198,
+ "tokens_seen": 1733099520
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837753259779338,
+ "loss": 1.2379,
+ "theoretical_loss": 3.5006463405024313,
+ "tokens_seen": 1733165056
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837592778335005,
+ "loss": 1.3383,
+ "theoretical_loss": 3.5006350831295414,
+ "tokens_seen": 1733230592
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837432296890672,
+ "loss": 1.2755,
+ "theoretical_loss": 3.5006238263014793,
+ "tokens_seen": 1733296128
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837271815446339,
+ "loss": 1.3077,
+ "theoretical_loss": 3.5006125700181996,
+ "tokens_seen": 1733361664
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 580942,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1904053688049316,
+ "objective/train/theoretical_loss": 3.5006013142796544,
+ "objective/train/tokens_used": 1753887200,
+ "theoretical_loss": 3.5006013142796544,
+ "tokens_seen": 1733427200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003837111334002006,
+ "loss": 1.2738,
+ "theoretical_loss": 3.5006013142796544,
+ "tokens_seen": 1733427200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003836950852557673,
+ "loss": 1.2081,
+ "theoretical_loss": 3.5005900590857975,
+ "tokens_seen": 1733492736
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000383679037111334,
+ "loss": 1.2873,
+ "theoretical_loss": 3.5005788044365818,
+ "tokens_seen": 1733558272
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003836629889669007,
+ "loss": 1.2783,
+ "theoretical_loss": 3.5005675503319598,
+ "tokens_seen": 1733623808
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003836469408224674,
+ "loss": 1.151,
+ "theoretical_loss": 3.5005562967718853,
+ "tokens_seen": 1733689344
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003836308926780341,
+ "loss": 1.3054,
+ "theoretical_loss": 3.5005450437563104,
+ "tokens_seen": 1733754880
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003836148445336008,
+ "loss": 1.2496,
+ "theoretical_loss": 3.5005337912851893,
+ "tokens_seen": 1733820416
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003835987963891675,
+ "loss": 1.3286,
+ "theoretical_loss": 3.5005225393584745,
+ "tokens_seen": 1733885952
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003835827482447342,
+ "loss": 1.2651,
+ "theoretical_loss": 3.500511287976119,
+ "tokens_seen": 1733951488
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003835667001003009,
+ "loss": 1.1749,
+ "theoretical_loss": 3.5005000371380763,
+ "tokens_seen": 1734017024
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038355065195586763,
+ "loss": 1.3233,
+ "theoretical_loss": 3.500488786844299,
+ "tokens_seen": 1734082560
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038353460381143433,
+ "loss": 1.2113,
+ "theoretical_loss": 3.5004775370947403,
+ "tokens_seen": 1734148096
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038351855566700103,
+ "loss": 1.3278,
+ "theoretical_loss": 3.5004662878893535,
+ "tokens_seen": 1734213632
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038350250752256773,
+ "loss": 1.2055,
+ "theoretical_loss": 3.500455039228092,
+ "tokens_seen": 1734279168
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038348645937813443,
+ "loss": 1.3052,
+ "theoretical_loss": 3.500443791110908,
+ "tokens_seen": 1734344704
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038347041123370113,
+ "loss": 1.3052,
+ "theoretical_loss": 3.500432543537756,
+ "tokens_seen": 1734410240
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038345436308926783,
+ "loss": 1.321,
+ "theoretical_loss": 3.5004212965085877,
+ "tokens_seen": 1734475776
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038343831494483453,
+ "loss": 1.308,
+ "theoretical_loss": 3.5004100500233575,
+ "tokens_seen": 1734541312
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038342226680040123,
+ "loss": 1.3135,
+ "theoretical_loss": 3.500398804082017,
+ "tokens_seen": 1734606848
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038340621865596793,
+ "loss": 1.2704,
+ "theoretical_loss": 3.500387558684521,
+ "tokens_seen": 1734672384
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038339017051153463,
+ "loss": 1.3219,
+ "theoretical_loss": 3.500376313830822,
+ "tokens_seen": 1734737920
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038337412236710133,
+ "loss": 1.2833,
+ "theoretical_loss": 3.500365069520872,
+ "tokens_seen": 1734803456
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038335807422266804,
+ "loss": 1.1523,
+ "theoretical_loss": 3.500353825754626,
+ "tokens_seen": 1734868992
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038334202607823474,
+ "loss": 1.2502,
+ "theoretical_loss": 3.5003425825320367,
+ "tokens_seen": 1734934528
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038332597793380144,
+ "loss": 1.2206,
+ "theoretical_loss": 3.500331339853057,
+ "tokens_seen": 1735000064
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038330992978936814,
+ "loss": 1.2775,
+ "theoretical_loss": 3.5003200977176396,
+ "tokens_seen": 1735065600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038329388164493484,
+ "loss": 1.2141,
+ "theoretical_loss": 3.5003088561257387,
+ "tokens_seen": 1735131136
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038327783350050154,
+ "loss": 1.1558,
+ "theoretical_loss": 3.500297615077306,
+ "tokens_seen": 1735196672
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038326178535606824,
+ "loss": 1.4139,
+ "theoretical_loss": 3.5002863745722967,
+ "tokens_seen": 1735262208
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038324573721163494,
+ "loss": 1.2256,
+ "theoretical_loss": 3.5002751346106624,
+ "tokens_seen": 1735327744
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038322968906720164,
+ "loss": 1.2166,
+ "theoretical_loss": 3.5002638951923575,
+ "tokens_seen": 1735393280
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038321364092276834,
+ "loss": 1.2284,
+ "theoretical_loss": 3.5002526563173344,
+ "tokens_seen": 1735458816
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038319759277833504,
+ "loss": 1.2755,
+ "theoretical_loss": 3.500241417985546,
+ "tokens_seen": 1735524352
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038318154463390174,
+ "loss": 1.2979,
+ "theoretical_loss": 3.5002301801969464,
+ "tokens_seen": 1735589888
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038316549648946844,
+ "loss": 1.1886,
+ "theoretical_loss": 3.5002189429514887,
+ "tokens_seen": 1735655424
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038314944834503514,
+ "loss": 1.1812,
+ "theoretical_loss": 3.500207706249126,
+ "tokens_seen": 1735720960
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038313340020060184,
+ "loss": 1.3195,
+ "theoretical_loss": 3.5001964700898114,
+ "tokens_seen": 1735786496
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038311735205616854,
+ "loss": 1.1482,
+ "theoretical_loss": 3.500185234473498,
+ "tokens_seen": 1735852032
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038310130391173525,
+ "loss": 1.2689,
+ "theoretical_loss": 3.5001739994001397,
+ "tokens_seen": 1735917568
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038308525576730195,
+ "loss": 1.2669,
+ "theoretical_loss": 3.500162764869689,
+ "tokens_seen": 1735983104
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038306920762286865,
+ "loss": 1.3127,
+ "theoretical_loss": 3.5001515308821,
+ "tokens_seen": 1736048640
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003830531594784353,
+ "loss": 1.2754,
+ "theoretical_loss": 3.5001402974373246,
+ "tokens_seen": 1736114176
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000383037111334002,
+ "loss": 1.2392,
+ "theoretical_loss": 3.500129064535318,
+ "tokens_seen": 1736179712
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003830210631895687,
+ "loss": 1.2436,
+ "theoretical_loss": 3.500117832176032,
+ "tokens_seen": 1736245248
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003830050150451354,
+ "loss": 1.3301,
+ "theoretical_loss": 3.5001066003594206,
+ "tokens_seen": 1736310784
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829889669007021,
+ "loss": 1.2287,
+ "theoretical_loss": 3.500095369085437,
+ "tokens_seen": 1736376320
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829729187562688,
+ "loss": 1.1851,
+ "theoretical_loss": 3.500084138354034,
+ "tokens_seen": 1736441856
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829568706118355,
+ "loss": 1.3487,
+ "theoretical_loss": 3.5000729081651656,
+ "tokens_seen": 1736507392
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829408224674022,
+ "loss": 1.3696,
+ "theoretical_loss": 3.5000616785187844,
+ "tokens_seen": 1736572928
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829247743229689,
+ "loss": 1.2771,
+ "theoretical_loss": 3.5000504494148448,
+ "tokens_seen": 1736638464
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 581741,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3218003511428833,
+ "objective/train/theoretical_loss": 3.5000392208532984,
+ "objective/train/tokens_used": 1757164000,
+ "theoretical_loss": 3.5000392208532984,
+ "tokens_seen": 1736704000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003829087261785356,
+ "loss": 1.2328,
+ "theoretical_loss": 3.5000392208532984,
+ "tokens_seen": 1736704000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003828926780341023,
+ "loss": 1.2375,
+ "theoretical_loss": 3.5000279928341005,
+ "tokens_seen": 1736769536
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000382876629889669,
+ "loss": 1.2961,
+ "theoretical_loss": 3.500016765357204,
+ "tokens_seen": 1736835072
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003828605817452357,
+ "loss": 1.2916,
+ "theoretical_loss": 3.5000055384225606,
+ "tokens_seen": 1736900608
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003828445336008024,
+ "loss": 1.2144,
+ "theoretical_loss": 3.499994312030126,
+ "tokens_seen": 1736966144
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003828284854563691,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4999830861798515,
+ "tokens_seen": 1737031680
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003828124373119358,
+ "loss": 1.2562,
+ "theoretical_loss": 3.4999718608716917,
+ "tokens_seen": 1737097216
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827963891675025,
+ "loss": 1.3202,
+ "theoretical_loss": 3.4999606361056,
+ "tokens_seen": 1737162752
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827803410230692,
+ "loss": 1.1997,
+ "theoretical_loss": 3.499949411881529,
+ "tokens_seen": 1737228288
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827642928786359,
+ "loss": 1.1846,
+ "theoretical_loss": 3.4999381881994323,
+ "tokens_seen": 1737293824
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827482447342026,
+ "loss": 1.332,
+ "theoretical_loss": 3.499926965059264,
+ "tokens_seen": 1737359360
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827321965897693,
+ "loss": 1.3656,
+ "theoretical_loss": 3.4999157424609764,
+ "tokens_seen": 1737424896
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000382716148445336,
+ "loss": 1.2244,
+ "theoretical_loss": 3.499904520404524,
+ "tokens_seen": 1737490432
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003827001003009027,
+ "loss": 1.3175,
+ "theoretical_loss": 3.4998932988898597,
+ "tokens_seen": 1737555968
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826840521564694,
+ "loss": 1.1929,
+ "theoretical_loss": 3.4998820779169364,
+ "tokens_seen": 1737621504
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826680040120361,
+ "loss": 1.2498,
+ "theoretical_loss": 3.4998708574857083,
+ "tokens_seen": 1737687040
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826519558676028,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4998596375961286,
+ "tokens_seen": 1737752576
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826359077231695,
+ "loss": 1.2156,
+ "theoretical_loss": 3.4998484182481504,
+ "tokens_seen": 1737818112
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826198595787362,
+ "loss": 1.1236,
+ "theoretical_loss": 3.4998371994417274,
+ "tokens_seen": 1737883648
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003826038114343029,
+ "loss": 1.3355,
+ "theoretical_loss": 3.499825981176813,
+ "tokens_seen": 1737949184
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003825877632898696,
+ "loss": 1.2855,
+ "theoretical_loss": 3.4998147634533607,
+ "tokens_seen": 1738014720
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038257171514543637,
+ "loss": 1.1721,
+ "theoretical_loss": 3.499803546271324,
+ "tokens_seen": 1738080256
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038255566700100307,
+ "loss": 1.3256,
+ "theoretical_loss": 3.499792329630657,
+ "tokens_seen": 1738145792
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038253961885656977,
+ "loss": 1.4245,
+ "theoretical_loss": 3.4997811135313115,
+ "tokens_seen": 1738211328
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038252357071213647,
+ "loss": 1.3257,
+ "theoretical_loss": 3.499769897973242,
+ "tokens_seen": 1738276864
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038250752256770317,
+ "loss": 1.2368,
+ "theoretical_loss": 3.4997586829564025,
+ "tokens_seen": 1738342400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038249147442326987,
+ "loss": 1.2761,
+ "theoretical_loss": 3.499747468480745,
+ "tokens_seen": 1738407936
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038247542627883657,
+ "loss": 1.1754,
+ "theoretical_loss": 3.4997362545462245,
+ "tokens_seen": 1738473472
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038245937813440327,
+ "loss": 1.1587,
+ "theoretical_loss": 3.499725041152793,
+ "tokens_seen": 1738539008
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038244332998996997,
+ "loss": 1.3838,
+ "theoretical_loss": 3.499713828300406,
+ "tokens_seen": 1738604544
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003824272818455366,
+ "loss": 1.2256,
+ "theoretical_loss": 3.4997026159890146,
+ "tokens_seen": 1738670080
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003824112337011033,
+ "loss": 1.2441,
+ "theoretical_loss": 3.4996914042185745,
+ "tokens_seen": 1738735616
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038239518555667,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4996801929890378,
+ "tokens_seen": 1738801152
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003823791374122367,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4996689823003586,
+ "tokens_seen": 1738866688
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003823630892678034,
+ "loss": 1.2692,
+ "theoretical_loss": 3.4996577721524904,
+ "tokens_seen": 1738932224
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003823470411233701,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4996465625453865,
+ "tokens_seen": 1738997760
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003823309929789368,
+ "loss": 1.301,
+ "theoretical_loss": 3.4996353534790003,
+ "tokens_seen": 1739063296
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003823149448345035,
+ "loss": 1.2231,
+ "theoretical_loss": 3.4996241449532857,
+ "tokens_seen": 1739128832
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822988966900702,
+ "loss": 1.2752,
+ "theoretical_loss": 3.4996129369681963,
+ "tokens_seen": 1739194368
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822828485456369,
+ "loss": 1.1965,
+ "theoretical_loss": 3.4996017295236856,
+ "tokens_seen": 1739259904
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822668004012036,
+ "loss": 1.1893,
+ "theoretical_loss": 3.499590522619707,
+ "tokens_seen": 1739325440
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822507522567703,
+ "loss": 1.2183,
+ "theoretical_loss": 3.499579316256214,
+ "tokens_seen": 1739390976
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000382234704112337,
+ "loss": 1.2983,
+ "theoretical_loss": 3.4995681104331604,
+ "tokens_seen": 1739456512
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822186559679037,
+ "loss": 1.2755,
+ "theoretical_loss": 3.4995569051504996,
+ "tokens_seen": 1739522048
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003822026078234704,
+ "loss": 1.1588,
+ "theoretical_loss": 3.499545700408185,
+ "tokens_seen": 1739587584
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038218655967903713,
+ "loss": 1.2943,
+ "theoretical_loss": 3.499534496206171,
+ "tokens_seen": 1739653120
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038217051153460383,
+ "loss": 1.2899,
+ "theoretical_loss": 3.49952329254441,
+ "tokens_seen": 1739718656
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038215446339017053,
+ "loss": 1.3464,
+ "theoretical_loss": 3.499512089422857,
+ "tokens_seen": 1739784192
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038213841524573723,
+ "loss": 1.2774,
+ "theoretical_loss": 3.4995008868414645,
+ "tokens_seen": 1739849728
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038212236710130393,
+ "loss": 1.334,
+ "theoretical_loss": 3.499489684800186,
+ "tokens_seen": 1739915264
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 583122,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2249778509140015,
+ "objective/train/theoretical_loss": 3.4994784832989763,
+ "objective/train/tokens_used": 1760440800,
+ "theoretical_loss": 3.4994784832989763,
+ "tokens_seen": 1739980800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038210631895687063,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4994784832989763,
+ "tokens_seen": 1739980800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038209027081243733,
+ "loss": 1.2504,
+ "theoretical_loss": 3.499467282337788,
+ "tokens_seen": 1740046336
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038207422266800403,
+ "loss": 1.1921,
+ "theoretical_loss": 3.499456081916575,
+ "tokens_seen": 1740111872
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038205817452357073,
+ "loss": 1.3552,
+ "theoretical_loss": 3.4994448820352906,
+ "tokens_seen": 1740177408
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038204212637913743,
+ "loss": 1.1832,
+ "theoretical_loss": 3.4994336826938897,
+ "tokens_seen": 1740242944
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038202607823470413,
+ "loss": 1.2714,
+ "theoretical_loss": 3.499422483892324,
+ "tokens_seen": 1740308480
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038201003009027083,
+ "loss": 1.2785,
+ "theoretical_loss": 3.499411285630549,
+ "tokens_seen": 1740374016
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038199398194583753,
+ "loss": 1.2713,
+ "theoretical_loss": 3.499400087908517,
+ "tokens_seen": 1740439552
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038197793380140424,
+ "loss": 1.2904,
+ "theoretical_loss": 3.4993888907261823,
+ "tokens_seen": 1740505088
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038196188565697094,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4993776940834986,
+ "tokens_seen": 1740570624
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038194583751253764,
+ "loss": 1.2595,
+ "theoretical_loss": 3.4993664979804198,
+ "tokens_seen": 1740636160
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038192978936810434,
+ "loss": 1.2244,
+ "theoretical_loss": 3.4993553024168986,
+ "tokens_seen": 1740701696
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038191374122367104,
+ "loss": 1.3059,
+ "theoretical_loss": 3.49934410739289,
+ "tokens_seen": 1740767232
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038189769307923774,
+ "loss": 1.2428,
+ "theoretical_loss": 3.499332912908346,
+ "tokens_seen": 1740832768
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038189769307923774,
+ "loss": 1.169,
+ "theoretical_loss": 3.4993217189632224,
+ "tokens_seen": 1740898304
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038188164493480444,
+ "loss": 1.2514,
+ "theoretical_loss": 3.499310525557471,
+ "tokens_seen": 1740963840
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038186559679037114,
+ "loss": 1.2729,
+ "theoretical_loss": 3.499299332691047,
+ "tokens_seen": 1741029376
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038184954864593784,
+ "loss": 1.1428,
+ "theoretical_loss": 3.4992881403639027,
+ "tokens_seen": 1741094912
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038183350050150454,
+ "loss": 1.2252,
+ "theoretical_loss": 3.499276948575993,
+ "tokens_seen": 1741160448
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038181745235707124,
+ "loss": 1.1769,
+ "theoretical_loss": 3.499265757327271,
+ "tokens_seen": 1741225984
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038180140421263794,
+ "loss": 1.2602,
+ "theoretical_loss": 3.499254566617691,
+ "tokens_seen": 1741291520
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038178535606820464,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4992433764472057,
+ "tokens_seen": 1741357056
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038176930792377134,
+ "loss": 1.4109,
+ "theoretical_loss": 3.49923218681577,
+ "tokens_seen": 1741422592
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038175325977933804,
+ "loss": 1.2601,
+ "theoretical_loss": 3.499220997723337,
+ "tokens_seen": 1741488128
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038173721163490474,
+ "loss": 1.3341,
+ "theoretical_loss": 3.4992098091698605,
+ "tokens_seen": 1741553664
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038172116349047145,
+ "loss": 1.3085,
+ "theoretical_loss": 3.499198621155294,
+ "tokens_seen": 1741619200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038170511534603815,
+ "loss": 1.2852,
+ "theoretical_loss": 3.4991874336795923,
+ "tokens_seen": 1741684736
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038168906720160485,
+ "loss": 1.2356,
+ "theoretical_loss": 3.4991762467427083,
+ "tokens_seen": 1741750272
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038167301905717155,
+ "loss": 1.287,
+ "theoretical_loss": 3.4991650603445956,
+ "tokens_seen": 1741815808
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003816569709127382,
+ "loss": 1.3658,
+ "theoretical_loss": 3.4991538744852084,
+ "tokens_seen": 1741881344
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003816409227683049,
+ "loss": 1.3557,
+ "theoretical_loss": 3.499142689164501,
+ "tokens_seen": 1741946880
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003816248746238716,
+ "loss": 1.282,
+ "theoretical_loss": 3.499131504382426,
+ "tokens_seen": 1742012416
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003816088264794383,
+ "loss": 1.1449,
+ "theoretical_loss": 3.499120320138938,
+ "tokens_seen": 1742077952
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000381592778335005,
+ "loss": 1.2108,
+ "theoretical_loss": 3.499109136433991,
+ "tokens_seen": 1742143488
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003815767301905717,
+ "loss": 1.2244,
+ "theoretical_loss": 3.499097953267538,
+ "tokens_seen": 1742209024
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003815606820461384,
+ "loss": 1.1925,
+ "theoretical_loss": 3.4990867706395337,
+ "tokens_seen": 1742274560
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003815446339017051,
+ "loss": 1.306,
+ "theoretical_loss": 3.499075588549931,
+ "tokens_seen": 1742340096
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003815285857572718,
+ "loss": 1.1186,
+ "theoretical_loss": 3.4990644069986843,
+ "tokens_seen": 1742405632
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003815125376128385,
+ "loss": 1.2629,
+ "theoretical_loss": 3.4990532259857474,
+ "tokens_seen": 1742471168
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814964894684052,
+ "loss": 1.229,
+ "theoretical_loss": 3.4990420455110742,
+ "tokens_seen": 1742536704
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814804413239719,
+ "loss": 1.1506,
+ "theoretical_loss": 3.4990308655746185,
+ "tokens_seen": 1742602240
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814643931795386,
+ "loss": 1.2068,
+ "theoretical_loss": 3.4990196861763336,
+ "tokens_seen": 1742667776
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814483450351053,
+ "loss": 1.2987,
+ "theoretical_loss": 3.499008507316174,
+ "tokens_seen": 1742733312
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000381432296890672,
+ "loss": 1.2061,
+ "theoretical_loss": 3.4989973289940934,
+ "tokens_seen": 1742798848
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814162487462387,
+ "loss": 1.1686,
+ "theoretical_loss": 3.4989861512100457,
+ "tokens_seen": 1742864384
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003814002006018054,
+ "loss": 1.3715,
+ "theoretical_loss": 3.498974973963985,
+ "tokens_seen": 1742929920
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003813841524573721,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4989637972558647,
+ "tokens_seen": 1742995456
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038136810431293886,
+ "loss": 1.229,
+ "theoretical_loss": 3.498952621085639,
+ "tokens_seen": 1743060992
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038135205616850556,
+ "loss": 1.2645,
+ "theoretical_loss": 3.498941445453262,
+ "tokens_seen": 1743126528
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038133600802407226,
+ "loss": 1.272,
+ "theoretical_loss": 3.4989302703586866,
+ "tokens_seen": 1743192064
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 584079,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4145455360412598,
+ "objective/train/theoretical_loss": 3.4989190958018677,
+ "objective/train/tokens_used": 1763717600,
+ "theoretical_loss": 3.4989190958018677,
+ "tokens_seen": 1743257600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038131995987963896,
+ "loss": 1.3625,
+ "theoretical_loss": 3.4989190958018677,
+ "tokens_seen": 1743257600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038130391173520566,
+ "loss": 1.1762,
+ "theoretical_loss": 3.498907921782759,
+ "tokens_seen": 1743323136
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038128786359077236,
+ "loss": 1.3146,
+ "theoretical_loss": 3.4988967483013145,
+ "tokens_seen": 1743388672
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038127181544633906,
+ "loss": 1.3495,
+ "theoretical_loss": 3.4988855753574875,
+ "tokens_seen": 1743454208
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038125576730190576,
+ "loss": 1.3109,
+ "theoretical_loss": 3.498874402951232,
+ "tokens_seen": 1743519744
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038123971915747246,
+ "loss": 1.3115,
+ "theoretical_loss": 3.4988632310825034,
+ "tokens_seen": 1743585280
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038122367101303917,
+ "loss": 1.3352,
+ "theoretical_loss": 3.498852059751254,
+ "tokens_seen": 1743650816
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038120762286860587,
+ "loss": 1.3012,
+ "theoretical_loss": 3.498840888957438,
+ "tokens_seen": 1743716352
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038119157472417257,
+ "loss": 1.3218,
+ "theoretical_loss": 3.4988297187010096,
+ "tokens_seen": 1743781888
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038117552657973927,
+ "loss": 1.4101,
+ "theoretical_loss": 3.498818548981923,
+ "tokens_seen": 1743847424
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038115947843530597,
+ "loss": 1.2624,
+ "theoretical_loss": 3.498807379800132,
+ "tokens_seen": 1743912960
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038114343029087267,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4987962111555904,
+ "tokens_seen": 1743978496
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038112738214643937,
+ "loss": 1.3151,
+ "theoretical_loss": 3.498785043048252,
+ "tokens_seen": 1744044032
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038111133400200607,
+ "loss": 1.3446,
+ "theoretical_loss": 3.4987738754780713,
+ "tokens_seen": 1744109568
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038109528585757277,
+ "loss": 1.3277,
+ "theoretical_loss": 3.498762708445002,
+ "tokens_seen": 1744175104
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038107923771313947,
+ "loss": 1.2807,
+ "theoretical_loss": 3.498751541948998,
+ "tokens_seen": 1744240640
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038106318956870617,
+ "loss": 1.2471,
+ "theoretical_loss": 3.4987403759900135,
+ "tokens_seen": 1744306176
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038104714142427287,
+ "loss": 1.253,
+ "theoretical_loss": 3.498729210568002,
+ "tokens_seen": 1744371712
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003810310932798396,
+ "loss": 1.1891,
+ "theoretical_loss": 3.4987180456829186,
+ "tokens_seen": 1744437248
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003810150451354062,
+ "loss": 1.2817,
+ "theoretical_loss": 3.498706881334716,
+ "tokens_seen": 1744502784
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809989969909729,
+ "loss": 1.2293,
+ "theoretical_loss": 3.4986957175233493,
+ "tokens_seen": 1744568320
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809829488465396,
+ "loss": 1.1663,
+ "theoretical_loss": 3.498684554248772,
+ "tokens_seen": 1744633856
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809669007021063,
+ "loss": 1.2083,
+ "theoretical_loss": 3.4986733915109376,
+ "tokens_seen": 1744699392
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000380950852557673,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4986622293098013,
+ "tokens_seen": 1744764928
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809348044132397,
+ "loss": 1.2886,
+ "theoretical_loss": 3.498651067645316,
+ "tokens_seen": 1744830464
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809187562688064,
+ "loss": 1.2427,
+ "theoretical_loss": 3.498639906517437,
+ "tokens_seen": 1744896000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003809027081243731,
+ "loss": 1.2386,
+ "theoretical_loss": 3.4986287459261174,
+ "tokens_seen": 1744961536
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003808866599799398,
+ "loss": 1.2453,
+ "theoretical_loss": 3.498617585871311,
+ "tokens_seen": 1745027072
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003808706118355065,
+ "loss": 1.1829,
+ "theoretical_loss": 3.498606426352973,
+ "tokens_seen": 1745092608
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003808545636910732,
+ "loss": 1.2011,
+ "theoretical_loss": 3.4985952673710567,
+ "tokens_seen": 1745158144
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003808385155466399,
+ "loss": 1.3406,
+ "theoretical_loss": 3.498584108925516,
+ "tokens_seen": 1745223680
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003808224674022066,
+ "loss": 1.1948,
+ "theoretical_loss": 3.498572951016305,
+ "tokens_seen": 1745289216
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038080641925777333,
+ "loss": 1.2913,
+ "theoretical_loss": 3.498561793643379,
+ "tokens_seen": 1745354752
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038079037111334003,
+ "loss": 1.243,
+ "theoretical_loss": 3.4985506368066903,
+ "tokens_seen": 1745420288
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038077432296890673,
+ "loss": 1.2639,
+ "theoretical_loss": 3.498539480506194,
+ "tokens_seen": 1745485824
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038075827482447343,
+ "loss": 1.2929,
+ "theoretical_loss": 3.4985283247418444,
+ "tokens_seen": 1745551360
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038074222668004013,
+ "loss": 1.2458,
+ "theoretical_loss": 3.498517169513595,
+ "tokens_seen": 1745616896
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038072617853560683,
+ "loss": 1.2286,
+ "theoretical_loss": 3.4985060148214,
+ "tokens_seen": 1745682432
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038071013039117353,
+ "loss": 1.1307,
+ "theoretical_loss": 3.4984948606652138,
+ "tokens_seen": 1745747968
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038069408224674023,
+ "loss": 1.2929,
+ "theoretical_loss": 3.4984837070449903,
+ "tokens_seen": 1745813504
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038067803410230693,
+ "loss": 1.2665,
+ "theoretical_loss": 3.498472553960684,
+ "tokens_seen": 1745879040
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038066198595787363,
+ "loss": 1.2547,
+ "theoretical_loss": 3.498461401412248,
+ "tokens_seen": 1745944576
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038064593781344033,
+ "loss": 1.2641,
+ "theoretical_loss": 3.4984502493996383,
+ "tokens_seen": 1746010112
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038062988966900703,
+ "loss": 1.212,
+ "theoretical_loss": 3.498439097922807,
+ "tokens_seen": 1746075648
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038061384152457374,
+ "loss": 1.2216,
+ "theoretical_loss": 3.49842794698171,
+ "tokens_seen": 1746141184
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038059779338014044,
+ "loss": 1.2021,
+ "theoretical_loss": 3.4984167965763002,
+ "tokens_seen": 1746206720
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038058174523570714,
+ "loss": 1.2376,
+ "theoretical_loss": 3.4984056467065323,
+ "tokens_seen": 1746272256
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038056569709127384,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4983944973723604,
+ "tokens_seen": 1746337792
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038054964894684054,
+ "loss": 1.2343,
+ "theoretical_loss": 3.4983833485737383,
+ "tokens_seen": 1746403328
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038053360080240724,
+ "loss": 1.2116,
+ "theoretical_loss": 3.4983722003106212,
+ "tokens_seen": 1746468864
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 585249,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2488512992858887,
+ "objective/train/theoretical_loss": 3.498361052582962,
+ "objective/train/tokens_used": 1766994400,
+ "theoretical_loss": 3.498361052582962,
+ "tokens_seen": 1746534400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038051755265797394,
+ "loss": 1.2671,
+ "theoretical_loss": 3.498361052582962,
+ "tokens_seen": 1746534400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038050150451354064,
+ "loss": 1.2579,
+ "theoretical_loss": 3.498349905390716,
+ "tokens_seen": 1746599936
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038048545636910734,
+ "loss": 1.3079,
+ "theoretical_loss": 3.4983387587338366,
+ "tokens_seen": 1746665472
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038046940822467404,
+ "loss": 1.3475,
+ "theoretical_loss": 3.4983276126122784,
+ "tokens_seen": 1746731008
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038045336008024074,
+ "loss": 1.1422,
+ "theoretical_loss": 3.4983164670259956,
+ "tokens_seen": 1746796544
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038043731193580744,
+ "loss": 1.3277,
+ "theoretical_loss": 3.498305321974942,
+ "tokens_seen": 1746862080
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038042126379137414,
+ "loss": 1.3043,
+ "theoretical_loss": 3.498294177459073,
+ "tokens_seen": 1746927616
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038040521564694084,
+ "loss": 1.1626,
+ "theoretical_loss": 3.498283033478341,
+ "tokens_seen": 1746993152
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038038916750250754,
+ "loss": 1.2902,
+ "theoretical_loss": 3.4982718900327017,
+ "tokens_seen": 1747058688
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038037311935807424,
+ "loss": 1.2238,
+ "theoretical_loss": 3.4982607471221083,
+ "tokens_seen": 1747124224
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038035707121364095,
+ "loss": 1.3565,
+ "theoretical_loss": 3.4982496047465164,
+ "tokens_seen": 1747189760
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038034102306920765,
+ "loss": 1.1619,
+ "theoretical_loss": 3.4982384629058787,
+ "tokens_seen": 1747255296
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038032497492477435,
+ "loss": 1.2325,
+ "theoretical_loss": 3.498227321600151,
+ "tokens_seen": 1747320832
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038030892678034105,
+ "loss": 1.3565,
+ "theoretical_loss": 3.498216180829286,
+ "tokens_seen": 1747386368
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038029287863590775,
+ "loss": 1.1512,
+ "theoretical_loss": 3.498205040593239,
+ "tokens_seen": 1747451904
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038027683049147445,
+ "loss": 1.2248,
+ "theoretical_loss": 3.498193900891964,
+ "tokens_seen": 1747517440
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038026078234704115,
+ "loss": 1.2392,
+ "theoretical_loss": 3.498182761725415,
+ "tokens_seen": 1747582976
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003802447342026078,
+ "loss": 1.368,
+ "theoretical_loss": 3.4981716230935467,
+ "tokens_seen": 1747648512
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003802286860581745,
+ "loss": 1.2008,
+ "theoretical_loss": 3.4981604849963133,
+ "tokens_seen": 1747714048
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003802126379137412,
+ "loss": 1.3921,
+ "theoretical_loss": 3.498149347433669,
+ "tokens_seen": 1747779584
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003801965897693079,
+ "loss": 1.3668,
+ "theoretical_loss": 3.498138210405568,
+ "tokens_seen": 1747845120
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003801805416248746,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4981270739119648,
+ "tokens_seen": 1747910656
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038016449348044135,
+ "loss": 1.1999,
+ "theoretical_loss": 3.4981159379528135,
+ "tokens_seen": 1747976192
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038014844533600805,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4981048025280685,
+ "tokens_seen": 1748041728
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038013239719157475,
+ "loss": 1.2299,
+ "theoretical_loss": 3.4980936676376846,
+ "tokens_seen": 1748107264
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038011634904714145,
+ "loss": 1.3647,
+ "theoretical_loss": 3.498082533281615,
+ "tokens_seen": 1748172800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038010030090270816,
+ "loss": 1.2543,
+ "theoretical_loss": 3.498071399459815,
+ "tokens_seen": 1748238336
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038008425275827486,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4980602661722386,
+ "tokens_seen": 1748303872
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038006820461384156,
+ "loss": 1.3479,
+ "theoretical_loss": 3.4980491334188404,
+ "tokens_seen": 1748369408
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038005215646940826,
+ "loss": 1.2316,
+ "theoretical_loss": 3.498038001199574,
+ "tokens_seen": 1748434944
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038003610832497496,
+ "loss": 1.3637,
+ "theoretical_loss": 3.498026869514395,
+ "tokens_seen": 1748500480
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038002006018054166,
+ "loss": 1.2802,
+ "theoretical_loss": 3.4980157383632564,
+ "tokens_seen": 1748566016
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00038000401203610836,
+ "loss": 1.1717,
+ "theoretical_loss": 3.4980046077461138,
+ "tokens_seen": 1748631552
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037998796389167506,
+ "loss": 1.1947,
+ "theoretical_loss": 3.497993477662921,
+ "tokens_seen": 1748697088
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037997191574724176,
+ "loss": 1.2212,
+ "theoretical_loss": 3.497982348113632,
+ "tokens_seen": 1748762624
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037995586760280846,
+ "loss": 1.2998,
+ "theoretical_loss": 3.4979712190982015,
+ "tokens_seen": 1748828160
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037993981945837516,
+ "loss": 1.265,
+ "theoretical_loss": 3.4979600906165844,
+ "tokens_seen": 1748893696
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037992377131394186,
+ "loss": 1.2805,
+ "theoretical_loss": 3.497948962668734,
+ "tokens_seen": 1748959232
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037990772316950856,
+ "loss": 1.2182,
+ "theoretical_loss": 3.4979378352546058,
+ "tokens_seen": 1749024768
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037989167502507526,
+ "loss": 1.2204,
+ "theoretical_loss": 3.4979267083741536,
+ "tokens_seen": 1749090304
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037987562688064196,
+ "loss": 1.3641,
+ "theoretical_loss": 3.497915582027332,
+ "tokens_seen": 1749155840
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037985957873620867,
+ "loss": 1.3524,
+ "theoretical_loss": 3.497904456214095,
+ "tokens_seen": 1749221376
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037984353059177537,
+ "loss": 1.178,
+ "theoretical_loss": 3.4978933309343976,
+ "tokens_seen": 1749286912
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037982748244734207,
+ "loss": 1.1753,
+ "theoretical_loss": 3.4978822061881942,
+ "tokens_seen": 1749352448
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037981143430290877,
+ "loss": 1.2949,
+ "theoretical_loss": 3.497871081975439,
+ "tokens_seen": 1749417984
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037979538615847547,
+ "loss": 1.216,
+ "theoretical_loss": 3.497859958296086,
+ "tokens_seen": 1749483520
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037977933801404217,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4978488351500907,
+ "tokens_seen": 1749549056
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037976328986960887,
+ "loss": 1.3318,
+ "theoretical_loss": 3.4978377125374065,
+ "tokens_seen": 1749614592
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037974724172517557,
+ "loss": 1.3575,
+ "theoretical_loss": 3.497826590457989,
+ "tokens_seen": 1749680128
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037973119358074227,
+ "loss": 1.2399,
+ "theoretical_loss": 3.4978154689117913,
+ "tokens_seen": 1749745664
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 586265,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2173155546188354,
+ "objective/train/theoretical_loss": 3.497804347898769,
+ "objective/train/tokens_used": 1770271200,
+ "theoretical_loss": 3.497804347898769,
+ "tokens_seen": 1749811200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037971514543630897,
+ "loss": 1.2606,
+ "theoretical_loss": 3.497804347898769,
+ "tokens_seen": 1749811200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037969909729187567,
+ "loss": 1.2775,
+ "theoretical_loss": 3.4977932274188754,
+ "tokens_seen": 1749876736
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037968304914744237,
+ "loss": 1.2137,
+ "theoretical_loss": 3.4977821074720663,
+ "tokens_seen": 1749942272
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037966700100300907,
+ "loss": 1.2222,
+ "theoretical_loss": 3.4977709880582957,
+ "tokens_seen": 1750007808
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003796509528585758,
+ "loss": 1.3053,
+ "theoretical_loss": 3.497759869177518,
+ "tokens_seen": 1750073344
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003796349047141425,
+ "loss": 1.2095,
+ "theoretical_loss": 3.497748750829687,
+ "tokens_seen": 1750138880
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003796188565697091,
+ "loss": 1.2132,
+ "theoretical_loss": 3.4977376330147583,
+ "tokens_seen": 1750204416
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003796028084252758,
+ "loss": 1.3074,
+ "theoretical_loss": 3.497726515732686,
+ "tokens_seen": 1750269952
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003795867602808425,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4977153989834244,
+ "tokens_seen": 1750335488
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003795707121364092,
+ "loss": 1.2309,
+ "theoretical_loss": 3.4977042827669282,
+ "tokens_seen": 1750401024
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003795546639919759,
+ "loss": 1.2332,
+ "theoretical_loss": 3.497693167083152,
+ "tokens_seen": 1750466560
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003795386158475426,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4976820519320504,
+ "tokens_seen": 1750532096
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003795225677031093,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4976709373135773,
+ "tokens_seen": 1750597632
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000379506519558676,
+ "loss": 1.2441,
+ "theoretical_loss": 3.4976598232276883,
+ "tokens_seen": 1750663168
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003794904714142427,
+ "loss": 1.3871,
+ "theoretical_loss": 3.497648709674337,
+ "tokens_seen": 1750728704
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003794744232698094,
+ "loss": 1.1394,
+ "theoretical_loss": 3.4976375966534787,
+ "tokens_seen": 1750794240
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003794583751253761,
+ "loss": 1.2654,
+ "theoretical_loss": 3.497626484165067,
+ "tokens_seen": 1750859776
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037944232698094283,
+ "loss": 1.3217,
+ "theoretical_loss": 3.497615372209057,
+ "tokens_seen": 1750925312
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037942627883650953,
+ "loss": 1.2868,
+ "theoretical_loss": 3.497604260785404,
+ "tokens_seen": 1750990848
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037941023069207623,
+ "loss": 1.3097,
+ "theoretical_loss": 3.4975931498940613,
+ "tokens_seen": 1751056384
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037939418254764293,
+ "loss": 1.2723,
+ "theoretical_loss": 3.497582039534984,
+ "tokens_seen": 1751121920
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037937813440320963,
+ "loss": 1.2866,
+ "theoretical_loss": 3.497570929708127,
+ "tokens_seen": 1751187456
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037936208625877633,
+ "loss": 1.3044,
+ "theoretical_loss": 3.4975598204134446,
+ "tokens_seen": 1751252992
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037934603811434303,
+ "loss": 1.2763,
+ "theoretical_loss": 3.497548711650891,
+ "tokens_seen": 1751318528
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037932998996990973,
+ "loss": 1.2528,
+ "theoretical_loss": 3.4975376034204215,
+ "tokens_seen": 1751384064
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037931394182547643,
+ "loss": 1.3591,
+ "theoretical_loss": 3.4975264957219907,
+ "tokens_seen": 1751449600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037929789368104313,
+ "loss": 1.3523,
+ "theoretical_loss": 3.497515388555553,
+ "tokens_seen": 1751515136
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037928184553660983,
+ "loss": 1.2446,
+ "theoretical_loss": 3.4975042819210627,
+ "tokens_seen": 1751580672
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037926579739217653,
+ "loss": 1.2758,
+ "theoretical_loss": 3.497493175818474,
+ "tokens_seen": 1751646208
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037924974924774323,
+ "loss": 1.4203,
+ "theoretical_loss": 3.497482070247743,
+ "tokens_seen": 1751711744
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037923370110330994,
+ "loss": 1.2685,
+ "theoretical_loss": 3.497470965208823,
+ "tokens_seen": 1751777280
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037921765295887664,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4974598607016696,
+ "tokens_seen": 1751842816
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037920160481444334,
+ "loss": 1.3125,
+ "theoretical_loss": 3.4974487567262367,
+ "tokens_seen": 1751908352
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037918555667001004,
+ "loss": 1.2584,
+ "theoretical_loss": 3.4974376532824794,
+ "tokens_seen": 1751973888
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037916950852557674,
+ "loss": 1.225,
+ "theoretical_loss": 3.4974265503703523,
+ "tokens_seen": 1752039424
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037915346038114344,
+ "loss": 1.2427,
+ "theoretical_loss": 3.49741544798981,
+ "tokens_seen": 1752104960
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037913741223671014,
+ "loss": 1.3059,
+ "theoretical_loss": 3.497404346140807,
+ "tokens_seen": 1752170496
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037912136409227684,
+ "loss": 1.2717,
+ "theoretical_loss": 3.4973932448232983,
+ "tokens_seen": 1752236032
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037910531594784354,
+ "loss": 1.3706,
+ "theoretical_loss": 3.497382144037238,
+ "tokens_seen": 1752301568
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037908926780341024,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4973710437825813,
+ "tokens_seen": 1752367104
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037907321965897694,
+ "loss": 1.3386,
+ "theoretical_loss": 3.4973599440592826,
+ "tokens_seen": 1752432640
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037905717151454364,
+ "loss": 1.2317,
+ "theoretical_loss": 3.497348844867297,
+ "tokens_seen": 1752498176
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037904112337011034,
+ "loss": 1.344,
+ "theoretical_loss": 3.497337746206579,
+ "tokens_seen": 1752563712
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037902507522567704,
+ "loss": 1.2787,
+ "theoretical_loss": 3.4973266480770837,
+ "tokens_seen": 1752629248
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037900902708124374,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4973155504787643,
+ "tokens_seen": 1752694784
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037899297893681045,
+ "loss": 1.3797,
+ "theoretical_loss": 3.4973044534115774,
+ "tokens_seen": 1752760320
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037897693079237715,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4972933568754767,
+ "tokens_seen": 1752825856
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037896088264794385,
+ "loss": 1.2052,
+ "theoretical_loss": 3.497282260870417,
+ "tokens_seen": 1752891392
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037894483450351055,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4972711653963535,
+ "tokens_seen": 1752956928
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037892878635907725,
+ "loss": 1.2338,
+ "theoretical_loss": 3.4972600704532404,
+ "tokens_seen": 1753022464
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 587536,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2830181121826172,
+ "objective/train/theoretical_loss": 3.497248976041033,
+ "objective/train/tokens_used": 1773548000,
+ "theoretical_loss": 3.497248976041033,
+ "tokens_seen": 1753088000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037891273821464395,
+ "loss": 1.3373,
+ "theoretical_loss": 3.497248976041033,
+ "tokens_seen": 1753088000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037889669007021065,
+ "loss": 1.3406,
+ "theoretical_loss": 3.497237882159685,
+ "tokens_seen": 1753153536
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037888064192577735,
+ "loss": 1.3413,
+ "theoretical_loss": 3.4972267888091526,
+ "tokens_seen": 1753219072
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037886459378134405,
+ "loss": 1.3309,
+ "theoretical_loss": 3.4972156959893894,
+ "tokens_seen": 1753284608
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037884854563691075,
+ "loss": 1.3091,
+ "theoretical_loss": 3.49720460370035,
+ "tokens_seen": 1753350144
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037883249749247745,
+ "loss": 1.3314,
+ "theoretical_loss": 3.497193511941991,
+ "tokens_seen": 1753415680
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037881644934804415,
+ "loss": 1.3395,
+ "theoretical_loss": 3.4971824207142657,
+ "tokens_seen": 1753481216
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037880040120361085,
+ "loss": 1.3523,
+ "theoretical_loss": 3.497171330017128,
+ "tokens_seen": 1753546752
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037878435305917755,
+ "loss": 1.338,
+ "theoretical_loss": 3.497160239850535,
+ "tokens_seen": 1753612288
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037876830491474425,
+ "loss": 1.2973,
+ "theoretical_loss": 3.49714915021444,
+ "tokens_seen": 1753677824
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037875225677031095,
+ "loss": 1.2021,
+ "theoretical_loss": 3.4971380611087977,
+ "tokens_seen": 1753743360
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037873620862587766,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4971269725335636,
+ "tokens_seen": 1753808896
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037872016048144436,
+ "loss": 1.3104,
+ "theoretical_loss": 3.4971158844886925,
+ "tokens_seen": 1753874432
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037870411233701106,
+ "loss": 1.3159,
+ "theoretical_loss": 3.4971047969741385,
+ "tokens_seen": 1753939968
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037868806419257776,
+ "loss": 1.3875,
+ "theoretical_loss": 3.4970937099898576,
+ "tokens_seen": 1754005504
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037867201604814446,
+ "loss": 1.3549,
+ "theoretical_loss": 3.4970826235358032,
+ "tokens_seen": 1754071040
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037865596790371116,
+ "loss": 1.2536,
+ "theoretical_loss": 3.497071537611931,
+ "tokens_seen": 1754136576
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037863991975927786,
+ "loss": 1.195,
+ "theoretical_loss": 3.4970604522181956,
+ "tokens_seen": 1754202112
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037862387161484456,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4970493673545517,
+ "tokens_seen": 1754267648
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037860782347041126,
+ "loss": 1.234,
+ "theoretical_loss": 3.497038283020955,
+ "tokens_seen": 1754333184
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037859177532597796,
+ "loss": 1.2261,
+ "theoretical_loss": 3.497027199217359,
+ "tokens_seen": 1754398720
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037857572718154466,
+ "loss": 1.19,
+ "theoretical_loss": 3.49701611594372,
+ "tokens_seen": 1754464256
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037855967903711136,
+ "loss": 1.2683,
+ "theoretical_loss": 3.497005033199992,
+ "tokens_seen": 1754529792
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037854363089267806,
+ "loss": 1.3002,
+ "theoretical_loss": 3.496993950986129,
+ "tokens_seen": 1754595328
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037852758274824476,
+ "loss": 1.3702,
+ "theoretical_loss": 3.496982869302088,
+ "tokens_seen": 1754660864
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037851153460381146,
+ "loss": 1.3319,
+ "theoretical_loss": 3.496971788147822,
+ "tokens_seen": 1754726400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037849548645937816,
+ "loss": 1.3352,
+ "theoretical_loss": 3.4969607075232876,
+ "tokens_seen": 1754791936
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037847943831494487,
+ "loss": 1.2652,
+ "theoretical_loss": 3.496949627428438,
+ "tokens_seen": 1754857472
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037846339017051157,
+ "loss": 1.3375,
+ "theoretical_loss": 3.4969385478632287,
+ "tokens_seen": 1754923008
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037844734202607827,
+ "loss": 1.3323,
+ "theoretical_loss": 3.4969274688276153,
+ "tokens_seen": 1754988544
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037843129388164497,
+ "loss": 1.2569,
+ "theoretical_loss": 3.496916390321552,
+ "tokens_seen": 1755054080
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037841524573721167,
+ "loss": 1.3081,
+ "theoretical_loss": 3.4969053123449942,
+ "tokens_seen": 1755119616
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037839919759277837,
+ "loss": 1.322,
+ "theoretical_loss": 3.496894234897896,
+ "tokens_seen": 1755185152
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037838314944834507,
+ "loss": 1.292,
+ "theoretical_loss": 3.4968831579802133,
+ "tokens_seen": 1755250688
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037836710130391177,
+ "loss": 1.3255,
+ "theoretical_loss": 3.4968720815919,
+ "tokens_seen": 1755316224
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037835105315947847,
+ "loss": 1.2901,
+ "theoretical_loss": 3.496861005732912,
+ "tokens_seen": 1755381760
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037833500501504517,
+ "loss": 1.3479,
+ "theoretical_loss": 3.4968499304032035,
+ "tokens_seen": 1755447296
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037831895687061187,
+ "loss": 1.118,
+ "theoretical_loss": 3.4968388556027303,
+ "tokens_seen": 1755512832
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037830290872617857,
+ "loss": 1.3388,
+ "theoretical_loss": 3.4968277813314463,
+ "tokens_seen": 1755578368
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003782868605817453,
+ "loss": 1.2706,
+ "theoretical_loss": 3.4968167075893075,
+ "tokens_seen": 1755643904
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000378270812437312,
+ "loss": 1.2414,
+ "theoretical_loss": 3.496805634376268,
+ "tokens_seen": 1755709440
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003782547642928787,
+ "loss": 1.4021,
+ "theoretical_loss": 3.4967945616922838,
+ "tokens_seen": 1755774976
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003782387161484454,
+ "loss": 1.3017,
+ "theoretical_loss": 3.4967834895373087,
+ "tokens_seen": 1755840512
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003782226680040121,
+ "loss": 1.2743,
+ "theoretical_loss": 3.4967724179112984,
+ "tokens_seen": 1755906048
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003782066198595787,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4967613468142074,
+ "tokens_seen": 1755971584
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781905717151454,
+ "loss": 1.2562,
+ "theoretical_loss": 3.496750276245991,
+ "tokens_seen": 1756037120
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781745235707121,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4967392062066045,
+ "tokens_seen": 1756102656
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781584754262788,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4967281366960026,
+ "tokens_seen": 1756168192
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781424272818455,
+ "loss": 1.2646,
+ "theoretical_loss": 3.4967170677141404,
+ "tokens_seen": 1756233728
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781263791374122,
+ "loss": 1.1734,
+ "theoretical_loss": 3.496705999260972,
+ "tokens_seen": 1756299264
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 588499,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2977731227874756,
+ "objective/train/theoretical_loss": 3.496694931336454,
+ "objective/train/tokens_used": 1776824800,
+ "theoretical_loss": 3.496694931336454,
+ "tokens_seen": 1756364800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003781103309929789,
+ "loss": 1.3492,
+ "theoretical_loss": 3.496694931336454,
+ "tokens_seen": 1756364800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003780942828485456,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4966838639405404,
+ "tokens_seen": 1756430336
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003780782347041123,
+ "loss": 1.3504,
+ "theoretical_loss": 3.496672797073187,
+ "tokens_seen": 1756495872
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037806218655967903,
+ "loss": 1.2986,
+ "theoretical_loss": 3.4966617307343477,
+ "tokens_seen": 1756561408
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037804613841524573,
+ "loss": 1.2992,
+ "theoretical_loss": 3.4966506649239784,
+ "tokens_seen": 1756626944
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037803009027081243,
+ "loss": 1.2713,
+ "theoretical_loss": 3.496639599642034,
+ "tokens_seen": 1756692480
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037801404212637913,
+ "loss": 1.3161,
+ "theoretical_loss": 3.4966285348884694,
+ "tokens_seen": 1756758016
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037799799398194583,
+ "loss": 1.3771,
+ "theoretical_loss": 3.4966174706632396,
+ "tokens_seen": 1756823552
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037798194583751253,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4966064069663,
+ "tokens_seen": 1756889088
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037796589769307923,
+ "loss": 1.2138,
+ "theoretical_loss": 3.4965953437976056,
+ "tokens_seen": 1756954624
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037794984954864593,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4965842811571113,
+ "tokens_seen": 1757020160
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037793380140421263,
+ "loss": 1.2421,
+ "theoretical_loss": 3.4965732190447723,
+ "tokens_seen": 1757085696
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037791775325977933,
+ "loss": 1.159,
+ "theoretical_loss": 3.496562157460543,
+ "tokens_seen": 1757151232
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037790170511534603,
+ "loss": 1.3358,
+ "theoretical_loss": 3.4965510964043798,
+ "tokens_seen": 1757216768
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037788565697091273,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4965400358762366,
+ "tokens_seen": 1757282304
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037786960882647944,
+ "loss": 1.3325,
+ "theoretical_loss": 3.4965289758760694,
+ "tokens_seen": 1757347840
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037785356068204614,
+ "loss": 1.3204,
+ "theoretical_loss": 3.4965179164038327,
+ "tokens_seen": 1757413376
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037783751253761284,
+ "loss": 1.2431,
+ "theoretical_loss": 3.496506857459482,
+ "tokens_seen": 1757478912
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037782146439317954,
+ "loss": 1.2312,
+ "theoretical_loss": 3.4964957990429726,
+ "tokens_seen": 1757544448
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037780541624874624,
+ "loss": 1.3462,
+ "theoretical_loss": 3.4964847411542586,
+ "tokens_seen": 1757609984
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037778936810431294,
+ "loss": 1.2712,
+ "theoretical_loss": 3.496473683793296,
+ "tokens_seen": 1757675520
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003777733199598797,
+ "loss": 1.3506,
+ "theoretical_loss": 3.49646262696004,
+ "tokens_seen": 1757741056
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003777572718154464,
+ "loss": 1.3224,
+ "theoretical_loss": 3.4964515706544455,
+ "tokens_seen": 1757806592
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003777412236710131,
+ "loss": 1.2802,
+ "theoretical_loss": 3.4964405148764675,
+ "tokens_seen": 1757872128
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003777251755265798,
+ "loss": 1.2428,
+ "theoretical_loss": 3.4964294596260608,
+ "tokens_seen": 1757937664
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003777091273821465,
+ "loss": 1.3147,
+ "theoretical_loss": 3.496418404903182,
+ "tokens_seen": 1758003200
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003776930792377132,
+ "loss": 1.1837,
+ "theoretical_loss": 3.496407350707785,
+ "tokens_seen": 1758068736
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003776770310932799,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4963962970398246,
+ "tokens_seen": 1758134272
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003776609829488466,
+ "loss": 1.3655,
+ "theoretical_loss": 3.4963852438992573,
+ "tokens_seen": 1758199808
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003776449348044133,
+ "loss": 1.169,
+ "theoretical_loss": 3.4963741912860375,
+ "tokens_seen": 1758265344
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037762888665998,
+ "loss": 1.1813,
+ "theoretical_loss": 3.4963631392001204,
+ "tokens_seen": 1758330880
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003776128385155467,
+ "loss": 1.2342,
+ "theoretical_loss": 3.496352087641462,
+ "tokens_seen": 1758396416
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003775967903711134,
+ "loss": 1.3225,
+ "theoretical_loss": 3.496341036610016,
+ "tokens_seen": 1758461952
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037758074222668005,
+ "loss": 1.3467,
+ "theoretical_loss": 3.4963299861057386,
+ "tokens_seen": 1758527488
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037756469408224675,
+ "loss": 1.2901,
+ "theoretical_loss": 3.496318936128585,
+ "tokens_seen": 1758593024
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037754864593781345,
+ "loss": 1.2335,
+ "theoretical_loss": 3.49630788667851,
+ "tokens_seen": 1758658560
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037753259779338015,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4962968377554695,
+ "tokens_seen": 1758724096
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037751654964894685,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4962857893594177,
+ "tokens_seen": 1758789632
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037750050150451355,
+ "loss": 1.3047,
+ "theoretical_loss": 3.4962747414903106,
+ "tokens_seen": 1758855168
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037748445336008025,
+ "loss": 1.3836,
+ "theoretical_loss": 3.4962636941481033,
+ "tokens_seen": 1758920704
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037746840521564695,
+ "loss": 1.3101,
+ "theoretical_loss": 3.496252647332751,
+ "tokens_seen": 1758986240
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037745235707121365,
+ "loss": 1.257,
+ "theoretical_loss": 3.496241601044209,
+ "tokens_seen": 1759051776
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037743630892678035,
+ "loss": 1.2424,
+ "theoretical_loss": 3.496230555282432,
+ "tokens_seen": 1759117312
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037742026078234705,
+ "loss": 1.2928,
+ "theoretical_loss": 3.4962195100473767,
+ "tokens_seen": 1759182848
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037740421263791375,
+ "loss": 1.333,
+ "theoretical_loss": 3.496208465338997,
+ "tokens_seen": 1759248384
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037738816449348045,
+ "loss": 1.1944,
+ "theoretical_loss": 3.496197421157248,
+ "tokens_seen": 1759313920
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037737211634904715,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4961863775020863,
+ "tokens_seen": 1759379456
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037735606820461386,
+ "loss": 1.3181,
+ "theoretical_loss": 3.496175334373466,
+ "tokens_seen": 1759444992
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037734002006018056,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4961642917713425,
+ "tokens_seen": 1759510528
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037732397191574726,
+ "loss": 1.2005,
+ "theoretical_loss": 3.4961532496956718,
+ "tokens_seen": 1759576064
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 589734,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2679030895233154,
+ "objective/train/theoretical_loss": 3.496142208146409,
+ "objective/train/tokens_used": 1780101600,
+ "theoretical_loss": 3.496142208146409,
+ "tokens_seen": 1759641600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037730792377131396,
+ "loss": 1.3017,
+ "theoretical_loss": 3.496142208146409,
+ "tokens_seen": 1759641600
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037729187562688066,
+ "loss": 1.1657,
+ "theoretical_loss": 3.496131167123509,
+ "tokens_seen": 1759707136
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037727582748244736,
+ "loss": 1.2008,
+ "theoretical_loss": 3.496120126626927,
+ "tokens_seen": 1759772672
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037725977933801406,
+ "loss": 1.1927,
+ "theoretical_loss": 3.4961090866566185,
+ "tokens_seen": 1759838208
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037724373119358076,
+ "loss": 1.4921,
+ "theoretical_loss": 3.4960980472125396,
+ "tokens_seen": 1759903744
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037722768304914746,
+ "loss": 1.3269,
+ "theoretical_loss": 3.4960870082946442,
+ "tokens_seen": 1759969280
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037721163490471416,
+ "loss": 1.2817,
+ "theoretical_loss": 3.496075969902889,
+ "tokens_seen": 1760034816
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037719558676028086,
+ "loss": 1.1871,
+ "theoretical_loss": 3.4960649320372283,
+ "tokens_seen": 1760100352
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037717953861584756,
+ "loss": 1.3425,
+ "theoretical_loss": 3.496053894697618,
+ "tokens_seen": 1760165888
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037716349047141426,
+ "loss": 1.224,
+ "theoretical_loss": 3.4960428578840137,
+ "tokens_seen": 1760231424
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037714744232698096,
+ "loss": 1.2625,
+ "theoretical_loss": 3.49603182159637,
+ "tokens_seen": 1760296960
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037713139418254766,
+ "loss": 1.3932,
+ "theoretical_loss": 3.4960207858346424,
+ "tokens_seen": 1760362496
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037711534603811437,
+ "loss": 1.2925,
+ "theoretical_loss": 3.4960097505987866,
+ "tokens_seen": 1760428032
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037709929789368107,
+ "loss": 1.2564,
+ "theoretical_loss": 3.4959987158887573,
+ "tokens_seen": 1760493568
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037708324974924777,
+ "loss": 1.2739,
+ "theoretical_loss": 3.495987681704511,
+ "tokens_seen": 1760559104
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037706720160481447,
+ "loss": 1.292,
+ "theoretical_loss": 3.4959766480460024,
+ "tokens_seen": 1760624640
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037705115346038117,
+ "loss": 1.3586,
+ "theoretical_loss": 3.495965614913187,
+ "tokens_seen": 1760690176
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037703510531594787,
+ "loss": 1.3021,
+ "theoretical_loss": 3.4959545823060196,
+ "tokens_seen": 1760755712
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037701905717151457,
+ "loss": 1.2686,
+ "theoretical_loss": 3.4959435502244567,
+ "tokens_seen": 1760821248
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037700300902708127,
+ "loss": 1.2292,
+ "theoretical_loss": 3.4959325186684533,
+ "tokens_seen": 1760886784
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037698696088264797,
+ "loss": 1.3947,
+ "theoretical_loss": 3.495921487637964,
+ "tokens_seen": 1760952320
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037697091273821467,
+ "loss": 1.285,
+ "theoretical_loss": 3.495910457132945,
+ "tokens_seen": 1761017856
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037695486459378137,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4958994271533523,
+ "tokens_seen": 1761083392
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037693881644934807,
+ "loss": 1.2148,
+ "theoretical_loss": 3.4958883976991397,
+ "tokens_seen": 1761148928
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037692276830491477,
+ "loss": 1.3451,
+ "theoretical_loss": 3.495877368770264,
+ "tokens_seen": 1761214464
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003769067201604815,
+ "loss": 1.1869,
+ "theoretical_loss": 3.49586634036668,
+ "tokens_seen": 1761280000
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003768906720160482,
+ "loss": 1.2616,
+ "theoretical_loss": 3.495855312488343,
+ "tokens_seen": 1761345536
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003768746238716149,
+ "loss": 1.2516,
+ "theoretical_loss": 3.4958442851352087,
+ "tokens_seen": 1761411072
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003768585757271816,
+ "loss": 1.3719,
+ "theoretical_loss": 3.4958332583072327,
+ "tokens_seen": 1761476608
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003768425275827483,
+ "loss": 1.3195,
+ "theoretical_loss": 3.495822232004371,
+ "tokens_seen": 1761542144
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000376826479438315,
+ "loss": 1.2961,
+ "theoretical_loss": 3.4958112062265774,
+ "tokens_seen": 1761607680
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003768104312938816,
+ "loss": 1.3393,
+ "theoretical_loss": 3.495800180973809,
+ "tokens_seen": 1761673216
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003767943831494483,
+ "loss": 1.3445,
+ "theoretical_loss": 3.49578915624602,
+ "tokens_seen": 1761738752
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000376778335005015,
+ "loss": 1.3252,
+ "theoretical_loss": 3.495778132043167,
+ "tokens_seen": 1761804288
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003767622868605817,
+ "loss": 1.2419,
+ "theoretical_loss": 3.4957671083652047,
+ "tokens_seen": 1761869824
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003767462387161484,
+ "loss": 1.4053,
+ "theoretical_loss": 3.495756085212089,
+ "tokens_seen": 1761935360
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003767301905717151,
+ "loss": 1.3721,
+ "theoretical_loss": 3.495745062583775,
+ "tokens_seen": 1762000896
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003767141424272818,
+ "loss": 1.2116,
+ "theoretical_loss": 3.495734040480219,
+ "tokens_seen": 1762066432
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037669809428284853,
+ "loss": 1.2582,
+ "theoretical_loss": 3.495723018901375,
+ "tokens_seen": 1762131968
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037668204613841523,
+ "loss": 1.3481,
+ "theoretical_loss": 3.4957119978472,
+ "tokens_seen": 1762197504
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037666599799398193,
+ "loss": 1.3263,
+ "theoretical_loss": 3.4957009773176493,
+ "tokens_seen": 1762263040
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037664994984954863,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4956899573126776,
+ "tokens_seen": 1762328576
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037663390170511533,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4956789378322406,
+ "tokens_seen": 1762394112
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037661785356068203,
+ "loss": 1.17,
+ "theoretical_loss": 3.4956679188762947,
+ "tokens_seen": 1762459648
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037660180541624873,
+ "loss": 1.3709,
+ "theoretical_loss": 3.4956569004447946,
+ "tokens_seen": 1762525184
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037658575727181543,
+ "loss": 1.3024,
+ "theoretical_loss": 3.495645882537696,
+ "tokens_seen": 1762590720
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003765697091273822,
+ "loss": 1.1902,
+ "theoretical_loss": 3.4956348651549547,
+ "tokens_seen": 1762656256
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003765536609829489,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4956238482965265,
+ "tokens_seen": 1762721792
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003765376128385156,
+ "loss": 1.2986,
+ "theoretical_loss": 3.495612831962366,
+ "tokens_seen": 1762787328
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003765215646940823,
+ "loss": 1.2204,
+ "theoretical_loss": 3.49560181615243,
+ "tokens_seen": 1762852864
+ },
+ {
+ "epoch": 0.53,
+ "objective/train/docs_used": 591063,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.269951343536377,
+ "objective/train/theoretical_loss": 3.4955908008666725,
+ "objective/train/tokens_used": 1783378400,
+ "theoretical_loss": 3.4955908008666725,
+ "tokens_seen": 1762918400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000376505516549649,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4955908008666725,
+ "tokens_seen": 1762918400
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764894684052157,
+ "loss": 1.3602,
+ "theoretical_loss": 3.4955797861050506,
+ "tokens_seen": 1762983936
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764734202607824,
+ "loss": 1.2533,
+ "theoretical_loss": 3.495568771867519,
+ "tokens_seen": 1763049472
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764573721163491,
+ "loss": 1.2646,
+ "theoretical_loss": 3.495557758154033,
+ "tokens_seen": 1763115008
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764413239719158,
+ "loss": 1.3023,
+ "theoretical_loss": 3.4955467449645496,
+ "tokens_seen": 1763180544
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764252758274825,
+ "loss": 1.3067,
+ "theoretical_loss": 3.4955357322990226,
+ "tokens_seen": 1763246080
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003764092276830492,
+ "loss": 1.315,
+ "theoretical_loss": 3.495524720157409,
+ "tokens_seen": 1763311616
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003763931795386159,
+ "loss": 1.3536,
+ "theoretical_loss": 3.495513708539664,
+ "tokens_seen": 1763377152
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003763771313941826,
+ "loss": 1.3975,
+ "theoretical_loss": 3.495502697445743,
+ "tokens_seen": 1763442688
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003763610832497493,
+ "loss": 1.1657,
+ "theoretical_loss": 3.495491686875602,
+ "tokens_seen": 1763508224
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.000376345035105316,
+ "loss": 1.2885,
+ "theoretical_loss": 3.495480676829196,
+ "tokens_seen": 1763573760
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003763289869608827,
+ "loss": 1.3532,
+ "theoretical_loss": 3.495469667306481,
+ "tokens_seen": 1763639296
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003763129388164494,
+ "loss": 1.3095,
+ "theoretical_loss": 3.495458658307413,
+ "tokens_seen": 1763704832
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762968906720161,
+ "loss": 1.2785,
+ "theoretical_loss": 3.495447649831947,
+ "tokens_seen": 1763770368
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762808425275828,
+ "loss": 1.2835,
+ "theoretical_loss": 3.4954366418800387,
+ "tokens_seen": 1763835904
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762647943831495,
+ "loss": 1.2842,
+ "theoretical_loss": 3.495425634451644,
+ "tokens_seen": 1763901440
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762487462387162,
+ "loss": 1.3379,
+ "theoretical_loss": 3.495414627546719,
+ "tokens_seen": 1763966976
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762326980942829,
+ "loss": 1.2861,
+ "theoretical_loss": 3.495403621165218,
+ "tokens_seen": 1764032512
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762166499498496,
+ "loss": 1.2767,
+ "theoretical_loss": 3.495392615307098,
+ "tokens_seen": 1764098048
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.0003762006018054163,
+ "loss": 1.3185,
+ "theoretical_loss": 3.495381609972314,
+ "tokens_seen": 1764163584
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037618455366098295,
+ "loss": 1.2642,
+ "theoretical_loss": 3.495370605160822,
+ "tokens_seen": 1764229120
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037616850551654965,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4953596008725776,
+ "tokens_seen": 1764294656
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037615245737211635,
+ "loss": 1.2559,
+ "theoretical_loss": 3.4953485971075366,
+ "tokens_seen": 1764360192
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037613640922768305,
+ "loss": 1.2036,
+ "theoretical_loss": 3.4953375938656537,
+ "tokens_seen": 1764425728
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037612036108324975,
+ "loss": 1.2947,
+ "theoretical_loss": 3.4953265911468865,
+ "tokens_seen": 1764491264
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037610431293881645,
+ "loss": 1.2635,
+ "theoretical_loss": 3.495315588951189,
+ "tokens_seen": 1764556800
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037608826479438315,
+ "loss": 1.1946,
+ "theoretical_loss": 3.4953045872785173,
+ "tokens_seen": 1764622336
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037607221664994985,
+ "loss": 1.3941,
+ "theoretical_loss": 3.495293586128828,
+ "tokens_seen": 1764687872
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037605616850551655,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4952825855020757,
+ "tokens_seen": 1764753408
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037604012036108325,
+ "loss": 1.2313,
+ "theoretical_loss": 3.495271585398217,
+ "tokens_seen": 1764818944
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037602407221664995,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4952605858172068,
+ "tokens_seen": 1764884480
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037600802407221665,
+ "loss": 1.3326,
+ "theoretical_loss": 3.495249586759001,
+ "tokens_seen": 1764950016
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037599197592778336,
+ "loss": 1.2739,
+ "theoretical_loss": 3.495238588223556,
+ "tokens_seen": 1765015552
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037597592778335006,
+ "loss": 1.2575,
+ "theoretical_loss": 3.495227590210827,
+ "tokens_seen": 1765081088
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037595987963891676,
+ "loss": 1.2384,
+ "theoretical_loss": 3.4952165927207703,
+ "tokens_seen": 1765146624
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037594383149448346,
+ "loss": 1.3129,
+ "theoretical_loss": 3.4952055957533408,
+ "tokens_seen": 1765212160
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037592778335005016,
+ "loss": 1.248,
+ "theoretical_loss": 3.495194599308495,
+ "tokens_seen": 1765277696
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037591173520561686,
+ "loss": 1.2642,
+ "theoretical_loss": 3.495183603386188,
+ "tokens_seen": 1765343232
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037589568706118356,
+ "loss": 1.3172,
+ "theoretical_loss": 3.495172607986376,
+ "tokens_seen": 1765408768
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 0.00037587963891675026,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4951616131090146,
+ "tokens_seen": 1765474304
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037586359077231696,
+ "loss": 1.3126,
+ "theoretical_loss": 3.49515061875406,
+ "tokens_seen": 1765539840
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037584754262788366,
+ "loss": 1.3983,
+ "theoretical_loss": 3.495139624921468,
+ "tokens_seen": 1765605376
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037583149448345036,
+ "loss": 1.24,
+ "theoretical_loss": 3.495128631611194,
+ "tokens_seen": 1765670912
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037581544633901706,
+ "loss": 1.3613,
+ "theoretical_loss": 3.495117638823193,
+ "tokens_seen": 1765736448
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037579939819458376,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4951066465574225,
+ "tokens_seen": 1765801984
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037578335005015046,
+ "loss": 1.218,
+ "theoretical_loss": 3.495095654813837,
+ "tokens_seen": 1765867520
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037576730190571716,
+ "loss": 1.2626,
+ "theoretical_loss": 3.495084663592393,
+ "tokens_seen": 1765933056
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037575125376128386,
+ "loss": 1.2772,
+ "theoretical_loss": 3.4950736728930467,
+ "tokens_seen": 1765998592
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037573520561685057,
+ "loss": 1.2854,
+ "theoretical_loss": 3.4950626827157523,
+ "tokens_seen": 1766064128
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037571915747241727,
+ "loss": 1.2305,
+ "theoretical_loss": 3.4950516930604674,
+ "tokens_seen": 1766129664
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 591956,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2556519508361816,
+ "objective/train/theoretical_loss": 3.495040703927147,
+ "objective/train/tokens_used": 1786655200,
+ "theoretical_loss": 3.495040703927147,
+ "tokens_seen": 1766195200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037570310932798397,
+ "loss": 1.3152,
+ "theoretical_loss": 3.495040703927147,
+ "tokens_seen": 1766195200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037568706118355067,
+ "loss": 1.3164,
+ "theoretical_loss": 3.495029715315747,
+ "tokens_seen": 1766260736
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037567101303911737,
+ "loss": 1.259,
+ "theoretical_loss": 3.495018727226223,
+ "tokens_seen": 1766326272
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037565496489468407,
+ "loss": 1.2204,
+ "theoretical_loss": 3.495007739658532,
+ "tokens_seen": 1766391808
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037563891675025077,
+ "loss": 1.2158,
+ "theoretical_loss": 3.4949967526126287,
+ "tokens_seen": 1766457344
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037562286860581747,
+ "loss": 1.2531,
+ "theoretical_loss": 3.494985766088469,
+ "tokens_seen": 1766522880
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037560682046138417,
+ "loss": 1.3381,
+ "theoretical_loss": 3.494974780086009,
+ "tokens_seen": 1766588416
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037559077231695087,
+ "loss": 1.4084,
+ "theoretical_loss": 3.494963794605205,
+ "tokens_seen": 1766653952
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037557472417251757,
+ "loss": 1.332,
+ "theoretical_loss": 3.494952809646012,
+ "tokens_seen": 1766719488
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037555867602808427,
+ "loss": 1.3173,
+ "theoretical_loss": 3.494941825208387,
+ "tokens_seen": 1766785024
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000375542627883651,
+ "loss": 1.3249,
+ "theoretical_loss": 3.494930841292285,
+ "tokens_seen": 1766850560
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003755265797392177,
+ "loss": 1.2097,
+ "theoretical_loss": 3.4949198578976626,
+ "tokens_seen": 1766916096
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003755105315947844,
+ "loss": 1.3193,
+ "theoretical_loss": 3.494908875024475,
+ "tokens_seen": 1766981632
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754944834503511,
+ "loss": 1.2514,
+ "theoretical_loss": 3.494897892672678,
+ "tokens_seen": 1767047168
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754784353059178,
+ "loss": 1.2798,
+ "theoretical_loss": 3.4948869108422285,
+ "tokens_seen": 1767112704
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754623871614845,
+ "loss": 1.3326,
+ "theoretical_loss": 3.494875929533081,
+ "tokens_seen": 1767178240
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754463390170512,
+ "loss": 1.184,
+ "theoretical_loss": 3.4948649487451933,
+ "tokens_seen": 1767243776
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754302908726179,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4948539684785196,
+ "tokens_seen": 1767309312
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003754142427281845,
+ "loss": 1.3068,
+ "theoretical_loss": 3.4948429887330166,
+ "tokens_seen": 1767374848
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753981945837512,
+ "loss": 1.1678,
+ "theoretical_loss": 3.4948320095086403,
+ "tokens_seen": 1767440384
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753821464393179,
+ "loss": 1.3387,
+ "theoretical_loss": 3.4948210308053462,
+ "tokens_seen": 1767505920
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753660982948847,
+ "loss": 1.396,
+ "theoretical_loss": 3.494810052623091,
+ "tokens_seen": 1767571456
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753500501504514,
+ "loss": 1.2773,
+ "theoretical_loss": 3.49479907496183,
+ "tokens_seen": 1767636992
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753340020060181,
+ "loss": 1.306,
+ "theoretical_loss": 3.494788097821519,
+ "tokens_seen": 1767702528
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753179538615848,
+ "loss": 1.2512,
+ "theoretical_loss": 3.494777121202115,
+ "tokens_seen": 1767768064
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003753019057171515,
+ "loss": 1.3901,
+ "theoretical_loss": 3.494766145103573,
+ "tokens_seen": 1767833600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003752858575727182,
+ "loss": 1.3101,
+ "theoretical_loss": 3.494755169525849,
+ "tokens_seen": 1767899136
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003752698094282849,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4947441944688995,
+ "tokens_seen": 1767964672
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003752537612838516,
+ "loss": 1.3844,
+ "theoretical_loss": 3.49473321993268,
+ "tokens_seen": 1768030208
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003752377131394183,
+ "loss": 1.2835,
+ "theoretical_loss": 3.494722245917147,
+ "tokens_seen": 1768095744
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000375221664994985,
+ "loss": 1.2095,
+ "theoretical_loss": 3.494711272422256,
+ "tokens_seen": 1768161280
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003752056168505517,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4947002994479632,
+ "tokens_seen": 1768226816
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751895687061184,
+ "loss": 1.2557,
+ "theoretical_loss": 3.4946893269942247,
+ "tokens_seen": 1768292352
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751735205616851,
+ "loss": 1.256,
+ "theoretical_loss": 3.494678355060997,
+ "tokens_seen": 1768357888
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751574724172518,
+ "loss": 1.2208,
+ "theoretical_loss": 3.4946673836482347,
+ "tokens_seen": 1768423424
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751414242728185,
+ "loss": 1.3012,
+ "theoretical_loss": 3.4946564127558952,
+ "tokens_seen": 1768488960
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751253761283852,
+ "loss": 1.3499,
+ "theoretical_loss": 3.494645442383934,
+ "tokens_seen": 1768554496
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003751093279839519,
+ "loss": 1.3661,
+ "theoretical_loss": 3.494634472532307,
+ "tokens_seen": 1768620032
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003750932798395186,
+ "loss": 1.2057,
+ "theoretical_loss": 3.494623503200971,
+ "tokens_seen": 1768685568
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003750772316950853,
+ "loss": 1.2916,
+ "theoretical_loss": 3.494612534389881,
+ "tokens_seen": 1768751104
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000375061183550652,
+ "loss": 1.3139,
+ "theoretical_loss": 3.4946015660989933,
+ "tokens_seen": 1768816640
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003750451354062187,
+ "loss": 1.3618,
+ "theoretical_loss": 3.494590598328264,
+ "tokens_seen": 1768882176
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003750290872617854,
+ "loss": 1.3754,
+ "theoretical_loss": 3.49457963107765,
+ "tokens_seen": 1768947712
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003750130391173521,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4945686643471063,
+ "tokens_seen": 1769013248
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749969909729188,
+ "loss": 1.3679,
+ "theoretical_loss": 3.4945576981365893,
+ "tokens_seen": 1769078784
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749809428284855,
+ "loss": 1.3123,
+ "theoretical_loss": 3.4945467324460555,
+ "tokens_seen": 1769144320
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749648946840522,
+ "loss": 1.2931,
+ "theoretical_loss": 3.49453576727546,
+ "tokens_seen": 1769209856
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749488465396189,
+ "loss": 1.3804,
+ "theoretical_loss": 3.4945248026247597,
+ "tokens_seen": 1769275392
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749327983951856,
+ "loss": 1.2771,
+ "theoretical_loss": 3.4945138384939107,
+ "tokens_seen": 1769340928
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003749167502507523,
+ "loss": 1.2594,
+ "theoretical_loss": 3.494502874882869,
+ "tokens_seen": 1769406464
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 593210,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3526359796524048,
+ "objective/train/theoretical_loss": 3.4944919117915907,
+ "objective/train/tokens_used": 1789932000,
+ "theoretical_loss": 3.4944919117915907,
+ "tokens_seen": 1769472000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000374900702106319,
+ "loss": 1.2735,
+ "theoretical_loss": 3.4944919117915907,
+ "tokens_seen": 1769472000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748846539618857,
+ "loss": 1.3899,
+ "theoretical_loss": 3.4944809492200317,
+ "tokens_seen": 1769537536
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748686058174524,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4944699871681477,
+ "tokens_seen": 1769603072
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748525576730191,
+ "loss": 1.2225,
+ "theoretical_loss": 3.494459025635896,
+ "tokens_seen": 1769668608
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748365095285858,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4944480646232314,
+ "tokens_seen": 1769734144
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748204613841525,
+ "loss": 1.2751,
+ "theoretical_loss": 3.494437104130111,
+ "tokens_seen": 1769799680
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003748044132397192,
+ "loss": 1.2445,
+ "theoretical_loss": 3.494426144156491,
+ "tokens_seen": 1769865216
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003747883650952859,
+ "loss": 1.2962,
+ "theoretical_loss": 3.4944151847023273,
+ "tokens_seen": 1769930752
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037477231695085255,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4944042257675756,
+ "tokens_seen": 1769996288
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037475626880641925,
+ "loss": 1.2421,
+ "theoretical_loss": 3.4943932673521925,
+ "tokens_seen": 1770061824
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037474022066198595,
+ "loss": 1.301,
+ "theoretical_loss": 3.4943823094561335,
+ "tokens_seen": 1770127360
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037472417251755265,
+ "loss": 1.3186,
+ "theoretical_loss": 3.494371352079356,
+ "tokens_seen": 1770192896
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037470812437311935,
+ "loss": 1.2732,
+ "theoretical_loss": 3.494360395221815,
+ "tokens_seen": 1770258432
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037469207622868605,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4943494388834675,
+ "tokens_seen": 1770323968
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037467602808425275,
+ "loss": 1.2879,
+ "theoretical_loss": 3.4943384830642694,
+ "tokens_seen": 1770389504
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037465997993981945,
+ "loss": 1.4281,
+ "theoretical_loss": 3.4943275277641765,
+ "tokens_seen": 1770455040
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037464393179538615,
+ "loss": 1.246,
+ "theoretical_loss": 3.4943165729831454,
+ "tokens_seen": 1770520576
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037462788365095286,
+ "loss": 1.2865,
+ "theoretical_loss": 3.494305618721132,
+ "tokens_seen": 1770586112
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037461183550651956,
+ "loss": 1.2833,
+ "theoretical_loss": 3.494294664978093,
+ "tokens_seen": 1770651648
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037459578736208626,
+ "loss": 1.3122,
+ "theoretical_loss": 3.4942837117539844,
+ "tokens_seen": 1770717184
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037457973921765296,
+ "loss": 1.3477,
+ "theoretical_loss": 3.494272759048762,
+ "tokens_seen": 1770782720
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037456369107321966,
+ "loss": 1.2849,
+ "theoretical_loss": 3.494261806862383,
+ "tokens_seen": 1770848256
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037454764292878636,
+ "loss": 1.2498,
+ "theoretical_loss": 3.494250855194802,
+ "tokens_seen": 1770913792
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037453159478435306,
+ "loss": 1.3322,
+ "theoretical_loss": 3.4942399040459766,
+ "tokens_seen": 1770979328
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037451554663991976,
+ "loss": 1.2993,
+ "theoretical_loss": 3.4942289534158624,
+ "tokens_seen": 1771044864
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037449949849548646,
+ "loss": 1.2036,
+ "theoretical_loss": 3.4942180033044163,
+ "tokens_seen": 1771110400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037448345035105316,
+ "loss": 1.2881,
+ "theoretical_loss": 3.4942070537115937,
+ "tokens_seen": 1771175936
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037446740220661986,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4941961046373518,
+ "tokens_seen": 1771241472
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037445135406218656,
+ "loss": 1.3372,
+ "theoretical_loss": 3.4941851560816457,
+ "tokens_seen": 1771307008
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037443530591775326,
+ "loss": 1.2596,
+ "theoretical_loss": 3.4941742080444325,
+ "tokens_seen": 1771372544
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037441925777331996,
+ "loss": 1.2261,
+ "theoretical_loss": 3.494163260525668,
+ "tokens_seen": 1771438080
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037440320962888666,
+ "loss": 1.2539,
+ "theoretical_loss": 3.494152313525309,
+ "tokens_seen": 1771503616
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037438716148445336,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4941413670433112,
+ "tokens_seen": 1771569152
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037437111334002007,
+ "loss": 1.2389,
+ "theoretical_loss": 3.494130421079632,
+ "tokens_seen": 1771634688
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037435506519558677,
+ "loss": 1.3776,
+ "theoretical_loss": 3.4941194756342258,
+ "tokens_seen": 1771700224
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037433901705115347,
+ "loss": 1.3169,
+ "theoretical_loss": 3.4941085307070496,
+ "tokens_seen": 1771765760
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037432296890672017,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4940975862980608,
+ "tokens_seen": 1771831296
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037430692076228687,
+ "loss": 1.3159,
+ "theoretical_loss": 3.494086642407215,
+ "tokens_seen": 1771896832
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037429087261785357,
+ "loss": 1.3105,
+ "theoretical_loss": 3.494075699034468,
+ "tokens_seen": 1771962368
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037427482447342027,
+ "loss": 1.2734,
+ "theoretical_loss": 3.494064756179777,
+ "tokens_seen": 1772027904
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037425877632898697,
+ "loss": 1.3476,
+ "theoretical_loss": 3.4940538138430974,
+ "tokens_seen": 1772093440
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037424272818455367,
+ "loss": 1.3377,
+ "theoretical_loss": 3.4940428720243863,
+ "tokens_seen": 1772158976
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037422668004012037,
+ "loss": 1.366,
+ "theoretical_loss": 3.494031930723599,
+ "tokens_seen": 1772224512
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037421063189568707,
+ "loss": 1.2121,
+ "theoretical_loss": 3.4940209899406933,
+ "tokens_seen": 1772290048
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037419458375125377,
+ "loss": 1.3367,
+ "theoretical_loss": 3.494010049675625,
+ "tokens_seen": 1772355584
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037417853560682047,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4939991099283496,
+ "tokens_seen": 1772421120
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037416248746238723,
+ "loss": 1.215,
+ "theoretical_loss": 3.493988170698824,
+ "tokens_seen": 1772486656
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003741464393179539,
+ "loss": 1.2512,
+ "theoretical_loss": 3.493977231987005,
+ "tokens_seen": 1772552192
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003741303911735206,
+ "loss": 1.3701,
+ "theoretical_loss": 3.493966293792848,
+ "tokens_seen": 1772617728
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003741143430290873,
+ "loss": 1.2991,
+ "theoretical_loss": 3.493955356116311,
+ "tokens_seen": 1772683264
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 594137,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2289804220199585,
+ "objective/train/theoretical_loss": 3.4939444189573483,
+ "objective/train/tokens_used": 1793208800,
+ "theoretical_loss": 3.4939444189573483,
+ "tokens_seen": 1772748800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000374098294884654,
+ "loss": 1.235,
+ "theoretical_loss": 3.4939444189573483,
+ "tokens_seen": 1772748800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740822467402207,
+ "loss": 1.2525,
+ "theoretical_loss": 3.493933482315918,
+ "tokens_seen": 1772814336
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740661985957874,
+ "loss": 1.2766,
+ "theoretical_loss": 3.4939225461919756,
+ "tokens_seen": 1772879872
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740501504513541,
+ "loss": 1.3094,
+ "theoretical_loss": 3.493911610585477,
+ "tokens_seen": 1772945408
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740341023069208,
+ "loss": 1.3317,
+ "theoretical_loss": 3.49390067549638,
+ "tokens_seen": 1773010944
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740180541624875,
+ "loss": 1.3131,
+ "theoretical_loss": 3.49388974092464,
+ "tokens_seen": 1773076480
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003740020060180542,
+ "loss": 1.1886,
+ "theoretical_loss": 3.493878806870214,
+ "tokens_seen": 1773142016
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003739859578736209,
+ "loss": 1.2711,
+ "theoretical_loss": 3.4938678733330573,
+ "tokens_seen": 1773207552
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003739699097291876,
+ "loss": 1.3242,
+ "theoretical_loss": 3.493856940313128,
+ "tokens_seen": 1773273088
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003739538615847543,
+ "loss": 1.2552,
+ "theoretical_loss": 3.493846007810381,
+ "tokens_seen": 1773338624
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000373937813440321,
+ "loss": 1.2606,
+ "theoretical_loss": 3.493835075824773,
+ "tokens_seen": 1773404160
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003739217652958877,
+ "loss": 1.2418,
+ "theoretical_loss": 3.493824144356261,
+ "tokens_seen": 1773469696
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003739057171514544,
+ "loss": 1.3952,
+ "theoretical_loss": 3.4938132134048017,
+ "tokens_seen": 1773535232
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738896690070211,
+ "loss": 1.295,
+ "theoretical_loss": 3.4938022829703503,
+ "tokens_seen": 1773600768
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738736208625878,
+ "loss": 1.2371,
+ "theoretical_loss": 3.493791353052864,
+ "tokens_seen": 1773666304
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738575727181545,
+ "loss": 1.2186,
+ "theoretical_loss": 3.4937804236523,
+ "tokens_seen": 1773731840
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738415245737212,
+ "loss": 1.2767,
+ "theoretical_loss": 3.493769494768613,
+ "tokens_seen": 1773797376
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738254764292879,
+ "loss": 1.286,
+ "theoretical_loss": 3.493758566401761,
+ "tokens_seen": 1773862912
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003738094282848546,
+ "loss": 1.3555,
+ "theoretical_loss": 3.4937476385516995,
+ "tokens_seen": 1773928448
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003737933801404213,
+ "loss": 1.162,
+ "theoretical_loss": 3.4937367112183857,
+ "tokens_seen": 1773993984
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000373777331995988,
+ "loss": 1.2317,
+ "theoretical_loss": 3.493725784401776,
+ "tokens_seen": 1774059520
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003737612838515547,
+ "loss": 1.2598,
+ "theoretical_loss": 3.493714858101826,
+ "tokens_seen": 1774125056
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003737452357071214,
+ "loss": 1.2244,
+ "theoretical_loss": 3.4937039323184926,
+ "tokens_seen": 1774190592
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003737291875626881,
+ "loss": 1.2337,
+ "theoretical_loss": 3.493693007051733,
+ "tokens_seen": 1774256128
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003737131394182548,
+ "loss": 1.3792,
+ "theoretical_loss": 3.493682082301503,
+ "tokens_seen": 1774321664
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736970912738215,
+ "loss": 1.3639,
+ "theoretical_loss": 3.4936711580677597,
+ "tokens_seen": 1774387200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736810431293882,
+ "loss": 1.3068,
+ "theoretical_loss": 3.493660234350459,
+ "tokens_seen": 1774452736
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736649949849549,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4936493111495572,
+ "tokens_seen": 1774518272
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736489468405216,
+ "loss": 1.324,
+ "theoretical_loss": 3.493638388465012,
+ "tokens_seen": 1774583808
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736328986960883,
+ "loss": 1.3445,
+ "theoretical_loss": 3.493627466296778,
+ "tokens_seen": 1774649344
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000373616850551655,
+ "loss": 1.2381,
+ "theoretical_loss": 3.493616544644814,
+ "tokens_seen": 1774714880
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003736008024072217,
+ "loss": 1.391,
+ "theoretical_loss": 3.4936056235090747,
+ "tokens_seen": 1774780416
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735847542627884,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4935947028895176,
+ "tokens_seen": 1774845952
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735687061183551,
+ "loss": 1.2259,
+ "theoretical_loss": 3.493583782786099,
+ "tokens_seen": 1774911488
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735526579739218,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4935728631987755,
+ "tokens_seen": 1774977024
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735366098294885,
+ "loss": 1.2185,
+ "theoretical_loss": 3.4935619441275034,
+ "tokens_seen": 1775042560
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735205616850552,
+ "loss": 1.3384,
+ "theoretical_loss": 3.4935510255722395,
+ "tokens_seen": 1775108096
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003735045135406219,
+ "loss": 1.3412,
+ "theoretical_loss": 3.4935401075329406,
+ "tokens_seen": 1775173632
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003734884653961886,
+ "loss": 1.2276,
+ "theoretical_loss": 3.4935291900095624,
+ "tokens_seen": 1775239168
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003734724172517553,
+ "loss": 1.2023,
+ "theoretical_loss": 3.4935182730020626,
+ "tokens_seen": 1775304704
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000373456369107322,
+ "loss": 1.375,
+ "theoretical_loss": 3.4935073565103973,
+ "tokens_seen": 1775370240
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003734403209628887,
+ "loss": 1.3523,
+ "theoretical_loss": 3.493496440534523,
+ "tokens_seen": 1775435776
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003734242728184554,
+ "loss": 1.2747,
+ "theoretical_loss": 3.493485525074396,
+ "tokens_seen": 1775501312
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003734082246740221,
+ "loss": 1.2721,
+ "theoretical_loss": 3.493474610129973,
+ "tokens_seen": 1775566848
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003733921765295888,
+ "loss": 1.2859,
+ "theoretical_loss": 3.4934636957012115,
+ "tokens_seen": 1775632384
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037337612838515545,
+ "loss": 1.3189,
+ "theoretical_loss": 3.4934527817880676,
+ "tokens_seen": 1775697920
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037336008024072215,
+ "loss": 1.4163,
+ "theoretical_loss": 3.493441868390497,
+ "tokens_seen": 1775763456
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037334403209628885,
+ "loss": 1.4387,
+ "theoretical_loss": 3.4934309555084573,
+ "tokens_seen": 1775828992
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037332798395185555,
+ "loss": 1.1661,
+ "theoretical_loss": 3.493420043141905,
+ "tokens_seen": 1775894528
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037331193580742225,
+ "loss": 1.3508,
+ "theoretical_loss": 3.4934091312907967,
+ "tokens_seen": 1775960064
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 595419,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3804264068603516,
+ "objective/train/theoretical_loss": 3.493398219955089,
+ "objective/train/tokens_used": 1796485600,
+ "theoretical_loss": 3.493398219955089,
+ "tokens_seen": 1776025600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037329588766298895,
+ "loss": 1.3509,
+ "theoretical_loss": 3.493398219955089,
+ "tokens_seen": 1776025600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037327983951855565,
+ "loss": 1.3068,
+ "theoretical_loss": 3.4933873091347385,
+ "tokens_seen": 1776091136
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037326379137412235,
+ "loss": 1.1843,
+ "theoretical_loss": 3.4933763988297013,
+ "tokens_seen": 1776156672
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037324774322968906,
+ "loss": 1.1732,
+ "theoretical_loss": 3.493365489039935,
+ "tokens_seen": 1776222208
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037323169508525576,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4933545797653958,
+ "tokens_seen": 1776287744
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037321564694082246,
+ "loss": 1.3126,
+ "theoretical_loss": 3.4933436710060404,
+ "tokens_seen": 1776353280
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037319959879638916,
+ "loss": 1.3525,
+ "theoretical_loss": 3.493332762761826,
+ "tokens_seen": 1776418816
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037318355065195586,
+ "loss": 1.3163,
+ "theoretical_loss": 3.4933218550327076,
+ "tokens_seen": 1776484352
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037316750250752256,
+ "loss": 1.3326,
+ "theoretical_loss": 3.493310947818644,
+ "tokens_seen": 1776549888
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037315145436308926,
+ "loss": 1.2645,
+ "theoretical_loss": 3.49330004111959,
+ "tokens_seen": 1776615424
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037313540621865596,
+ "loss": 1.2377,
+ "theoretical_loss": 3.493289134935504,
+ "tokens_seen": 1776680960
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037311935807422266,
+ "loss": 1.3154,
+ "theoretical_loss": 3.4932782292663416,
+ "tokens_seen": 1776746496
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037310330992978936,
+ "loss": 1.3308,
+ "theoretical_loss": 3.4932673241120593,
+ "tokens_seen": 1776812032
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037308726178535606,
+ "loss": 1.1598,
+ "theoretical_loss": 3.4932564194726154,
+ "tokens_seen": 1776877568
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037307121364092276,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4932455153479642,
+ "tokens_seen": 1776943104
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037305516549648946,
+ "loss": 1.2541,
+ "theoretical_loss": 3.493234611738065,
+ "tokens_seen": 1777008640
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037303911735205616,
+ "loss": 1.2887,
+ "theoretical_loss": 3.493223708642872,
+ "tokens_seen": 1777074176
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037302306920762286,
+ "loss": 1.2249,
+ "theoretical_loss": 3.4932128060623437,
+ "tokens_seen": 1777139712
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037300702106318956,
+ "loss": 1.2834,
+ "theoretical_loss": 3.493201903996436,
+ "tokens_seen": 1777205248
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037299097291875627,
+ "loss": 1.339,
+ "theoretical_loss": 3.493191002445106,
+ "tokens_seen": 1777270784
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037297492477432297,
+ "loss": 1.163,
+ "theoretical_loss": 3.4931801014083104,
+ "tokens_seen": 1777336320
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003729588766298897,
+ "loss": 1.3499,
+ "theoretical_loss": 3.4931692008860056,
+ "tokens_seen": 1777401856
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003729428284854564,
+ "loss": 1.3439,
+ "theoretical_loss": 3.493158300878149,
+ "tokens_seen": 1777467392
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003729267803410231,
+ "loss": 1.3143,
+ "theoretical_loss": 3.4931474013846966,
+ "tokens_seen": 1777532928
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003729107321965898,
+ "loss": 1.2063,
+ "theoretical_loss": 3.493136502405606,
+ "tokens_seen": 1777598464
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003728946840521565,
+ "loss": 1.3282,
+ "theoretical_loss": 3.4931256039408334,
+ "tokens_seen": 1777664000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003728786359077232,
+ "loss": 1.2519,
+ "theoretical_loss": 3.4931147059903354,
+ "tokens_seen": 1777729536
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003728625877632899,
+ "loss": 1.2361,
+ "theoretical_loss": 3.493103808554069,
+ "tokens_seen": 1777795072
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003728465396188566,
+ "loss": 1.3067,
+ "theoretical_loss": 3.493092911631991,
+ "tokens_seen": 1777860608
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003728304914744233,
+ "loss": 1.3457,
+ "theoretical_loss": 3.4930820152240587,
+ "tokens_seen": 1777926144
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037281444332999,
+ "loss": 1.3067,
+ "theoretical_loss": 3.493071119330228,
+ "tokens_seen": 1777991680
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037279839518555673,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4930602239504562,
+ "tokens_seen": 1778057216
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037278234704112343,
+ "loss": 1.2514,
+ "theoretical_loss": 3.4930493290847,
+ "tokens_seen": 1778122752
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037276629889669013,
+ "loss": 1.312,
+ "theoretical_loss": 3.493038434732916,
+ "tokens_seen": 1778188288
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003727502507522568,
+ "loss": 1.2231,
+ "theoretical_loss": 3.4930275408950617,
+ "tokens_seen": 1778253824
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003727342026078235,
+ "loss": 1.3043,
+ "theoretical_loss": 3.4930166475710926,
+ "tokens_seen": 1778319360
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003727181544633902,
+ "loss": 1.2251,
+ "theoretical_loss": 3.493005754760967,
+ "tokens_seen": 1778384896
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003727021063189569,
+ "loss": 1.1795,
+ "theoretical_loss": 3.4929948624646414,
+ "tokens_seen": 1778450432
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003726860581745236,
+ "loss": 1.2545,
+ "theoretical_loss": 3.4929839706820713,
+ "tokens_seen": 1778515968
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003726700100300903,
+ "loss": 1.3244,
+ "theoretical_loss": 3.4929730794132157,
+ "tokens_seen": 1778581504
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000372653961885657,
+ "loss": 1.3092,
+ "theoretical_loss": 3.4929621886580295,
+ "tokens_seen": 1778647040
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003726379137412237,
+ "loss": 1.2789,
+ "theoretical_loss": 3.492951298416471,
+ "tokens_seen": 1778712576
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003726218655967904,
+ "loss": 1.3034,
+ "theoretical_loss": 3.4929404086884954,
+ "tokens_seen": 1778778112
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003726058174523571,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4929295194740613,
+ "tokens_seen": 1778843648
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725897693079238,
+ "loss": 1.2807,
+ "theoretical_loss": 3.4929186307731244,
+ "tokens_seen": 1778909184
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725737211634905,
+ "loss": 1.288,
+ "theoretical_loss": 3.4929077425856425,
+ "tokens_seen": 1778974720
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725576730190572,
+ "loss": 1.2739,
+ "theoretical_loss": 3.4928968549115713,
+ "tokens_seen": 1779040256
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725416248746239,
+ "loss": 1.2927,
+ "theoretical_loss": 3.492885967750869,
+ "tokens_seen": 1779105792
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725255767301906,
+ "loss": 1.3796,
+ "theoretical_loss": 3.4928750811034917,
+ "tokens_seen": 1779171328
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003725095285857573,
+ "loss": 1.375,
+ "theoretical_loss": 3.492864194969396,
+ "tokens_seen": 1779236864
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 596357,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4457015991210938,
+ "objective/train/theoretical_loss": 3.4928533093485394,
+ "objective/train/tokens_used": 1799762400,
+ "theoretical_loss": 3.4928533093485394,
+ "tokens_seen": 1779302400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000372493480441324,
+ "loss": 1.3579,
+ "theoretical_loss": 3.4928533093485394,
+ "tokens_seen": 1779302400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003724774322968907,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4928424242408784,
+ "tokens_seen": 1779367936
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003724613841524574,
+ "loss": 1.4106,
+ "theoretical_loss": 3.492831539646371,
+ "tokens_seen": 1779433472
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003724453360080241,
+ "loss": 1.2412,
+ "theoretical_loss": 3.492820655564972,
+ "tokens_seen": 1779499008
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003724292878635908,
+ "loss": 1.3766,
+ "theoretical_loss": 3.4928097719966402,
+ "tokens_seen": 1779564544
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003724132397191575,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4927988889413317,
+ "tokens_seen": 1779630080
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723971915747242,
+ "loss": 1.348,
+ "theoretical_loss": 3.492788006399004,
+ "tokens_seen": 1779695616
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723811434302909,
+ "loss": 1.2678,
+ "theoretical_loss": 3.4927771243696135,
+ "tokens_seen": 1779761152
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723650952858576,
+ "loss": 1.281,
+ "theoretical_loss": 3.4927662428531168,
+ "tokens_seen": 1779826688
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723490471414243,
+ "loss": 1.2367,
+ "theoretical_loss": 3.492755361849472,
+ "tokens_seen": 1779892224
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000372332998996991,
+ "loss": 1.3321,
+ "theoretical_loss": 3.492744481358635,
+ "tokens_seen": 1779957760
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723169508525577,
+ "loss": 1.2936,
+ "theoretical_loss": 3.4927336013805625,
+ "tokens_seen": 1780023296
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003723009027081244,
+ "loss": 1.343,
+ "theoretical_loss": 3.4927227219152126,
+ "tokens_seen": 1780088832
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722848545636911,
+ "loss": 1.3221,
+ "theoretical_loss": 3.492711842962542,
+ "tokens_seen": 1780154368
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722688064192578,
+ "loss": 1.2065,
+ "theoretical_loss": 3.4927009645225073,
+ "tokens_seen": 1780219904
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722527582748245,
+ "loss": 1.3838,
+ "theoretical_loss": 3.492690086595065,
+ "tokens_seen": 1780285440
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722367101303912,
+ "loss": 1.2958,
+ "theoretical_loss": 3.492679209180173,
+ "tokens_seen": 1780350976
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722206619859579,
+ "loss": 1.2921,
+ "theoretical_loss": 3.492668332277788,
+ "tokens_seen": 1780416512
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003722046138415246,
+ "loss": 1.2606,
+ "theoretical_loss": 3.492657455887867,
+ "tokens_seen": 1780482048
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003721885656970913,
+ "loss": 1.3063,
+ "theoretical_loss": 3.4926465800103665,
+ "tokens_seen": 1780547584
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000372172517552658,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4926357046452443,
+ "tokens_seen": 1780613120
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003721564694082247,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4926248297924567,
+ "tokens_seen": 1780678656
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003721404212637914,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4926139554519615,
+ "tokens_seen": 1780744192
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003721243731193581,
+ "loss": 1.2603,
+ "theoretical_loss": 3.492603081623715,
+ "tokens_seen": 1780809728
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003721083249749248,
+ "loss": 1.2487,
+ "theoretical_loss": 3.492592208307674,
+ "tokens_seen": 1780875264
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003720922768304915,
+ "loss": 1.337,
+ "theoretical_loss": 3.4925813355037967,
+ "tokens_seen": 1780940800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003720762286860582,
+ "loss": 1.3738,
+ "theoretical_loss": 3.4925704632120387,
+ "tokens_seen": 1781006336
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003720601805416249,
+ "loss": 1.2783,
+ "theoretical_loss": 3.492559591432358,
+ "tokens_seen": 1781071872
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003720441323971916,
+ "loss": 1.2845,
+ "theoretical_loss": 3.4925487201647116,
+ "tokens_seen": 1781137408
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003720280842527583,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4925378494090564,
+ "tokens_seen": 1781202944
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000372012036108325,
+ "loss": 1.3961,
+ "theoretical_loss": 3.4925269791653486,
+ "tokens_seen": 1781268480
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003719959879638917,
+ "loss": 1.2661,
+ "theoretical_loss": 3.492516109433547,
+ "tokens_seen": 1781334016
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003719799398194584,
+ "loss": 1.2576,
+ "theoretical_loss": 3.4925052402136076,
+ "tokens_seen": 1781399552
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037196389167502505,
+ "loss": 1.2348,
+ "theoretical_loss": 3.4924943715054866,
+ "tokens_seen": 1781465088
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037194784353059175,
+ "loss": 1.2802,
+ "theoretical_loss": 3.492483503309143,
+ "tokens_seen": 1781530624
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037193179538615845,
+ "loss": 1.2739,
+ "theoretical_loss": 3.492472635624533,
+ "tokens_seen": 1781596160
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037191574724172515,
+ "loss": 1.3493,
+ "theoretical_loss": 3.492461768451613,
+ "tokens_seen": 1781661696
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037189969909729185,
+ "loss": 1.2575,
+ "theoretical_loss": 3.492450901790341,
+ "tokens_seen": 1781727232
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037188365095285856,
+ "loss": 1.322,
+ "theoretical_loss": 3.4924400356406737,
+ "tokens_seen": 1781792768
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037186760280842526,
+ "loss": 1.1625,
+ "theoretical_loss": 3.492429170002568,
+ "tokens_seen": 1781858304
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037185155466399196,
+ "loss": 1.2551,
+ "theoretical_loss": 3.492418304875981,
+ "tokens_seen": 1781923840
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037183550651955866,
+ "loss": 1.2944,
+ "theoretical_loss": 3.4924074402608714,
+ "tokens_seen": 1781989376
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037181945837512536,
+ "loss": 1.3602,
+ "theoretical_loss": 3.4923965761571942,
+ "tokens_seen": 1782054912
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037180341023069206,
+ "loss": 1.1933,
+ "theoretical_loss": 3.492385712564907,
+ "tokens_seen": 1782120448
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037178736208625876,
+ "loss": 1.3251,
+ "theoretical_loss": 3.4923748494839675,
+ "tokens_seen": 1782185984
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003717713139418255,
+ "loss": 1.29,
+ "theoretical_loss": 3.4923639869143326,
+ "tokens_seen": 1782251520
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003717552657973922,
+ "loss": 1.3486,
+ "theoretical_loss": 3.4923531248559594,
+ "tokens_seen": 1782317056
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003717392176529589,
+ "loss": 1.1895,
+ "theoretical_loss": 3.4923422633088053,
+ "tokens_seen": 1782382592
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003717231695085256,
+ "loss": 1.229,
+ "theoretical_loss": 3.4923314022728267,
+ "tokens_seen": 1782448128
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003717071213640923,
+ "loss": 1.3417,
+ "theoretical_loss": 3.492320541747982,
+ "tokens_seen": 1782513664
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 597594,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.43216872215271,
+ "objective/train/theoretical_loss": 3.4923096817342265,
+ "objective/train/tokens_used": 1803039200,
+ "theoretical_loss": 3.4923096817342265,
+ "tokens_seen": 1782579200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000371691073219659,
+ "loss": 1.3708,
+ "theoretical_loss": 3.4923096817342265,
+ "tokens_seen": 1782579200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003716750250752257,
+ "loss": 1.2503,
+ "theoretical_loss": 3.492298822231519,
+ "tokens_seen": 1782644736
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003716589769307924,
+ "loss": 1.2937,
+ "theoretical_loss": 3.492287963239816,
+ "tokens_seen": 1782710272
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003716429287863591,
+ "loss": 1.287,
+ "theoretical_loss": 3.492277104759075,
+ "tokens_seen": 1782775808
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003716268806419258,
+ "loss": 1.2429,
+ "theoretical_loss": 3.492266246789253,
+ "tokens_seen": 1782841344
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003716108324974925,
+ "loss": 1.2975,
+ "theoretical_loss": 3.4922553893303068,
+ "tokens_seen": 1782906880
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003715947843530592,
+ "loss": 1.2665,
+ "theoretical_loss": 3.4922445323821942,
+ "tokens_seen": 1782972416
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003715787362086259,
+ "loss": 1.2447,
+ "theoretical_loss": 3.492233675944872,
+ "tokens_seen": 1783037952
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003715626880641926,
+ "loss": 1.2945,
+ "theoretical_loss": 3.4922228200182976,
+ "tokens_seen": 1783103488
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003715466399197593,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4922119646024283,
+ "tokens_seen": 1783169024
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000371530591775326,
+ "loss": 1.3223,
+ "theoretical_loss": 3.4922011096972208,
+ "tokens_seen": 1783234560
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003715145436308927,
+ "loss": 1.2997,
+ "theoretical_loss": 3.4921902553026327,
+ "tokens_seen": 1783300096
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003714984954864594,
+ "loss": 1.2471,
+ "theoretical_loss": 3.492179401418621,
+ "tokens_seen": 1783365632
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003714824473420261,
+ "loss": 1.2847,
+ "theoretical_loss": 3.492168548045143,
+ "tokens_seen": 1783431168
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003714663991975928,
+ "loss": 1.2639,
+ "theoretical_loss": 3.492157695182156,
+ "tokens_seen": 1783496704
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003714503510531595,
+ "loss": 1.1435,
+ "theoretical_loss": 3.4921468428296176,
+ "tokens_seen": 1783562240
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037143430290872623,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4921359909874847,
+ "tokens_seen": 1783627776
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037141825476429293,
+ "loss": 1.3278,
+ "theoretical_loss": 3.4921251396557142,
+ "tokens_seen": 1783693312
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037140220661985963,
+ "loss": 1.2553,
+ "theoretical_loss": 3.492114288834264,
+ "tokens_seen": 1783758848
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037138615847542633,
+ "loss": 1.3382,
+ "theoretical_loss": 3.4921034385230905,
+ "tokens_seen": 1783824384
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037137011033099303,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4920925887221514,
+ "tokens_seen": 1783889920
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037135406218655973,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4920817394314048,
+ "tokens_seen": 1783955456
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003713380140421264,
+ "loss": 1.203,
+ "theoretical_loss": 3.4920708906508064,
+ "tokens_seen": 1784020992
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003713219658976931,
+ "loss": 1.3214,
+ "theoretical_loss": 3.4920600423803148,
+ "tokens_seen": 1784086528
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003713059177532598,
+ "loss": 1.2705,
+ "theoretical_loss": 3.492049194619886,
+ "tokens_seen": 1784152064
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003712898696088265,
+ "loss": 1.3463,
+ "theoretical_loss": 3.492038347369479,
+ "tokens_seen": 1784217600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003712738214643932,
+ "loss": 1.312,
+ "theoretical_loss": 3.4920275006290495,
+ "tokens_seen": 1784283136
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003712577733199599,
+ "loss": 1.2994,
+ "theoretical_loss": 3.4920166543985554,
+ "tokens_seen": 1784348672
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003712417251755266,
+ "loss": 1.2465,
+ "theoretical_loss": 3.492005808677954,
+ "tokens_seen": 1784414208
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003712256770310933,
+ "loss": 1.2741,
+ "theoretical_loss": 3.491994963467203,
+ "tokens_seen": 1784479744
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037120962888666,
+ "loss": 1.2434,
+ "theoretical_loss": 3.491984118766259,
+ "tokens_seen": 1784545280
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711935807422267,
+ "loss": 1.2654,
+ "theoretical_loss": 3.4919732745750798,
+ "tokens_seen": 1784610816
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711775325977934,
+ "loss": 1.3727,
+ "theoretical_loss": 3.491962430893622,
+ "tokens_seen": 1784676352
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711614844533601,
+ "loss": 1.2808,
+ "theoretical_loss": 3.4919515877218443,
+ "tokens_seen": 1784741888
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711454363089268,
+ "loss": 1.2759,
+ "theoretical_loss": 3.491940745059703,
+ "tokens_seen": 1784807424
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711293881644935,
+ "loss": 1.3148,
+ "theoretical_loss": 3.491929902907155,
+ "tokens_seen": 1784872960
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003711133400200602,
+ "loss": 1.2479,
+ "theoretical_loss": 3.4919190612641593,
+ "tokens_seen": 1784938496
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710972918756269,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4919082201306715,
+ "tokens_seen": 1785004032
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710812437311936,
+ "loss": 1.3539,
+ "theoretical_loss": 3.49189737950665,
+ "tokens_seen": 1785069568
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710651955867603,
+ "loss": 1.2847,
+ "theoretical_loss": 3.491886539392052,
+ "tokens_seen": 1785135104
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000371049147442327,
+ "loss": 1.2675,
+ "theoretical_loss": 3.491875699786834,
+ "tokens_seen": 1785200640
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710330992978937,
+ "loss": 1.3089,
+ "theoretical_loss": 3.4918648606909546,
+ "tokens_seen": 1785266176
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710170511534604,
+ "loss": 1.2881,
+ "theoretical_loss": 3.4918540221043703,
+ "tokens_seen": 1785331712
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003710010030090271,
+ "loss": 1.2959,
+ "theoretical_loss": 3.491843184027039,
+ "tokens_seen": 1785397248
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709849548645938,
+ "loss": 1.4386,
+ "theoretical_loss": 3.491832346458918,
+ "tokens_seen": 1785462784
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709689067201605,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4918215093999643,
+ "tokens_seen": 1785528320
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709528585757272,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4918106728501357,
+ "tokens_seen": 1785593856
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709368104312939,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4917998368093897,
+ "tokens_seen": 1785659392
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709207622868606,
+ "loss": 1.1769,
+ "theoretical_loss": 3.491789001277683,
+ "tokens_seen": 1785724928
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003709047141424273,
+ "loss": 1.2448,
+ "theoretical_loss": 3.4917781662549734,
+ "tokens_seen": 1785790464
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 598857,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3066961765289307,
+ "objective/train/theoretical_loss": 3.491767331741219,
+ "objective/train/tokens_used": 1806316000,
+ "theoretical_loss": 3.491767331741219,
+ "tokens_seen": 1785856000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370888665997994,
+ "loss": 1.2125,
+ "theoretical_loss": 3.491767331741219,
+ "tokens_seen": 1785856000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003708726178535607,
+ "loss": 1.3361,
+ "theoretical_loss": 3.491756497736376,
+ "tokens_seen": 1785921536
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003708565697091274,
+ "loss": 1.1691,
+ "theoretical_loss": 3.491745664240403,
+ "tokens_seen": 1785987072
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003708405215646941,
+ "loss": 1.1888,
+ "theoretical_loss": 3.4917348312532566,
+ "tokens_seen": 1786052608
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003708244734202608,
+ "loss": 1.2728,
+ "theoretical_loss": 3.4917239987748943,
+ "tokens_seen": 1786118144
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003708084252758275,
+ "loss": 1.2376,
+ "theoretical_loss": 3.491713166805274,
+ "tokens_seen": 1786183680
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003707923771313942,
+ "loss": 1.2818,
+ "theoretical_loss": 3.4917023353443524,
+ "tokens_seen": 1786249216
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003707763289869609,
+ "loss": 1.3009,
+ "theoretical_loss": 3.4916915043920875,
+ "tokens_seen": 1786314752
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003707602808425276,
+ "loss": 1.3365,
+ "theoretical_loss": 3.491680673948437,
+ "tokens_seen": 1786380288
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003707442326980943,
+ "loss": 1.2533,
+ "theoretical_loss": 3.491669844013358,
+ "tokens_seen": 1786445824
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370728184553661,
+ "loss": 1.1861,
+ "theoretical_loss": 3.4916590145868076,
+ "tokens_seen": 1786511360
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003707121364092277,
+ "loss": 1.1946,
+ "theoretical_loss": 3.491648185668744,
+ "tokens_seen": 1786576896
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706960882647944,
+ "loss": 1.3617,
+ "theoretical_loss": 3.4916373572591244,
+ "tokens_seen": 1786642432
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706800401203611,
+ "loss": 1.2532,
+ "theoretical_loss": 3.491626529357906,
+ "tokens_seen": 1786707968
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706639919759278,
+ "loss": 1.2814,
+ "theoretical_loss": 3.4916157019650464,
+ "tokens_seen": 1786773504
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706479438314945,
+ "loss": 1.311,
+ "theoretical_loss": 3.4916048750805033,
+ "tokens_seen": 1786839040
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706318956870612,
+ "loss": 1.2634,
+ "theoretical_loss": 3.4915940487042345,
+ "tokens_seen": 1786904576
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003706158475426279,
+ "loss": 1.3685,
+ "theoretical_loss": 3.491583222836196,
+ "tokens_seen": 1786970112
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705997993981946,
+ "loss": 1.2525,
+ "theoretical_loss": 3.491572397476347,
+ "tokens_seen": 1787035648
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705837512537613,
+ "loss": 1.3504,
+ "theoretical_loss": 3.4915615726246445,
+ "tokens_seen": 1787101184
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370567703109328,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4915507482810457,
+ "tokens_seen": 1787166720
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705516549648947,
+ "loss": 1.2395,
+ "theoretical_loss": 3.4915399244455085,
+ "tokens_seen": 1787232256
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705356068204614,
+ "loss": 1.2051,
+ "theoretical_loss": 3.4915291011179903,
+ "tokens_seen": 1787297792
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705195586760281,
+ "loss": 1.2563,
+ "theoretical_loss": 3.491518278298448,
+ "tokens_seen": 1787363328
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003705035105315948,
+ "loss": 1.229,
+ "theoretical_loss": 3.4915074559868398,
+ "tokens_seen": 1787428864
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003704874623871615,
+ "loss": 1.3566,
+ "theoretical_loss": 3.4914966341831235,
+ "tokens_seen": 1787494400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003704714142427282,
+ "loss": 1.2101,
+ "theoretical_loss": 3.491485812887256,
+ "tokens_seen": 1787559936
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003704553660982949,
+ "loss": 1.3985,
+ "theoretical_loss": 3.491474992099196,
+ "tokens_seen": 1787625472
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003704393179538616,
+ "loss": 1.269,
+ "theoretical_loss": 3.4914641718188992,
+ "tokens_seen": 1787691008
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003704232698094283,
+ "loss": 1.2999,
+ "theoretical_loss": 3.4914533520463245,
+ "tokens_seen": 1787756544
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370407221664995,
+ "loss": 1.2251,
+ "theoretical_loss": 3.491442532781429,
+ "tokens_seen": 1787822080
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703911735205617,
+ "loss": 1.2227,
+ "theoretical_loss": 3.49143171402417,
+ "tokens_seen": 1787887616
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703751253761284,
+ "loss": 1.2596,
+ "theoretical_loss": 3.491420895774506,
+ "tokens_seen": 1787953152
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703590772316951,
+ "loss": 1.2984,
+ "theoretical_loss": 3.491410078032394,
+ "tokens_seen": 1788018688
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703430290872618,
+ "loss": 1.2125,
+ "theoretical_loss": 3.4913992607977917,
+ "tokens_seen": 1788084224
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703269809428285,
+ "loss": 1.2306,
+ "theoretical_loss": 3.4913884440706564,
+ "tokens_seen": 1788149760
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003703109327983952,
+ "loss": 1.3528,
+ "theoretical_loss": 3.4913776278509463,
+ "tokens_seen": 1788215296
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003702948846539619,
+ "loss": 1.2936,
+ "theoretical_loss": 3.491366812138618,
+ "tokens_seen": 1788280832
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003702788365095286,
+ "loss": 1.2766,
+ "theoretical_loss": 3.49135599693363,
+ "tokens_seen": 1788346368
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003702627883650953,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4913451822359396,
+ "tokens_seen": 1788411904
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370246740220662,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4913343680455045,
+ "tokens_seen": 1788477440
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003702306920762287,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4913235543622823,
+ "tokens_seen": 1788542976
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003702146439317954,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4913127411862304,
+ "tokens_seen": 1788608512
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701985957873621,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4913019285173066,
+ "tokens_seen": 1788674048
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701825476429288,
+ "loss": 1.3148,
+ "theoretical_loss": 3.4912911163554687,
+ "tokens_seen": 1788739584
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701664994984955,
+ "loss": 1.303,
+ "theoretical_loss": 3.491280304700674,
+ "tokens_seen": 1788805120
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701504513540622,
+ "loss": 1.269,
+ "theoretical_loss": 3.491269493552881,
+ "tokens_seen": 1788870656
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701344032096289,
+ "loss": 1.2141,
+ "theoretical_loss": 3.491258682912046,
+ "tokens_seen": 1788936192
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701183550651956,
+ "loss": 1.2542,
+ "theoretical_loss": 3.491247872778128,
+ "tokens_seen": 1789001728
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003701023069207623,
+ "loss": 1.2917,
+ "theoretical_loss": 3.4912370631510834,
+ "tokens_seen": 1789067264
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 599715,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.152665138244629,
+ "objective/train/theoretical_loss": 3.491226254030871,
+ "objective/train/tokens_used": 1809592800,
+ "theoretical_loss": 3.491226254030871,
+ "tokens_seen": 1789132800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000370086258776329,
+ "loss": 1.1846,
+ "theoretical_loss": 3.491226254030871,
+ "tokens_seen": 1789132800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003700702106318957,
+ "loss": 1.3383,
+ "theoretical_loss": 3.4912154454174478,
+ "tokens_seen": 1789198336
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037005416248746243,
+ "loss": 1.215,
+ "theoretical_loss": 3.4912046373107715,
+ "tokens_seen": 1789263872
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037003811434302913,
+ "loss": 1.2956,
+ "theoretical_loss": 3.4911938297108,
+ "tokens_seen": 1789329408
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037002206619859583,
+ "loss": 1.248,
+ "theoretical_loss": 3.491183022617491,
+ "tokens_seen": 1789394944
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00037000601805416253,
+ "loss": 1.2806,
+ "theoretical_loss": 3.491172216030802,
+ "tokens_seen": 1789460480
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036998996990972923,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4911614099506907,
+ "tokens_seen": 1789526016
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036997392176529593,
+ "loss": 1.2419,
+ "theoretical_loss": 3.4911506043771148,
+ "tokens_seen": 1789591552
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036995787362086263,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4911397993100324,
+ "tokens_seen": 1789657088
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003699418254764293,
+ "loss": 1.3144,
+ "theoretical_loss": 3.491128994749401,
+ "tokens_seen": 1789722624
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000369925777331996,
+ "loss": 1.4211,
+ "theoretical_loss": 3.491118190695178,
+ "tokens_seen": 1789788160
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003699097291875627,
+ "loss": 1.366,
+ "theoretical_loss": 3.491107387147321,
+ "tokens_seen": 1789853696
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698936810431294,
+ "loss": 1.3039,
+ "theoretical_loss": 3.4910965841057884,
+ "tokens_seen": 1789919232
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698776328986961,
+ "loss": 1.4293,
+ "theoretical_loss": 3.491085781570538,
+ "tokens_seen": 1789984768
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698615847542628,
+ "loss": 1.2793,
+ "theoretical_loss": 3.491074979541527,
+ "tokens_seen": 1790050304
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698455366098295,
+ "loss": 1.3377,
+ "theoretical_loss": 3.4910641780187133,
+ "tokens_seen": 1790115840
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698294884653962,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4910533770020544,
+ "tokens_seen": 1790181376
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003698134403209629,
+ "loss": 1.289,
+ "theoretical_loss": 3.491042576491508,
+ "tokens_seen": 1790246912
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697973921765296,
+ "loss": 1.3521,
+ "theoretical_loss": 3.491031776487033,
+ "tokens_seen": 1790312448
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697813440320963,
+ "loss": 1.3469,
+ "theoretical_loss": 3.491020976988586,
+ "tokens_seen": 1790377984
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000369765295887663,
+ "loss": 1.2454,
+ "theoretical_loss": 3.4910101779961247,
+ "tokens_seen": 1790443520
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697492477432297,
+ "loss": 1.1692,
+ "theoretical_loss": 3.4909993795096077,
+ "tokens_seen": 1790509056
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697331995987964,
+ "loss": 1.3322,
+ "theoretical_loss": 3.4909885815289923,
+ "tokens_seen": 1790574592
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697171514543631,
+ "loss": 1.2742,
+ "theoretical_loss": 3.490977784054236,
+ "tokens_seen": 1790640128
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003697011033099298,
+ "loss": 1.3179,
+ "theoretical_loss": 3.4909669870852973,
+ "tokens_seen": 1790705664
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003696850551654965,
+ "loss": 1.2333,
+ "theoretical_loss": 3.490956190622134,
+ "tokens_seen": 1790771200
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003696690070210632,
+ "loss": 1.3423,
+ "theoretical_loss": 3.4909453946647027,
+ "tokens_seen": 1790836736
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003696529588766299,
+ "loss": 1.3452,
+ "theoretical_loss": 3.490934599212962,
+ "tokens_seen": 1790902272
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003696369107321966,
+ "loss": 1.2635,
+ "theoretical_loss": 3.4909238042668704,
+ "tokens_seen": 1790967808
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003696208625877633,
+ "loss": 1.3571,
+ "theoretical_loss": 3.490913009826385,
+ "tokens_seen": 1791033344
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036960481444333,
+ "loss": 1.3289,
+ "theoretical_loss": 3.490902215891463,
+ "tokens_seen": 1791098880
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695887662988967,
+ "loss": 1.2266,
+ "theoretical_loss": 3.4908914224620635,
+ "tokens_seen": 1791164416
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695727181544634,
+ "loss": 1.3161,
+ "theoretical_loss": 3.490880629538143,
+ "tokens_seen": 1791229952
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695566700100301,
+ "loss": 1.2959,
+ "theoretical_loss": 3.4908698371196607,
+ "tokens_seen": 1791295488
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695406218655968,
+ "loss": 1.1958,
+ "theoretical_loss": 3.490859045206574,
+ "tokens_seen": 1791361024
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695245737211635,
+ "loss": 1.3658,
+ "theoretical_loss": 3.4908482537988403,
+ "tokens_seen": 1791426560
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003695085255767302,
+ "loss": 1.3027,
+ "theoretical_loss": 3.490837462896417,
+ "tokens_seen": 1791492096
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003694924774322969,
+ "loss": 1.3298,
+ "theoretical_loss": 3.4908266724992636,
+ "tokens_seen": 1791557632
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003694764292878636,
+ "loss": 1.2188,
+ "theoretical_loss": 3.4908158826073366,
+ "tokens_seen": 1791623168
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003694603811434303,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4908050932205943,
+ "tokens_seen": 1791688704
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000369444332998997,
+ "loss": 1.2216,
+ "theoretical_loss": 3.490794304338994,
+ "tokens_seen": 1791754240
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003694282848545637,
+ "loss": 1.2462,
+ "theoretical_loss": 3.490783515962495,
+ "tokens_seen": 1791819776
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003694122367101304,
+ "loss": 1.2614,
+ "theoretical_loss": 3.490772728091054,
+ "tokens_seen": 1791885312
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003693961885656971,
+ "loss": 1.2888,
+ "theoretical_loss": 3.490761940724629,
+ "tokens_seen": 1791950848
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003693801404212638,
+ "loss": 1.3756,
+ "theoretical_loss": 3.4907511538631777,
+ "tokens_seen": 1792016384
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036936409227683055,
+ "loss": 1.1894,
+ "theoretical_loss": 3.4907403675066595,
+ "tokens_seen": 1792081920
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036934804413239726,
+ "loss": 1.4534,
+ "theoretical_loss": 3.49072958165503,
+ "tokens_seen": 1792147456
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036933199598796396,
+ "loss": 1.3039,
+ "theoretical_loss": 3.490718796308249,
+ "tokens_seen": 1792212992
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036931594784353066,
+ "loss": 1.2747,
+ "theoretical_loss": 3.490708011466273,
+ "tokens_seen": 1792278528
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692998996990973,
+ "loss": 1.3275,
+ "theoretical_loss": 3.4906972271290613,
+ "tokens_seen": 1792344064
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 601014,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2282359600067139,
+ "objective/train/theoretical_loss": 3.4906864432965707,
+ "objective/train/tokens_used": 1812869600,
+ "theoretical_loss": 3.4906864432965707,
+ "tokens_seen": 1792409600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000369283851554664,
+ "loss": 1.2805,
+ "theoretical_loss": 3.4906864432965707,
+ "tokens_seen": 1792409600
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692678034102307,
+ "loss": 1.4108,
+ "theoretical_loss": 3.4906756599687596,
+ "tokens_seen": 1792475136
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692517552657974,
+ "loss": 1.2863,
+ "theoretical_loss": 3.490664877145586,
+ "tokens_seen": 1792540672
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692357071213641,
+ "loss": 1.329,
+ "theoretical_loss": 3.4906540948270077,
+ "tokens_seen": 1792606208
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692196589769308,
+ "loss": 1.3037,
+ "theoretical_loss": 3.490643313012982,
+ "tokens_seen": 1792671744
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003692036108324975,
+ "loss": 1.2022,
+ "theoretical_loss": 3.4906325317034685,
+ "tokens_seen": 1792737280
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003691875626880642,
+ "loss": 1.3753,
+ "theoretical_loss": 3.4906217508984234,
+ "tokens_seen": 1792802816
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003691715145436309,
+ "loss": 1.3744,
+ "theoretical_loss": 3.490610970597806,
+ "tokens_seen": 1792868352
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003691554663991976,
+ "loss": 1.3385,
+ "theoretical_loss": 3.4906001908015734,
+ "tokens_seen": 1792933888
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003691394182547643,
+ "loss": 1.4016,
+ "theoretical_loss": 3.490589411509683,
+ "tokens_seen": 1792999424
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000369123370110331,
+ "loss": 1.2883,
+ "theoretical_loss": 3.490578632722095,
+ "tokens_seen": 1793064960
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003691073219658977,
+ "loss": 1.3234,
+ "theoretical_loss": 3.490567854438765,
+ "tokens_seen": 1793130496
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690912738214644,
+ "loss": 1.3384,
+ "theoretical_loss": 3.4905570766596523,
+ "tokens_seen": 1793196032
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690752256770311,
+ "loss": 1.4166,
+ "theoretical_loss": 3.4905462993847145,
+ "tokens_seen": 1793261568
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690591775325978,
+ "loss": 1.2816,
+ "theoretical_loss": 3.49053552261391,
+ "tokens_seen": 1793327104
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690431293881645,
+ "loss": 1.2996,
+ "theoretical_loss": 3.4905247463471953,
+ "tokens_seen": 1793392640
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690270812437312,
+ "loss": 1.3015,
+ "theoretical_loss": 3.490513970584531,
+ "tokens_seen": 1793458176
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003690110330992979,
+ "loss": 1.3316,
+ "theoretical_loss": 3.4905031953258723,
+ "tokens_seen": 1793523712
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003689949849548646,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4904924205711794,
+ "tokens_seen": 1793589248
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003689789368104313,
+ "loss": 1.2784,
+ "theoretical_loss": 3.490481646320409,
+ "tokens_seen": 1793654784
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000368962888665998,
+ "loss": 1.2901,
+ "theoretical_loss": 3.4904708725735203,
+ "tokens_seen": 1793720320
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003689468405215647,
+ "loss": 1.3297,
+ "theoretical_loss": 3.4904600993304697,
+ "tokens_seen": 1793785856
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003689307923771314,
+ "loss": 1.3194,
+ "theoretical_loss": 3.4904493265912166,
+ "tokens_seen": 1793851392
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003689147442326981,
+ "loss": 1.3115,
+ "theoretical_loss": 3.4904385543557184,
+ "tokens_seen": 1793916928
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688986960882648,
+ "loss": 1.311,
+ "theoretical_loss": 3.490427782623933,
+ "tokens_seen": 1793982464
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688826479438315,
+ "loss": 1.2727,
+ "theoretical_loss": 3.4904170113958193,
+ "tokens_seen": 1794048000
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688665997993982,
+ "loss": 1.2672,
+ "theoretical_loss": 3.490406240671335,
+ "tokens_seen": 1794113536
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688505516549649,
+ "loss": 1.2585,
+ "theoretical_loss": 3.4903954704504376,
+ "tokens_seen": 1794179072
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688345035105316,
+ "loss": 1.3117,
+ "theoretical_loss": 3.4903847007330855,
+ "tokens_seen": 1794244608
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003688184553660983,
+ "loss": 1.2732,
+ "theoretical_loss": 3.490373931519237,
+ "tokens_seen": 1794310144
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000368802407221665,
+ "loss": 1.2919,
+ "theoretical_loss": 3.49036316280885,
+ "tokens_seen": 1794375680
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687863590772317,
+ "loss": 1.2599,
+ "theoretical_loss": 3.4903523946018824,
+ "tokens_seen": 1794441216
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687703109327984,
+ "loss": 1.3624,
+ "theoretical_loss": 3.4903416268982923,
+ "tokens_seen": 1794506752
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687542627883651,
+ "loss": 1.1819,
+ "theoretical_loss": 3.490330859698038,
+ "tokens_seen": 1794572288
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687382146439318,
+ "loss": 1.3663,
+ "theoretical_loss": 3.4903200930010776,
+ "tokens_seen": 1794637824
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687221664994985,
+ "loss": 1.1524,
+ "theoretical_loss": 3.4903093268073686,
+ "tokens_seen": 1794703360
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003687061183550652,
+ "loss": 1.3235,
+ "theoretical_loss": 3.4902985611168704,
+ "tokens_seen": 1794768896
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036869007021063193,
+ "loss": 1.3086,
+ "theoretical_loss": 3.4902877959295395,
+ "tokens_seen": 1794834432
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036867402206619863,
+ "loss": 1.3833,
+ "theoretical_loss": 3.4902770312453355,
+ "tokens_seen": 1794899968
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036865797392176533,
+ "loss": 1.2455,
+ "theoretical_loss": 3.4902662670642157,
+ "tokens_seen": 1794965504
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036864192577733203,
+ "loss": 1.3019,
+ "theoretical_loss": 3.490255503386138,
+ "tokens_seen": 1795031040
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036862587763289873,
+ "loss": 1.3645,
+ "theoretical_loss": 3.490244740211061,
+ "tokens_seen": 1795096576
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036860982948846543,
+ "loss": 1.3092,
+ "theoretical_loss": 3.490233977538943,
+ "tokens_seen": 1795162112
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036859378134403213,
+ "loss": 1.3018,
+ "theoretical_loss": 3.490223215369742,
+ "tokens_seen": 1795227648
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036857773319959883,
+ "loss": 1.2883,
+ "theoretical_loss": 3.4902124537034154,
+ "tokens_seen": 1795293184
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036856168505516553,
+ "loss": 1.2532,
+ "theoretical_loss": 3.490201692539922,
+ "tokens_seen": 1795358720
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036854563691073223,
+ "loss": 1.2823,
+ "theoretical_loss": 3.49019093187922,
+ "tokens_seen": 1795424256
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003685295887662989,
+ "loss": 1.3471,
+ "theoretical_loss": 3.490180171721268,
+ "tokens_seen": 1795489792
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003685135406218656,
+ "loss": 1.3271,
+ "theoretical_loss": 3.490169412066023,
+ "tokens_seen": 1795555328
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684974924774323,
+ "loss": 1.3352,
+ "theoretical_loss": 3.4901586529134443,
+ "tokens_seen": 1795620864
+ },
+ {
+ "epoch": 0.54,
+ "objective/train/docs_used": 601905,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3266921043395996,
+ "objective/train/theoretical_loss": 3.490147894263489,
+ "objective/train/tokens_used": 1816146400,
+ "theoretical_loss": 3.490147894263489,
+ "tokens_seen": 1795686400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000368481444332999,
+ "loss": 1.2901,
+ "theoretical_loss": 3.490147894263489,
+ "tokens_seen": 1795686400
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684653961885657,
+ "loss": 1.1648,
+ "theoretical_loss": 3.490137136116116,
+ "tokens_seen": 1795751936
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684493480441324,
+ "loss": 1.2923,
+ "theoretical_loss": 3.4901263784712837,
+ "tokens_seen": 1795817472
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684332998996991,
+ "loss": 1.3094,
+ "theoretical_loss": 3.4901156213289495,
+ "tokens_seen": 1795883008
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684172517552658,
+ "loss": 1.3039,
+ "theoretical_loss": 3.490104864689072,
+ "tokens_seen": 1795948544
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003684012036108325,
+ "loss": 1.3145,
+ "theoretical_loss": 3.4900941085516095,
+ "tokens_seen": 1796014080
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003683851554663992,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4900833529165194,
+ "tokens_seen": 1796079616
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003683691073219659,
+ "loss": 1.3541,
+ "theoretical_loss": 3.4900725977837617,
+ "tokens_seen": 1796145152
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003683530591775326,
+ "loss": 1.3685,
+ "theoretical_loss": 3.490061843153293,
+ "tokens_seen": 1796210688
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003683370110330993,
+ "loss": 1.3124,
+ "theoretical_loss": 3.490051089025072,
+ "tokens_seen": 1796276224
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000368320962888666,
+ "loss": 1.2261,
+ "theoretical_loss": 3.4900403353990566,
+ "tokens_seen": 1796341760
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003683049147442327,
+ "loss": 1.2574,
+ "theoretical_loss": 3.490029582275206,
+ "tokens_seen": 1796407296
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682888665997994,
+ "loss": 1.1785,
+ "theoretical_loss": 3.4900188296534775,
+ "tokens_seen": 1796472832
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682728184553661,
+ "loss": 1.2935,
+ "theoretical_loss": 3.4900080775338296,
+ "tokens_seen": 1796538368
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682567703109328,
+ "loss": 1.2904,
+ "theoretical_loss": 3.489997325916221,
+ "tokens_seen": 1796603904
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682407221664995,
+ "loss": 1.3337,
+ "theoretical_loss": 3.489986574800609,
+ "tokens_seen": 1796669440
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682246740220662,
+ "loss": 1.2769,
+ "theoretical_loss": 3.4899758241869527,
+ "tokens_seen": 1796734976
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003682086258776329,
+ "loss": 1.3017,
+ "theoretical_loss": 3.4899650740752097,
+ "tokens_seen": 1796800512
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003681925777331996,
+ "loss": 1.3803,
+ "theoretical_loss": 3.489954324465339,
+ "tokens_seen": 1796866048
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003681765295887663,
+ "loss": 1.3225,
+ "theoretical_loss": 3.489943575357298,
+ "tokens_seen": 1796931584
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036816048144433305,
+ "loss": 1.2369,
+ "theoretical_loss": 3.489932826751046,
+ "tokens_seen": 1796997120
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036814443329989975,
+ "loss": 1.3214,
+ "theoretical_loss": 3.4899220786465404,
+ "tokens_seen": 1797062656
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036812838515546645,
+ "loss": 1.2718,
+ "theoretical_loss": 3.4899113310437397,
+ "tokens_seen": 1797128192
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036811233701103315,
+ "loss": 1.363,
+ "theoretical_loss": 3.4899005839426023,
+ "tokens_seen": 1797193728
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036809628886659985,
+ "loss": 1.227,
+ "theoretical_loss": 3.4898898373430867,
+ "tokens_seen": 1797259264
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036808024072216655,
+ "loss": 1.319,
+ "theoretical_loss": 3.489879091245151,
+ "tokens_seen": 1797324800
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036806419257773325,
+ "loss": 1.3035,
+ "theoretical_loss": 3.489868345648753,
+ "tokens_seen": 1797390336
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036804814443329995,
+ "loss": 1.2971,
+ "theoretical_loss": 3.4898576005538517,
+ "tokens_seen": 1797455872
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036803209628886665,
+ "loss": 1.3193,
+ "theoretical_loss": 3.4898468559604057,
+ "tokens_seen": 1797521408
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036801604814443335,
+ "loss": 1.3089,
+ "theoretical_loss": 3.4898361118683723,
+ "tokens_seen": 1797586944
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036800000000000005,
+ "loss": 1.2996,
+ "theoretical_loss": 3.4898253682777103,
+ "tokens_seen": 1797652480
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036798395185556676,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4898146251883784,
+ "tokens_seen": 1797718016
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036796790371113346,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4898038826003344,
+ "tokens_seen": 1797783552
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036795185556670016,
+ "loss": 1.2822,
+ "theoretical_loss": 3.4897931405135365,
+ "tokens_seen": 1797849088
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036793580742226686,
+ "loss": 1.3579,
+ "theoretical_loss": 3.489782398927944,
+ "tokens_seen": 1797914624
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.00036791975927783356,
+ "loss": 1.3209,
+ "theoretical_loss": 3.489771657843514,
+ "tokens_seen": 1797980160
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003679037111334002,
+ "loss": 1.2417,
+ "theoretical_loss": 3.489760917260206,
+ "tokens_seen": 1798045696
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003678876629889669,
+ "loss": 1.2935,
+ "theoretical_loss": 3.489750177177977,
+ "tokens_seen": 1798111232
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003678716148445336,
+ "loss": 1.2485,
+ "theoretical_loss": 3.4897394375967874,
+ "tokens_seen": 1798176768
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003678555667001003,
+ "loss": 1.2728,
+ "theoretical_loss": 3.4897286985165934,
+ "tokens_seen": 1798242304
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.000367839518555667,
+ "loss": 1.4074,
+ "theoretical_loss": 3.4897179599373547,
+ "tokens_seen": 1798307840
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003678234704112337,
+ "loss": 1.3086,
+ "theoretical_loss": 3.489707221859029,
+ "tokens_seen": 1798373376
+ },
+ {
+ "epoch": 0.54,
+ "learning_rate": 0.0003678074222668004,
+ "loss": 1.3222,
+ "theoretical_loss": 3.489696484281575,
+ "tokens_seen": 1798438912
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677913741223671,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4896857472049514,
+ "tokens_seen": 1798504448
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677753259779338,
+ "loss": 1.2677,
+ "theoretical_loss": 3.4896750106291163,
+ "tokens_seen": 1798569984
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677592778335005,
+ "loss": 1.3476,
+ "theoretical_loss": 3.489664274554028,
+ "tokens_seen": 1798635520
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677432296890672,
+ "loss": 1.3758,
+ "theoretical_loss": 3.4896535389796446,
+ "tokens_seen": 1798701056
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677271815446339,
+ "loss": 1.2741,
+ "theoretical_loss": 3.489642803905925,
+ "tokens_seen": 1798766592
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003677111334002006,
+ "loss": 1.3326,
+ "theoretical_loss": 3.4896320693328278,
+ "tokens_seen": 1798832128
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003676950852557673,
+ "loss": 1.3004,
+ "theoretical_loss": 3.4896213352603107,
+ "tokens_seen": 1798897664
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 603132,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2088148593902588,
+ "objective/train/theoretical_loss": 3.489610601688333,
+ "objective/train/tokens_used": 1819423200,
+ "theoretical_loss": 3.489610601688333,
+ "tokens_seen": 1798963200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000367679037111334,
+ "loss": 1.2736,
+ "theoretical_loss": 3.489610601688333,
+ "tokens_seen": 1798963200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003676629889669007,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4895998686168515,
+ "tokens_seen": 1799028736
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003676469408224674,
+ "loss": 1.4063,
+ "theoretical_loss": 3.4895891360458267,
+ "tokens_seen": 1799094272
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003676308926780341,
+ "loss": 1.3199,
+ "theoretical_loss": 3.489578403975216,
+ "tokens_seen": 1799159808
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003676148445336008,
+ "loss": 1.2766,
+ "theoretical_loss": 3.489567672404978,
+ "tokens_seen": 1799225344
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675987963891675,
+ "loss": 1.2983,
+ "theoretical_loss": 3.4895569413350707,
+ "tokens_seen": 1799290880
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675827482447342,
+ "loss": 1.3561,
+ "theoretical_loss": 3.4895462107654533,
+ "tokens_seen": 1799356416
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675667001003009,
+ "loss": 1.2474,
+ "theoretical_loss": 3.489535480696084,
+ "tokens_seen": 1799421952
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675506519558676,
+ "loss": 1.3403,
+ "theoretical_loss": 3.489524751126921,
+ "tokens_seen": 1799487488
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675346038114343,
+ "loss": 1.2994,
+ "theoretical_loss": 3.4895140220579224,
+ "tokens_seen": 1799553024
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000367518555667001,
+ "loss": 1.1748,
+ "theoretical_loss": 3.4895032934890478,
+ "tokens_seen": 1799618560
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003675025075225677,
+ "loss": 1.2566,
+ "theoretical_loss": 3.489492565420255,
+ "tokens_seen": 1799684096
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674864593781344,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4894818378515025,
+ "tokens_seen": 1799749632
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674704112337011,
+ "loss": 1.2793,
+ "theoretical_loss": 3.489471110782749,
+ "tokens_seen": 1799815168
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674543630892678,
+ "loss": 1.2856,
+ "theoretical_loss": 3.4894603842139524,
+ "tokens_seen": 1799880704
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674383149448345,
+ "loss": 1.2609,
+ "theoretical_loss": 3.489449658145072,
+ "tokens_seen": 1799946240
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674222668004012,
+ "loss": 1.2692,
+ "theoretical_loss": 3.489438932576066,
+ "tokens_seen": 1800011776
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003674062186559679,
+ "loss": 1.2663,
+ "theoretical_loss": 3.4894282075068923,
+ "tokens_seen": 1800077312
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003673901705115346,
+ "loss": 1.2696,
+ "theoretical_loss": 3.48941748293751,
+ "tokens_seen": 1800142848
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003673741223671013,
+ "loss": 1.2318,
+ "theoretical_loss": 3.489406758867878,
+ "tokens_seen": 1800208384
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000367358074222668,
+ "loss": 1.2533,
+ "theoretical_loss": 3.489396035297954,
+ "tokens_seen": 1800273920
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003673420260782347,
+ "loss": 1.1788,
+ "theoretical_loss": 3.4893853122276974,
+ "tokens_seen": 1800339456
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003673259779338014,
+ "loss": 1.1501,
+ "theoretical_loss": 3.489374589657066,
+ "tokens_seen": 1800404992
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036730992978936813,
+ "loss": 1.2524,
+ "theoretical_loss": 3.489363867586018,
+ "tokens_seen": 1800470528
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036729388164493483,
+ "loss": 1.3698,
+ "theoretical_loss": 3.4893531460145133,
+ "tokens_seen": 1800536064
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036727783350050153,
+ "loss": 1.2802,
+ "theoretical_loss": 3.4893424249425093,
+ "tokens_seen": 1800601600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036726178535606823,
+ "loss": 1.2505,
+ "theoretical_loss": 3.489331704369965,
+ "tokens_seen": 1800667136
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036724573721163493,
+ "loss": 1.2847,
+ "theoretical_loss": 3.489320984296839,
+ "tokens_seen": 1800732672
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036722968906720163,
+ "loss": 1.299,
+ "theoretical_loss": 3.4893102647230894,
+ "tokens_seen": 1800798208
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036721364092276833,
+ "loss": 1.2856,
+ "theoretical_loss": 3.4892995456486746,
+ "tokens_seen": 1800863744
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036719759277833503,
+ "loss": 1.2903,
+ "theoretical_loss": 3.4892888270735547,
+ "tokens_seen": 1800929280
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036718154463390173,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4892781089976865,
+ "tokens_seen": 1800994816
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036716549648946843,
+ "loss": 1.3398,
+ "theoretical_loss": 3.4892673914210297,
+ "tokens_seen": 1801060352
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036714944834503513,
+ "loss": 1.2801,
+ "theoretical_loss": 3.489256674343542,
+ "tokens_seen": 1801125888
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003671334002006018,
+ "loss": 1.3084,
+ "theoretical_loss": 3.489245957765183,
+ "tokens_seen": 1801191424
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003671173520561685,
+ "loss": 1.357,
+ "theoretical_loss": 3.4892352416859107,
+ "tokens_seen": 1801256960
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003671013039117352,
+ "loss": 1.3455,
+ "theoretical_loss": 3.4892245261056836,
+ "tokens_seen": 1801322496
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003670852557673019,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4892138110244604,
+ "tokens_seen": 1801388032
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003670692076228686,
+ "loss": 1.3372,
+ "theoretical_loss": 3.4892030964422003,
+ "tokens_seen": 1801453568
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003670531594784353,
+ "loss": 1.3567,
+ "theoretical_loss": 3.489192382358861,
+ "tokens_seen": 1801519104
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000367037111334002,
+ "loss": 1.3404,
+ "theoretical_loss": 3.4891816687744015,
+ "tokens_seen": 1801584640
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003670210631895687,
+ "loss": 1.4004,
+ "theoretical_loss": 3.48917095568878,
+ "tokens_seen": 1801650176
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003670050150451354,
+ "loss": 1.3492,
+ "theoretical_loss": 3.4891602431019564,
+ "tokens_seen": 1801715712
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003669889669007021,
+ "loss": 1.2796,
+ "theoretical_loss": 3.4891495310138883,
+ "tokens_seen": 1801781248
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003669729187562688,
+ "loss": 1.2911,
+ "theoretical_loss": 3.4891388194245345,
+ "tokens_seen": 1801846784
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036695687061183554,
+ "loss": 1.2793,
+ "theoretical_loss": 3.489128108333854,
+ "tokens_seen": 1801912320
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036694082246740224,
+ "loss": 1.2687,
+ "theoretical_loss": 3.489117397741804,
+ "tokens_seen": 1801977856
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036692477432296894,
+ "loss": 1.3156,
+ "theoretical_loss": 3.4891066876483454,
+ "tokens_seen": 1802043392
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036690872617853564,
+ "loss": 1.2252,
+ "theoretical_loss": 3.4890959780534354,
+ "tokens_seen": 1802108928
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036689267803410234,
+ "loss": 1.2341,
+ "theoretical_loss": 3.489085268957033,
+ "tokens_seen": 1802174464
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 604041,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2103506326675415,
+ "objective/train/theoretical_loss": 3.489074560359097,
+ "objective/train/tokens_used": 1822700000,
+ "theoretical_loss": 3.489074560359097,
+ "tokens_seen": 1802240000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036687662988966904,
+ "loss": 1.221,
+ "theoretical_loss": 3.489074560359097,
+ "tokens_seen": 1802240000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036686058174523575,
+ "loss": 1.2555,
+ "theoretical_loss": 3.489063852259586,
+ "tokens_seen": 1802305536
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036684453360080245,
+ "loss": 1.3352,
+ "theoretical_loss": 3.4890531446584583,
+ "tokens_seen": 1802371072
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036682848545636915,
+ "loss": 1.3494,
+ "theoretical_loss": 3.489042437555673,
+ "tokens_seen": 1802436608
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036681243731193585,
+ "loss": 1.3403,
+ "theoretical_loss": 3.489031730951189,
+ "tokens_seen": 1802502144
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036679638916750255,
+ "loss": 1.2712,
+ "theoretical_loss": 3.4890210248449645,
+ "tokens_seen": 1802567680
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036678034102306925,
+ "loss": 1.2242,
+ "theoretical_loss": 3.489010319236958,
+ "tokens_seen": 1802633216
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036676429287863595,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4889996141271293,
+ "tokens_seen": 1802698752
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036674824473420265,
+ "loss": 1.2561,
+ "theoretical_loss": 3.4889889095154363,
+ "tokens_seen": 1802764288
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036673219658976935,
+ "loss": 1.3386,
+ "theoretical_loss": 3.4889782054018377,
+ "tokens_seen": 1802829824
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036671614844533605,
+ "loss": 1.1877,
+ "theoretical_loss": 3.4889675017862922,
+ "tokens_seen": 1802895360
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036670010030090275,
+ "loss": 1.4449,
+ "theoretical_loss": 3.4889567986687586,
+ "tokens_seen": 1802960896
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036668405215646945,
+ "loss": 1.2252,
+ "theoretical_loss": 3.488946096049196,
+ "tokens_seen": 1803026432
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036666800401203615,
+ "loss": 1.3736,
+ "theoretical_loss": 3.488935393927563,
+ "tokens_seen": 1803091968
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036665195586760285,
+ "loss": 1.3421,
+ "theoretical_loss": 3.4889246923038177,
+ "tokens_seen": 1803157504
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036663590772316955,
+ "loss": 1.2742,
+ "theoretical_loss": 3.4889139911779194,
+ "tokens_seen": 1803223040
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036661985957873625,
+ "loss": 1.3049,
+ "theoretical_loss": 3.4889032905498274,
+ "tokens_seen": 1803288576
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036660381143430296,
+ "loss": 1.2682,
+ "theoretical_loss": 3.488892590419499,
+ "tokens_seen": 1803354112
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036658776328986966,
+ "loss": 1.2359,
+ "theoretical_loss": 3.488881890786894,
+ "tokens_seen": 1803419648
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036657171514543636,
+ "loss": 1.2045,
+ "theoretical_loss": 3.488871191651971,
+ "tokens_seen": 1803485184
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036655566700100306,
+ "loss": 1.2634,
+ "theoretical_loss": 3.4888604930146885,
+ "tokens_seen": 1803550720
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036653961885656976,
+ "loss": 1.2235,
+ "theoretical_loss": 3.4888497948750055,
+ "tokens_seen": 1803616256
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036652357071213646,
+ "loss": 1.2785,
+ "theoretical_loss": 3.488839097232881,
+ "tokens_seen": 1803681792
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003665075225677031,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4888284000882734,
+ "tokens_seen": 1803747328
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664914744232698,
+ "loss": 1.2322,
+ "theoretical_loss": 3.488817703441141,
+ "tokens_seen": 1803812864
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664754262788365,
+ "loss": 1.1507,
+ "theoretical_loss": 3.4888070072914443,
+ "tokens_seen": 1803878400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664593781344032,
+ "loss": 1.2894,
+ "theoretical_loss": 3.48879631163914,
+ "tokens_seen": 1803943936
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664433299899699,
+ "loss": 1.296,
+ "theoretical_loss": 3.4887856164841886,
+ "tokens_seen": 1804009472
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664272818455366,
+ "loss": 1.2533,
+ "theoretical_loss": 3.488774921826548,
+ "tokens_seen": 1804075008
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003664112337011033,
+ "loss": 1.2278,
+ "theoretical_loss": 3.488764227666177,
+ "tokens_seen": 1804140544
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036639518555667,
+ "loss": 1.1999,
+ "theoretical_loss": 3.4887535340030347,
+ "tokens_seen": 1804206080
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003663791374122367,
+ "loss": 1.3648,
+ "theoretical_loss": 3.48874284083708,
+ "tokens_seen": 1804271616
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003663630892678034,
+ "loss": 1.2293,
+ "theoretical_loss": 3.488732148168272,
+ "tokens_seen": 1804337152
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003663470411233701,
+ "loss": 1.2551,
+ "theoretical_loss": 3.4887214559965685,
+ "tokens_seen": 1804402688
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003663309929789368,
+ "loss": 1.2348,
+ "theoretical_loss": 3.488710764321929,
+ "tokens_seen": 1804468224
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003663149448345035,
+ "loss": 1.3269,
+ "theoretical_loss": 3.4887000731443125,
+ "tokens_seen": 1804533760
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662988966900702,
+ "loss": 1.1643,
+ "theoretical_loss": 3.4886893824636775,
+ "tokens_seen": 1804599296
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662828485456369,
+ "loss": 1.2309,
+ "theoretical_loss": 3.488678692279983,
+ "tokens_seen": 1804664832
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662668004012036,
+ "loss": 1.1484,
+ "theoretical_loss": 3.4886680025931875,
+ "tokens_seen": 1804730368
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662507522567703,
+ "loss": 1.2447,
+ "theoretical_loss": 3.488657313403251,
+ "tokens_seen": 1804795904
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000366234704112337,
+ "loss": 1.2493,
+ "theoretical_loss": 3.488646624710131,
+ "tokens_seen": 1804861440
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662186559679037,
+ "loss": 1.2746,
+ "theoretical_loss": 3.488635936513787,
+ "tokens_seen": 1804926976
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003662026078234704,
+ "loss": 1.2255,
+ "theoretical_loss": 3.488625248814178,
+ "tokens_seen": 1804992512
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661865596790371,
+ "loss": 1.2239,
+ "theoretical_loss": 3.4886145616112625,
+ "tokens_seen": 1805058048
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661705115346038,
+ "loss": 1.2817,
+ "theoretical_loss": 3.4886038749049995,
+ "tokens_seen": 1805123584
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661544633901705,
+ "loss": 1.1934,
+ "theoretical_loss": 3.488593188695348,
+ "tokens_seen": 1805189120
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661384152457372,
+ "loss": 1.165,
+ "theoretical_loss": 3.488582502982267,
+ "tokens_seen": 1805254656
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661223671013039,
+ "loss": 1.3509,
+ "theoretical_loss": 3.4885718177657155,
+ "tokens_seen": 1805320192
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003661063189568706,
+ "loss": 1.2895,
+ "theoretical_loss": 3.4885611330456516,
+ "tokens_seen": 1805385728
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003660902708124373,
+ "loss": 1.3409,
+ "theoretical_loss": 3.488550448822035,
+ "tokens_seen": 1805451264
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 605221,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3439946174621582,
+ "objective/train/theoretical_loss": 3.4885397650948247,
+ "objective/train/tokens_used": 1825976800,
+ "theoretical_loss": 3.4885397650948247,
+ "tokens_seen": 1805516800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000366074222668004,
+ "loss": 1.3831,
+ "theoretical_loss": 3.4885397650948247,
+ "tokens_seen": 1805516800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003660581745235707,
+ "loss": 1.1929,
+ "theoretical_loss": 3.488529081863979,
+ "tokens_seen": 1805582336
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003660421263791374,
+ "loss": 1.1917,
+ "theoretical_loss": 3.488518399129457,
+ "tokens_seen": 1805647872
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003660260782347041,
+ "loss": 1.2,
+ "theoretical_loss": 3.4885077168912177,
+ "tokens_seen": 1805713408
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003660100300902708,
+ "loss": 1.172,
+ "theoretical_loss": 3.4884970351492206,
+ "tokens_seen": 1805778944
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003659939819458375,
+ "loss": 1.1592,
+ "theoretical_loss": 3.488486353903424,
+ "tokens_seen": 1805844480
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003659779338014042,
+ "loss": 1.2067,
+ "theoretical_loss": 3.4884756731537867,
+ "tokens_seen": 1805910016
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003659618856569709,
+ "loss": 1.2015,
+ "theoretical_loss": 3.4884649929002682,
+ "tokens_seen": 1805975552
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036594583751253763,
+ "loss": 1.3029,
+ "theoretical_loss": 3.488454313142827,
+ "tokens_seen": 1806041088
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036592978936810433,
+ "loss": 1.2363,
+ "theoretical_loss": 3.488443633881422,
+ "tokens_seen": 1806106624
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036591374122367103,
+ "loss": 1.3062,
+ "theoretical_loss": 3.488432955116013,
+ "tokens_seen": 1806172160
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036589769307923773,
+ "loss": 1.2663,
+ "theoretical_loss": 3.488422276846558,
+ "tokens_seen": 1806237696
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036588164493480443,
+ "loss": 1.2229,
+ "theoretical_loss": 3.4884115990730162,
+ "tokens_seen": 1806303232
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036586559679037113,
+ "loss": 1.2928,
+ "theoretical_loss": 3.488400921795347,
+ "tokens_seen": 1806368768
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036584954864593783,
+ "loss": 1.3814,
+ "theoretical_loss": 3.488390245013509,
+ "tokens_seen": 1806434304
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036583350050150453,
+ "loss": 1.3392,
+ "theoretical_loss": 3.488379568727461,
+ "tokens_seen": 1806499840
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036581745235707123,
+ "loss": 1.2796,
+ "theoretical_loss": 3.488368892937163,
+ "tokens_seen": 1806565376
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036580140421263793,
+ "loss": 1.3161,
+ "theoretical_loss": 3.4883582176425723,
+ "tokens_seen": 1806630912
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036578535606820463,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4883475428436492,
+ "tokens_seen": 1806696448
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036576930792377133,
+ "loss": 1.3398,
+ "theoretical_loss": 3.488336868540353,
+ "tokens_seen": 1806761984
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036575325977933803,
+ "loss": 1.1897,
+ "theoretical_loss": 3.4883261947326414,
+ "tokens_seen": 1806827520
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036573721163490474,
+ "loss": 1.2651,
+ "theoretical_loss": 3.488315521420475,
+ "tokens_seen": 1806893056
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036572116349047144,
+ "loss": 1.2436,
+ "theoretical_loss": 3.4883048486038106,
+ "tokens_seen": 1806958592
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036570511534603814,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4882941762826096,
+ "tokens_seen": 1807024128
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036568906720160484,
+ "loss": 1.3836,
+ "theoretical_loss": 3.4882835044568297,
+ "tokens_seen": 1807089664
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036567301905717154,
+ "loss": 1.2462,
+ "theoretical_loss": 3.4882728331264303,
+ "tokens_seen": 1807155200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036565697091273824,
+ "loss": 1.2729,
+ "theoretical_loss": 3.4882621622913703,
+ "tokens_seen": 1807220736
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036564092276830494,
+ "loss": 1.208,
+ "theoretical_loss": 3.4882514919516088,
+ "tokens_seen": 1807286272
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036562487462387164,
+ "loss": 1.2772,
+ "theoretical_loss": 3.488240822107105,
+ "tokens_seen": 1807351808
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036560882647943834,
+ "loss": 1.258,
+ "theoretical_loss": 3.4882301527578177,
+ "tokens_seen": 1807417344
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036559277833500504,
+ "loss": 1.2259,
+ "theoretical_loss": 3.4882194839037055,
+ "tokens_seen": 1807482880
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036557673019057174,
+ "loss": 1.1876,
+ "theoretical_loss": 3.488208815544729,
+ "tokens_seen": 1807548416
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036556068204613844,
+ "loss": 1.2081,
+ "theoretical_loss": 3.488198147680846,
+ "tokens_seen": 1807613952
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036554463390170514,
+ "loss": 1.3262,
+ "theoretical_loss": 3.488187480312016,
+ "tokens_seen": 1807679488
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036552858575727184,
+ "loss": 1.3382,
+ "theoretical_loss": 3.488176813438198,
+ "tokens_seen": 1807745024
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036551253761283854,
+ "loss": 1.1967,
+ "theoretical_loss": 3.48816614705935,
+ "tokens_seen": 1807810560
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036549648946840525,
+ "loss": 1.3265,
+ "theoretical_loss": 3.4881554811754336,
+ "tokens_seen": 1807876096
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036548044132397195,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4881448157864057,
+ "tokens_seen": 1807941632
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036546439317953865,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4881341508922263,
+ "tokens_seen": 1808007168
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036544834503510535,
+ "loss": 1.2415,
+ "theoretical_loss": 3.4881234864928543,
+ "tokens_seen": 1808072704
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036543229689067205,
+ "loss": 1.2662,
+ "theoretical_loss": 3.488112822588249,
+ "tokens_seen": 1808138240
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036541624874623875,
+ "loss": 1.3863,
+ "theoretical_loss": 3.488102159178369,
+ "tokens_seen": 1808203776
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036540020060180545,
+ "loss": 1.3339,
+ "theoretical_loss": 3.4880914962631744,
+ "tokens_seen": 1808269312
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036538415245737215,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4880808338426235,
+ "tokens_seen": 1808334848
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036536810431293885,
+ "loss": 1.142,
+ "theoretical_loss": 3.488070171916675,
+ "tokens_seen": 1808400384
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036535205616850555,
+ "loss": 1.3827,
+ "theoretical_loss": 3.4880595104852894,
+ "tokens_seen": 1808465920
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036533600802407225,
+ "loss": 1.3717,
+ "theoretical_loss": 3.488048849548425,
+ "tokens_seen": 1808531456
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036531995987963895,
+ "loss": 1.3093,
+ "theoretical_loss": 3.488038189106041,
+ "tokens_seen": 1808596992
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036530391173520565,
+ "loss": 1.3356,
+ "theoretical_loss": 3.4880275291580967,
+ "tokens_seen": 1808662528
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036528786359077235,
+ "loss": 1.254,
+ "theoretical_loss": 3.488016869704551,
+ "tokens_seen": 1808728064
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 606156,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4112968444824219,
+ "objective/train/theoretical_loss": 3.488006210745363,
+ "objective/train/tokens_used": 1829253600,
+ "theoretical_loss": 3.488006210745363,
+ "tokens_seen": 1808793600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036527181544633905,
+ "loss": 1.3484,
+ "theoretical_loss": 3.488006210745363,
+ "tokens_seen": 1808793600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036525576730190575,
+ "loss": 1.2864,
+ "theoretical_loss": 3.4879955522804926,
+ "tokens_seen": 1808859136
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036523971915747246,
+ "loss": 1.3229,
+ "theoretical_loss": 3.487984894309898,
+ "tokens_seen": 1808924672
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036522367101303916,
+ "loss": 1.2487,
+ "theoretical_loss": 3.487974236833539,
+ "tokens_seen": 1808990208
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036520762286860586,
+ "loss": 1.1661,
+ "theoretical_loss": 3.4879635798513746,
+ "tokens_seen": 1809055744
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036519157472417256,
+ "loss": 1.1866,
+ "theoretical_loss": 3.4879529233633644,
+ "tokens_seen": 1809121280
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036517552657973926,
+ "loss": 1.2133,
+ "theoretical_loss": 3.4879422673694664,
+ "tokens_seen": 1809186816
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036515947843530596,
+ "loss": 1.2459,
+ "theoretical_loss": 3.487931611869641,
+ "tokens_seen": 1809252352
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036514343029087266,
+ "loss": 1.2336,
+ "theoretical_loss": 3.4879209568638467,
+ "tokens_seen": 1809317888
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036512738214643936,
+ "loss": 1.1863,
+ "theoretical_loss": 3.4879103023520432,
+ "tokens_seen": 1809383424
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036511133400200606,
+ "loss": 1.2961,
+ "theoretical_loss": 3.4878996483341895,
+ "tokens_seen": 1809448960
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650952858575727,
+ "loss": 1.1978,
+ "theoretical_loss": 3.4878889948102447,
+ "tokens_seen": 1809514496
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650792377131394,
+ "loss": 1.2247,
+ "theoretical_loss": 3.487878341780168,
+ "tokens_seen": 1809580032
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650631895687061,
+ "loss": 1.1912,
+ "theoretical_loss": 3.4878676892439184,
+ "tokens_seen": 1809645568
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650471414242728,
+ "loss": 1.2955,
+ "theoretical_loss": 3.487857037201456,
+ "tokens_seen": 1809711104
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650310932798395,
+ "loss": 1.192,
+ "theoretical_loss": 3.4878463856527393,
+ "tokens_seen": 1809776640
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003650150451354062,
+ "loss": 1.3272,
+ "theoretical_loss": 3.487835734597728,
+ "tokens_seen": 1809842176
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649989969909729,
+ "loss": 1.3164,
+ "theoretical_loss": 3.4878250840363805,
+ "tokens_seen": 1809907712
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649829488465396,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4878144339686568,
+ "tokens_seen": 1809973248
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649669007021063,
+ "loss": 1.36,
+ "theoretical_loss": 3.487803784394516,
+ "tokens_seen": 1810038784
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000364950852557673,
+ "loss": 1.3253,
+ "theoretical_loss": 3.4877931353139173,
+ "tokens_seen": 1810104320
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649348044132397,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4877824867268195,
+ "tokens_seen": 1810169856
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649187562688064,
+ "loss": 1.3384,
+ "theoretical_loss": 3.487771838633183,
+ "tokens_seen": 1810235392
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003649027081243731,
+ "loss": 1.3045,
+ "theoretical_loss": 3.4877611910329662,
+ "tokens_seen": 1810300928
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648866599799398,
+ "loss": 1.3564,
+ "theoretical_loss": 3.4877505439261283,
+ "tokens_seen": 1810366464
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648706118355065,
+ "loss": 1.2878,
+ "theoretical_loss": 3.487739897312629,
+ "tokens_seen": 1810432000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648545636910732,
+ "loss": 1.2484,
+ "theoretical_loss": 3.487729251192427,
+ "tokens_seen": 1810497536
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648385155466399,
+ "loss": 1.2649,
+ "theoretical_loss": 3.4877186055654823,
+ "tokens_seen": 1810563072
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648224674022066,
+ "loss": 1.2481,
+ "theoretical_loss": 3.487707960431754,
+ "tokens_seen": 1810628608
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003648064192577733,
+ "loss": 1.3836,
+ "theoretical_loss": 3.487697315791202,
+ "tokens_seen": 1810694144
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036479037111334,
+ "loss": 1.3128,
+ "theoretical_loss": 3.4876866716437838,
+ "tokens_seen": 1810759680
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003647743229689067,
+ "loss": 1.3066,
+ "theoretical_loss": 3.48767602798946,
+ "tokens_seen": 1810825216
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003647582748244734,
+ "loss": 1.2664,
+ "theoretical_loss": 3.4876653848281904,
+ "tokens_seen": 1810890752
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003647422266800401,
+ "loss": 1.3305,
+ "theoretical_loss": 3.4876547421599327,
+ "tokens_seen": 1810956288
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003647261785356068,
+ "loss": 1.368,
+ "theoretical_loss": 3.4876440999846476,
+ "tokens_seen": 1811021824
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003647101303911735,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4876334583022945,
+ "tokens_seen": 1811087360
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003646940822467402,
+ "loss": 1.2326,
+ "theoretical_loss": 3.4876228171128316,
+ "tokens_seen": 1811152896
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003646780341023069,
+ "loss": 1.3704,
+ "theoretical_loss": 3.487612176416219,
+ "tokens_seen": 1811218432
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003646619859578736,
+ "loss": 1.3057,
+ "theoretical_loss": 3.487601536212416,
+ "tokens_seen": 1811283968
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003646459378134403,
+ "loss": 1.2593,
+ "theoretical_loss": 3.4875908965013815,
+ "tokens_seen": 1811349504
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000364629889669007,
+ "loss": 1.316,
+ "theoretical_loss": 3.487580257283076,
+ "tokens_seen": 1811415040
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003646138415245737,
+ "loss": 1.3584,
+ "theoretical_loss": 3.4875696185574574,
+ "tokens_seen": 1811480576
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003645977933801404,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4875589803244855,
+ "tokens_seen": 1811546112
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003645817452357071,
+ "loss": 1.2751,
+ "theoretical_loss": 3.4875483425841205,
+ "tokens_seen": 1811611648
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036456569709127383,
+ "loss": 1.2514,
+ "theoretical_loss": 3.487537705336321,
+ "tokens_seen": 1811677184
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003645496489468406,
+ "loss": 1.2563,
+ "theoretical_loss": 3.4875270685810467,
+ "tokens_seen": 1811742720
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003645336008024073,
+ "loss": 1.3224,
+ "theoretical_loss": 3.487516432318256,
+ "tokens_seen": 1811808256
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000364517552657974,
+ "loss": 1.2943,
+ "theoretical_loss": 3.48750579654791,
+ "tokens_seen": 1811873792
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003645015045135407,
+ "loss": 1.2754,
+ "theoretical_loss": 3.4874951612699667,
+ "tokens_seen": 1811939328
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003644854563691074,
+ "loss": 1.1284,
+ "theoretical_loss": 3.487484526484386,
+ "tokens_seen": 1812004864
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 607421,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2716153860092163,
+ "objective/train/theoretical_loss": 3.4874738921911277,
+ "objective/train/tokens_used": 1832530400,
+ "theoretical_loss": 3.4874738921911277,
+ "tokens_seen": 1812070400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036446940822467403,
+ "loss": 1.3402,
+ "theoretical_loss": 3.4874738921911277,
+ "tokens_seen": 1812070400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036445336008024073,
+ "loss": 1.2999,
+ "theoretical_loss": 3.48746325839015,
+ "tokens_seen": 1812135936
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036443731193580743,
+ "loss": 1.1687,
+ "theoretical_loss": 3.4874526250814144,
+ "tokens_seen": 1812201472
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036442126379137413,
+ "loss": 1.2976,
+ "theoretical_loss": 3.487441992264878,
+ "tokens_seen": 1812267008
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036440521564694083,
+ "loss": 1.311,
+ "theoretical_loss": 3.4874313599405014,
+ "tokens_seen": 1812332544
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036438916750250753,
+ "loss": 1.2737,
+ "theoretical_loss": 3.4874207281082437,
+ "tokens_seen": 1812398080
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036437311935807424,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4874100967680643,
+ "tokens_seen": 1812463616
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036435707121364094,
+ "loss": 1.3513,
+ "theoretical_loss": 3.487399465919924,
+ "tokens_seen": 1812529152
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036434102306920764,
+ "loss": 1.3616,
+ "theoretical_loss": 3.4873888355637797,
+ "tokens_seen": 1812594688
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036432497492477434,
+ "loss": 1.2454,
+ "theoretical_loss": 3.487378205699593,
+ "tokens_seen": 1812660224
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036430892678034104,
+ "loss": 1.2765,
+ "theoretical_loss": 3.487367576327322,
+ "tokens_seen": 1812725760
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036429287863590774,
+ "loss": 1.2213,
+ "theoretical_loss": 3.4873569474469273,
+ "tokens_seen": 1812791296
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036427683049147444,
+ "loss": 1.3066,
+ "theoretical_loss": 3.487346319058367,
+ "tokens_seen": 1812856832
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036426078234704114,
+ "loss": 1.2707,
+ "theoretical_loss": 3.4873356911616016,
+ "tokens_seen": 1812922368
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036424473420260784,
+ "loss": 1.247,
+ "theoretical_loss": 3.4873250637565905,
+ "tokens_seen": 1812987904
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036422868605817454,
+ "loss": 1.3099,
+ "theoretical_loss": 3.4873144368432927,
+ "tokens_seen": 1813053440
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036421263791374124,
+ "loss": 1.255,
+ "theoretical_loss": 3.4873038104216683,
+ "tokens_seen": 1813118976
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036419658976930794,
+ "loss": 1.1848,
+ "theoretical_loss": 3.4872931844916764,
+ "tokens_seen": 1813184512
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036418054162487464,
+ "loss": 1.247,
+ "theoretical_loss": 3.487282559053276,
+ "tokens_seen": 1813250048
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036416449348044134,
+ "loss": 1.2147,
+ "theoretical_loss": 3.4872719341064276,
+ "tokens_seen": 1813315584
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036414844533600804,
+ "loss": 1.2911,
+ "theoretical_loss": 3.48726130965109,
+ "tokens_seen": 1813381120
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036413239719157474,
+ "loss": 1.3377,
+ "theoretical_loss": 3.487250685687223,
+ "tokens_seen": 1813446656
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036411634904714145,
+ "loss": 1.2274,
+ "theoretical_loss": 3.4872400622147857,
+ "tokens_seen": 1813512192
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036410030090270815,
+ "loss": 1.3628,
+ "theoretical_loss": 3.487229439233738,
+ "tokens_seen": 1813577728
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036408425275827485,
+ "loss": 1.1811,
+ "theoretical_loss": 3.487218816744039,
+ "tokens_seen": 1813643264
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036406820461384155,
+ "loss": 1.3189,
+ "theoretical_loss": 3.4872081947456492,
+ "tokens_seen": 1813708800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036405215646940825,
+ "loss": 1.1791,
+ "theoretical_loss": 3.487197573238527,
+ "tokens_seen": 1813774336
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036403610832497495,
+ "loss": 1.4073,
+ "theoretical_loss": 3.4871869522226326,
+ "tokens_seen": 1813839872
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036402006018054165,
+ "loss": 1.268,
+ "theoretical_loss": 3.487176331697925,
+ "tokens_seen": 1813905408
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036400401203610835,
+ "loss": 1.3516,
+ "theoretical_loss": 3.487165711664365,
+ "tokens_seen": 1813970944
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036398796389167505,
+ "loss": 1.2671,
+ "theoretical_loss": 3.48715509212191,
+ "tokens_seen": 1814036480
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036397191574724175,
+ "loss": 1.2873,
+ "theoretical_loss": 3.4871444730705212,
+ "tokens_seen": 1814102016
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036395586760280845,
+ "loss": 1.2495,
+ "theoretical_loss": 3.487133854510158,
+ "tokens_seen": 1814167552
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036393981945837515,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4871232364407794,
+ "tokens_seen": 1814233088
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036392377131394185,
+ "loss": 1.256,
+ "theoretical_loss": 3.487112618862345,
+ "tokens_seen": 1814298624
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036390772316950855,
+ "loss": 1.2645,
+ "theoretical_loss": 3.487102001774815,
+ "tokens_seen": 1814364160
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036389167502507525,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4870913851781484,
+ "tokens_seen": 1814429696
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036387562688064195,
+ "loss": 1.2648,
+ "theoretical_loss": 3.487080769072305,
+ "tokens_seen": 1814495232
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036385957873620866,
+ "loss": 1.2338,
+ "theoretical_loss": 3.4870701534572444,
+ "tokens_seen": 1814560768
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036384353059177536,
+ "loss": 1.3548,
+ "theoretical_loss": 3.487059538332926,
+ "tokens_seen": 1814626304
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036382748244734206,
+ "loss": 1.3588,
+ "theoretical_loss": 3.4870489236993096,
+ "tokens_seen": 1814691840
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036381143430290876,
+ "loss": 1.3121,
+ "theoretical_loss": 3.4870383095563544,
+ "tokens_seen": 1814757376
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036379538615847546,
+ "loss": 1.3529,
+ "theoretical_loss": 3.487027695904021,
+ "tokens_seen": 1814822912
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036377933801404216,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4870170827422675,
+ "tokens_seen": 1814888448
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036376328986960886,
+ "loss": 1.3314,
+ "theoretical_loss": 3.487006470071055,
+ "tokens_seen": 1814953984
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036374724172517556,
+ "loss": 1.188,
+ "theoretical_loss": 3.486995857890342,
+ "tokens_seen": 1815019520
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036373119358074226,
+ "loss": 1.2663,
+ "theoretical_loss": 3.4869852462000885,
+ "tokens_seen": 1815085056
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036371514543630896,
+ "loss": 1.2313,
+ "theoretical_loss": 3.486974635000254,
+ "tokens_seen": 1815150592
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636990972918756,
+ "loss": 1.2599,
+ "theoretical_loss": 3.486964024290799,
+ "tokens_seen": 1815216128
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636830491474423,
+ "loss": 1.3257,
+ "theoretical_loss": 3.486953414071682,
+ "tokens_seen": 1815281664
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 608441,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3015462160110474,
+ "objective/train/theoretical_loss": 3.4869428043428634,
+ "objective/train/tokens_used": 1835807200,
+ "theoretical_loss": 3.4869428043428634,
+ "tokens_seen": 1815347200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000363667001003009,
+ "loss": 1.3048,
+ "theoretical_loss": 3.4869428043428634,
+ "tokens_seen": 1815347200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636509528585757,
+ "loss": 1.3369,
+ "theoretical_loss": 3.486932195104302,
+ "tokens_seen": 1815412736
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636349047141424,
+ "loss": 1.2625,
+ "theoretical_loss": 3.4869215863559586,
+ "tokens_seen": 1815478272
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636188565697091,
+ "loss": 1.3374,
+ "theoretical_loss": 3.486910978097792,
+ "tokens_seen": 1815543808
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003636028084252758,
+ "loss": 1.2534,
+ "theoretical_loss": 3.486900370329762,
+ "tokens_seen": 1815609344
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003635867602808425,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4868897630518285,
+ "tokens_seen": 1815674880
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003635707121364092,
+ "loss": 1.3043,
+ "theoretical_loss": 3.4868791562639507,
+ "tokens_seen": 1815740416
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003635546639919759,
+ "loss": 1.3031,
+ "theoretical_loss": 3.486868549966089,
+ "tokens_seen": 1815805952
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003635386158475426,
+ "loss": 1.2724,
+ "theoretical_loss": 3.486857944158202,
+ "tokens_seen": 1815871488
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003635225677031093,
+ "loss": 1.3081,
+ "theoretical_loss": 3.4868473388402506,
+ "tokens_seen": 1815937024
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000363506519558676,
+ "loss": 1.2642,
+ "theoretical_loss": 3.486836734012194,
+ "tokens_seen": 1816002560
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634904714142427,
+ "loss": 1.3277,
+ "theoretical_loss": 3.4868261296739917,
+ "tokens_seen": 1816068096
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634744232698094,
+ "loss": 1.2766,
+ "theoretical_loss": 3.4868155258256035,
+ "tokens_seen": 1816133632
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634583751253761,
+ "loss": 1.2914,
+ "theoretical_loss": 3.4868049224669893,
+ "tokens_seen": 1816199168
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634423269809428,
+ "loss": 1.2828,
+ "theoretical_loss": 3.4867943195981086,
+ "tokens_seen": 1816264704
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634262788365095,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4867837172189207,
+ "tokens_seen": 1816330240
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003634102306920762,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4867731153293864,
+ "tokens_seen": 1816395776
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633941825476429,
+ "loss": 1.271,
+ "theoretical_loss": 3.486762513929464,
+ "tokens_seen": 1816461312
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633781344032096,
+ "loss": 1.194,
+ "theoretical_loss": 3.486751913019115,
+ "tokens_seen": 1816526848
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633620862587764,
+ "loss": 1.256,
+ "theoretical_loss": 3.486741312598298,
+ "tokens_seen": 1816592384
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633460381143431,
+ "loss": 1.2416,
+ "theoretical_loss": 3.486730712666972,
+ "tokens_seen": 1816657920
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633299899699098,
+ "loss": 1.2039,
+ "theoretical_loss": 3.4867201132250982,
+ "tokens_seen": 1816723456
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003633139418254765,
+ "loss": 1.2122,
+ "theoretical_loss": 3.486709514272636,
+ "tokens_seen": 1816788992
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632978936810432,
+ "loss": 1.1513,
+ "theoretical_loss": 3.4866989158095443,
+ "tokens_seen": 1816854528
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632818455366099,
+ "loss": 1.2599,
+ "theoretical_loss": 3.486688317835784,
+ "tokens_seen": 1816920064
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632657973921766,
+ "loss": 1.256,
+ "theoretical_loss": 3.486677720351314,
+ "tokens_seen": 1816985600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632497492477433,
+ "loss": 1.2604,
+ "theoretical_loss": 3.486667123356095,
+ "tokens_seen": 1817051136
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036323370110331,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4866565268500853,
+ "tokens_seen": 1817116672
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632176529588767,
+ "loss": 1.2261,
+ "theoretical_loss": 3.486645930833246,
+ "tokens_seen": 1817182208
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003632016048144434,
+ "loss": 1.3218,
+ "theoretical_loss": 3.486635335305536,
+ "tokens_seen": 1817247744
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631855566700101,
+ "loss": 1.2733,
+ "theoretical_loss": 3.486624740266916,
+ "tokens_seen": 1817313280
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631695085255768,
+ "loss": 1.2521,
+ "theoretical_loss": 3.486614145717345,
+ "tokens_seen": 1817378816
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631534603811435,
+ "loss": 1.2266,
+ "theoretical_loss": 3.486603551656783,
+ "tokens_seen": 1817444352
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631374122367102,
+ "loss": 1.251,
+ "theoretical_loss": 3.48659295808519,
+ "tokens_seen": 1817509888
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631213640922769,
+ "loss": 1.3789,
+ "theoretical_loss": 3.486582365002526,
+ "tokens_seen": 1817575424
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003631053159478436,
+ "loss": 1.3212,
+ "theoretical_loss": 3.48657177240875,
+ "tokens_seen": 1817640960
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003630892678034103,
+ "loss": 1.2854,
+ "theoretical_loss": 3.4865611803038226,
+ "tokens_seen": 1817706496
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000363073219658977,
+ "loss": 1.3285,
+ "theoretical_loss": 3.4865505886877033,
+ "tokens_seen": 1817772032
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036305717151454363,
+ "loss": 1.3461,
+ "theoretical_loss": 3.4865399975603513,
+ "tokens_seen": 1817837568
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036304112337011033,
+ "loss": 1.3236,
+ "theoretical_loss": 3.4865294069217274,
+ "tokens_seen": 1817903104
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036302507522567703,
+ "loss": 1.3279,
+ "theoretical_loss": 3.4865188167717913,
+ "tokens_seen": 1817968640
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036300902708124374,
+ "loss": 1.3376,
+ "theoretical_loss": 3.4865082271105026,
+ "tokens_seen": 1818034176
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036299297893681044,
+ "loss": 1.273,
+ "theoretical_loss": 3.4864976379378207,
+ "tokens_seen": 1818099712
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036297693079237714,
+ "loss": 1.2072,
+ "theoretical_loss": 3.4864870492537063,
+ "tokens_seen": 1818165248
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036296088264794384,
+ "loss": 1.2285,
+ "theoretical_loss": 3.4864764610581194,
+ "tokens_seen": 1818230784
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036294483450351054,
+ "loss": 1.2893,
+ "theoretical_loss": 3.486465873351018,
+ "tokens_seen": 1818296320
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036292878635907724,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4864552861323643,
+ "tokens_seen": 1818361856
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036291273821464394,
+ "loss": 1.2505,
+ "theoretical_loss": 3.4864446994021168,
+ "tokens_seen": 1818427392
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036289669007021064,
+ "loss": 1.2763,
+ "theoretical_loss": 3.486434113160236,
+ "tokens_seen": 1818492928
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036288064192577734,
+ "loss": 1.3113,
+ "theoretical_loss": 3.4864235274066804,
+ "tokens_seen": 1818558464
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 609685,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2688941955566406,
+ "objective/train/theoretical_loss": 3.486412942141412,
+ "objective/train/tokens_used": 1839084000,
+ "theoretical_loss": 3.486412942141412,
+ "tokens_seen": 1818624000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036286459378134404,
+ "loss": 1.2658,
+ "theoretical_loss": 3.486412942141412,
+ "tokens_seen": 1818624000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036284854563691074,
+ "loss": 1.2564,
+ "theoretical_loss": 3.486402357364389,
+ "tokens_seen": 1818689536
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036283249749247744,
+ "loss": 1.3313,
+ "theoretical_loss": 3.4863917730755727,
+ "tokens_seen": 1818755072
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036281644934804414,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4863811892749212,
+ "tokens_seen": 1818820608
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036280040120361084,
+ "loss": 1.2665,
+ "theoretical_loss": 3.486370605962396,
+ "tokens_seen": 1818886144
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036278435305917754,
+ "loss": 1.3522,
+ "theoretical_loss": 3.4863600231379563,
+ "tokens_seen": 1818951680
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036276830491474424,
+ "loss": 1.38,
+ "theoretical_loss": 3.486349440801562,
+ "tokens_seen": 1819017216
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036275225677031095,
+ "loss": 1.3221,
+ "theoretical_loss": 3.4863388589531734,
+ "tokens_seen": 1819082752
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036273620862587765,
+ "loss": 1.2921,
+ "theoretical_loss": 3.48632827759275,
+ "tokens_seen": 1819148288
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036272016048144435,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4863176967202514,
+ "tokens_seen": 1819213824
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036270411233701105,
+ "loss": 1.3125,
+ "theoretical_loss": 3.4863071163356385,
+ "tokens_seen": 1819279360
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036268806419257775,
+ "loss": 1.3842,
+ "theoretical_loss": 3.4862965364388705,
+ "tokens_seen": 1819344896
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036267201604814445,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4862859570299074,
+ "tokens_seen": 1819410432
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036265596790371115,
+ "loss": 1.3334,
+ "theoretical_loss": 3.4862753781087097,
+ "tokens_seen": 1819475968
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036263991975927785,
+ "loss": 1.359,
+ "theoretical_loss": 3.4862647996752365,
+ "tokens_seen": 1819541504
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036262387161484455,
+ "loss": 1.3526,
+ "theoretical_loss": 3.4862542217294483,
+ "tokens_seen": 1819607040
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036260782347041125,
+ "loss": 1.2612,
+ "theoretical_loss": 3.4862436442713047,
+ "tokens_seen": 1819672576
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036259177532597795,
+ "loss": 1.227,
+ "theoretical_loss": 3.486233067300766,
+ "tokens_seen": 1819738112
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036257572718154465,
+ "loss": 1.2963,
+ "theoretical_loss": 3.486222490817793,
+ "tokens_seen": 1819803648
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036255967903711135,
+ "loss": 1.3204,
+ "theoretical_loss": 3.486211914822344,
+ "tokens_seen": 1819869184
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036254363089267805,
+ "loss": 1.2492,
+ "theoretical_loss": 3.486201339314379,
+ "tokens_seen": 1819934720
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036252758274824475,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4861907642938594,
+ "tokens_seen": 1820000256
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036251153460381145,
+ "loss": 1.3057,
+ "theoretical_loss": 3.486180189760744,
+ "tokens_seen": 1820065792
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036249548645937816,
+ "loss": 1.1972,
+ "theoretical_loss": 3.486169615714994,
+ "tokens_seen": 1820131328
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036247943831494486,
+ "loss": 1.1669,
+ "theoretical_loss": 3.486159042156568,
+ "tokens_seen": 1820196864
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036246339017051156,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4861484690854265,
+ "tokens_seen": 1820262400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036244734202607826,
+ "loss": 1.269,
+ "theoretical_loss": 3.48613789650153,
+ "tokens_seen": 1820327936
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036243129388164496,
+ "loss": 1.3196,
+ "theoretical_loss": 3.4861273244048383,
+ "tokens_seen": 1820393472
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036241524573721166,
+ "loss": 1.3406,
+ "theoretical_loss": 3.4861167527953105,
+ "tokens_seen": 1820459008
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036239919759277836,
+ "loss": 1.2703,
+ "theoretical_loss": 3.486106181672908,
+ "tokens_seen": 1820524544
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036238314944834506,
+ "loss": 1.2174,
+ "theoretical_loss": 3.48609561103759,
+ "tokens_seen": 1820590080
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036236710130391176,
+ "loss": 1.2678,
+ "theoretical_loss": 3.4860850408893165,
+ "tokens_seen": 1820655616
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036235105315947846,
+ "loss": 1.3515,
+ "theoretical_loss": 3.486074471228048,
+ "tokens_seen": 1820721152
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036233500501504516,
+ "loss": 1.3061,
+ "theoretical_loss": 3.4860639020537443,
+ "tokens_seen": 1820786688
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036231895687061186,
+ "loss": 1.2274,
+ "theoretical_loss": 3.486053333366365,
+ "tokens_seen": 1820852224
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036230290872617856,
+ "loss": 1.2016,
+ "theoretical_loss": 3.4860427651658705,
+ "tokens_seen": 1820917760
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003622868605817452,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4860321974522215,
+ "tokens_seen": 1820983296
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003622708124373119,
+ "loss": 1.3436,
+ "theoretical_loss": 3.486021630225377,
+ "tokens_seen": 1821048832
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003622547642928786,
+ "loss": 1.235,
+ "theoretical_loss": 3.4860110634852974,
+ "tokens_seen": 1821114368
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003622387161484453,
+ "loss": 1.3445,
+ "theoretical_loss": 3.486000497231943,
+ "tokens_seen": 1821179904
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000362222668004012,
+ "loss": 1.2943,
+ "theoretical_loss": 3.485989931465274,
+ "tokens_seen": 1821245440
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003622066198595787,
+ "loss": 1.3383,
+ "theoretical_loss": 3.4859793661852496,
+ "tokens_seen": 1821310976
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003621905717151454,
+ "loss": 1.2164,
+ "theoretical_loss": 3.485968801391831,
+ "tokens_seen": 1821376512
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003621745235707121,
+ "loss": 1.3033,
+ "theoretical_loss": 3.4859582370849775,
+ "tokens_seen": 1821442048
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036215847542627887,
+ "loss": 1.2653,
+ "theoretical_loss": 3.4859476732646493,
+ "tokens_seen": 1821507584
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036214242728184557,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4859371099308065,
+ "tokens_seen": 1821573120
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036212637913741227,
+ "loss": 1.2727,
+ "theoretical_loss": 3.4859265470834098,
+ "tokens_seen": 1821638656
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036211033099297897,
+ "loss": 1.1472,
+ "theoretical_loss": 3.4859159847224186,
+ "tokens_seen": 1821704192
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036209428284854567,
+ "loss": 1.1873,
+ "theoretical_loss": 3.485905422847793,
+ "tokens_seen": 1821769728
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036207823470411237,
+ "loss": 1.2393,
+ "theoretical_loss": 3.485894861459494,
+ "tokens_seen": 1821835264
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 610785,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2064809799194336,
+ "objective/train/theoretical_loss": 3.4858843005574807,
+ "objective/train/tokens_used": 1842360800,
+ "theoretical_loss": 3.4858843005574807,
+ "tokens_seen": 1821900800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036206218655967907,
+ "loss": 1.223,
+ "theoretical_loss": 3.4858843005574807,
+ "tokens_seen": 1821900800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003620461384152458,
+ "loss": 1.2586,
+ "theoretical_loss": 3.485873740141713,
+ "tokens_seen": 1821966336
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003620300902708125,
+ "loss": 1.334,
+ "theoretical_loss": 3.4858631802121525,
+ "tokens_seen": 1822031872
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003620140421263792,
+ "loss": 1.2901,
+ "theoretical_loss": 3.4858526207687577,
+ "tokens_seen": 1822097408
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619979939819459,
+ "loss": 1.2435,
+ "theoretical_loss": 3.48584206181149,
+ "tokens_seen": 1822162944
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619819458375126,
+ "loss": 1.1528,
+ "theoretical_loss": 3.4858315033403087,
+ "tokens_seen": 1822228480
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619658976930793,
+ "loss": 1.3139,
+ "theoretical_loss": 3.485820945355174,
+ "tokens_seen": 1822294016
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000361949849548646,
+ "loss": 1.2167,
+ "theoretical_loss": 3.4858103878560467,
+ "tokens_seen": 1822359552
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619338014042127,
+ "loss": 1.2569,
+ "theoretical_loss": 3.485799830842886,
+ "tokens_seen": 1822425088
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619177532597794,
+ "loss": 1.1894,
+ "theoretical_loss": 3.4857892743156533,
+ "tokens_seen": 1822490624
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003619017051153461,
+ "loss": 1.1715,
+ "theoretical_loss": 3.4857787182743074,
+ "tokens_seen": 1822556160
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618856569709128,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4857681627188097,
+ "tokens_seen": 1822621696
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618696088264795,
+ "loss": 1.2832,
+ "theoretical_loss": 3.4857576076491195,
+ "tokens_seen": 1822687232
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618535606820462,
+ "loss": 1.2599,
+ "theoretical_loss": 3.4857470530651975,
+ "tokens_seen": 1822752768
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618375125376129,
+ "loss": 1.2569,
+ "theoretical_loss": 3.485736498967003,
+ "tokens_seen": 1822818304
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618214643931796,
+ "loss": 1.2071,
+ "theoretical_loss": 3.4857259453544973,
+ "tokens_seen": 1822883840
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003618054162487463,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4857153922276396,
+ "tokens_seen": 1822949376
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000361789368104313,
+ "loss": 1.4541,
+ "theoretical_loss": 3.485704839586391,
+ "tokens_seen": 1823014912
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003617733199598797,
+ "loss": 1.2774,
+ "theoretical_loss": 3.4856942874307117,
+ "tokens_seen": 1823080448
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003617572718154464,
+ "loss": 1.166,
+ "theoretical_loss": 3.485683735760561,
+ "tokens_seen": 1823145984
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003617412236710131,
+ "loss": 1.2274,
+ "theoretical_loss": 3.4856731845758997,
+ "tokens_seen": 1823211520
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003617251755265798,
+ "loss": 1.1972,
+ "theoretical_loss": 3.4856626338766876,
+ "tokens_seen": 1823277056
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003617091273821465,
+ "loss": 1.3944,
+ "theoretical_loss": 3.4856520836628855,
+ "tokens_seen": 1823342592
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003616930792377132,
+ "loss": 1.2454,
+ "theoretical_loss": 3.4856415339344533,
+ "tokens_seen": 1823408128
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003616770310932799,
+ "loss": 1.2749,
+ "theoretical_loss": 3.485630984691352,
+ "tokens_seen": 1823473664
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036166098294884653,
+ "loss": 1.316,
+ "theoretical_loss": 3.48562043593354,
+ "tokens_seen": 1823539200
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036164493480441323,
+ "loss": 1.2725,
+ "theoretical_loss": 3.4856098876609796,
+ "tokens_seen": 1823604736
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036162888665997994,
+ "loss": 1.2374,
+ "theoretical_loss": 3.4855993398736294,
+ "tokens_seen": 1823670272
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036161283851554664,
+ "loss": 1.3047,
+ "theoretical_loss": 3.4855887925714506,
+ "tokens_seen": 1823735808
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036159679037111334,
+ "loss": 1.3376,
+ "theoretical_loss": 3.485578245754403,
+ "tokens_seen": 1823801344
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036158074222668004,
+ "loss": 1.294,
+ "theoretical_loss": 3.4855676994224476,
+ "tokens_seen": 1823866880
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036156469408224674,
+ "loss": 1.2798,
+ "theoretical_loss": 3.4855571535755434,
+ "tokens_seen": 1823932416
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036154864593781344,
+ "loss": 1.1986,
+ "theoretical_loss": 3.485546608213652,
+ "tokens_seen": 1823997952
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036153259779338014,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4855360633367325,
+ "tokens_seen": 1824063488
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036151654964894684,
+ "loss": 1.268,
+ "theoretical_loss": 3.485525518944746,
+ "tokens_seen": 1824129024
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036150050150451354,
+ "loss": 1.4095,
+ "theoretical_loss": 3.4855149750376526,
+ "tokens_seen": 1824194560
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036148445336008024,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4855044316154125,
+ "tokens_seen": 1824260096
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036146840521564694,
+ "loss": 1.1598,
+ "theoretical_loss": 3.4854938886779854,
+ "tokens_seen": 1824325632
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036145235707121364,
+ "loss": 1.2525,
+ "theoretical_loss": 3.4854833462253323,
+ "tokens_seen": 1824391168
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036143630892678034,
+ "loss": 1.4092,
+ "theoretical_loss": 3.4854728042574137,
+ "tokens_seen": 1824456704
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036142026078234704,
+ "loss": 1.1669,
+ "theoretical_loss": 3.485462262774189,
+ "tokens_seen": 1824522240
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036140421263791374,
+ "loss": 1.2821,
+ "theoretical_loss": 3.48545172177562,
+ "tokens_seen": 1824587776
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036138816449348044,
+ "loss": 1.169,
+ "theoretical_loss": 3.4854411812616655,
+ "tokens_seen": 1824653312
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036137211634904715,
+ "loss": 1.2792,
+ "theoretical_loss": 3.485430641232286,
+ "tokens_seen": 1824718848
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036135606820461385,
+ "loss": 1.1355,
+ "theoretical_loss": 3.485420101687443,
+ "tokens_seen": 1824784384
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036134002006018055,
+ "loss": 1.1121,
+ "theoretical_loss": 3.485409562627096,
+ "tokens_seen": 1824849920
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036132397191574725,
+ "loss": 1.2295,
+ "theoretical_loss": 3.4853990240512047,
+ "tokens_seen": 1824915456
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036130792377131395,
+ "loss": 1.2294,
+ "theoretical_loss": 3.48538848595973,
+ "tokens_seen": 1824980992
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036129187562688065,
+ "loss": 1.2275,
+ "theoretical_loss": 3.485377948352633,
+ "tokens_seen": 1825046528
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036127582748244735,
+ "loss": 1.3045,
+ "theoretical_loss": 3.4853674112298734,
+ "tokens_seen": 1825112064
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 611941,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2710858583450317,
+ "objective/train/theoretical_loss": 3.4853568745914107,
+ "objective/train/tokens_used": 1845637600,
+ "theoretical_loss": 3.4853568745914107,
+ "tokens_seen": 1825177600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036125977933801405,
+ "loss": 1.2599,
+ "theoretical_loss": 3.4853568745914107,
+ "tokens_seen": 1825177600
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036124373119358075,
+ "loss": 1.2478,
+ "theoretical_loss": 3.485346338437207,
+ "tokens_seen": 1825243136
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036122768304914745,
+ "loss": 1.1488,
+ "theoretical_loss": 3.4853358027672208,
+ "tokens_seen": 1825308672
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036121163490471415,
+ "loss": 1.2829,
+ "theoretical_loss": 3.485325267581414,
+ "tokens_seen": 1825374208
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036119558676028085,
+ "loss": 1.2769,
+ "theoretical_loss": 3.485314732879746,
+ "tokens_seen": 1825439744
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036117953861584755,
+ "loss": 1.2999,
+ "theoretical_loss": 3.485304198662178,
+ "tokens_seen": 1825505280
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036116349047141425,
+ "loss": 1.2482,
+ "theoretical_loss": 3.48529366492867,
+ "tokens_seen": 1825570816
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036114744232698095,
+ "loss": 1.2877,
+ "theoretical_loss": 3.4852831316791817,
+ "tokens_seen": 1825636352
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036113139418254766,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4852725989136744,
+ "tokens_seen": 1825701888
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036111534603811436,
+ "loss": 1.2646,
+ "theoretical_loss": 3.485262066632108,
+ "tokens_seen": 1825767424
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036109929789368106,
+ "loss": 1.2557,
+ "theoretical_loss": 3.4852515348344433,
+ "tokens_seen": 1825832960
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036108324974924776,
+ "loss": 1.2524,
+ "theoretical_loss": 3.48524100352064,
+ "tokens_seen": 1825898496
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036106720160481446,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4852304726906596,
+ "tokens_seen": 1825964032
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036105115346038116,
+ "loss": 1.2649,
+ "theoretical_loss": 3.4852199423444614,
+ "tokens_seen": 1826029568
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036103510531594786,
+ "loss": 1.3101,
+ "theoretical_loss": 3.485209412482006,
+ "tokens_seen": 1826095104
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036101905717151456,
+ "loss": 1.3538,
+ "theoretical_loss": 3.485198883103255,
+ "tokens_seen": 1826160640
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036100300902708126,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4851883542081676,
+ "tokens_seen": 1826226176
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036098696088264796,
+ "loss": 1.1941,
+ "theoretical_loss": 3.485177825796704,
+ "tokens_seen": 1826291712
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036097091273821466,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4851672978688257,
+ "tokens_seen": 1826357248
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036095486459378136,
+ "loss": 1.2495,
+ "theoretical_loss": 3.4851567704244926,
+ "tokens_seen": 1826422784
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036093881644934806,
+ "loss": 1.1728,
+ "theoretical_loss": 3.485146243463665,
+ "tokens_seen": 1826488320
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036092276830491476,
+ "loss": 1.1705,
+ "theoretical_loss": 3.4851357169863038,
+ "tokens_seen": 1826553856
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036090672016048146,
+ "loss": 1.2066,
+ "theoretical_loss": 3.4851251909923686,
+ "tokens_seen": 1826619392
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036089067201604816,
+ "loss": 1.3087,
+ "theoretical_loss": 3.485114665481821,
+ "tokens_seen": 1826684928
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036087462387161487,
+ "loss": 1.2307,
+ "theoretical_loss": 3.4851041404546206,
+ "tokens_seen": 1826750464
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036085857572718157,
+ "loss": 1.2749,
+ "theoretical_loss": 3.485093615910728,
+ "tokens_seen": 1826816000
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036084252758274827,
+ "loss": 1.1746,
+ "theoretical_loss": 3.485083091850104,
+ "tokens_seen": 1826881536
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036082647943831497,
+ "loss": 1.2735,
+ "theoretical_loss": 3.485072568272709,
+ "tokens_seen": 1826947072
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036081043129388167,
+ "loss": 1.1848,
+ "theoretical_loss": 3.4850620451785033,
+ "tokens_seen": 1827012608
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036079438314944837,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4850515225674474,
+ "tokens_seen": 1827078144
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036077833500501507,
+ "loss": 1.335,
+ "theoretical_loss": 3.4850410004395016,
+ "tokens_seen": 1827143680
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036076228686058177,
+ "loss": 1.3148,
+ "theoretical_loss": 3.485030478794627,
+ "tokens_seen": 1827209216
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036074623871614847,
+ "loss": 1.3177,
+ "theoretical_loss": 3.4850199576327836,
+ "tokens_seen": 1827274752
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036073019057171517,
+ "loss": 1.3288,
+ "theoretical_loss": 3.4850094369539315,
+ "tokens_seen": 1827340288
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036071414242728187,
+ "loss": 1.2212,
+ "theoretical_loss": 3.4849989167580326,
+ "tokens_seen": 1827405824
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036069809428284857,
+ "loss": 1.3141,
+ "theoretical_loss": 3.484988397045046,
+ "tokens_seen": 1827471360
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036068204613841527,
+ "loss": 1.4041,
+ "theoretical_loss": 3.484977877814933,
+ "tokens_seen": 1827536896
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000360665997993982,
+ "loss": 1.2151,
+ "theoretical_loss": 3.484967359067654,
+ "tokens_seen": 1827602432
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003606499498495487,
+ "loss": 1.2378,
+ "theoretical_loss": 3.484956840803169,
+ "tokens_seen": 1827667968
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003606339017051154,
+ "loss": 1.2024,
+ "theoretical_loss": 3.484946323021439,
+ "tokens_seen": 1827733504
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003606178535606821,
+ "loss": 1.2173,
+ "theoretical_loss": 3.484935805722425,
+ "tokens_seen": 1827799040
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003606018054162488,
+ "loss": 1.349,
+ "theoretical_loss": 3.484925288906086,
+ "tokens_seen": 1827864576
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003605857572718155,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4849147725723846,
+ "tokens_seen": 1827930112
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003605697091273822,
+ "loss": 1.2905,
+ "theoretical_loss": 3.48490425672128,
+ "tokens_seen": 1827995648
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003605536609829489,
+ "loss": 1.2654,
+ "theoretical_loss": 3.4848937413527326,
+ "tokens_seen": 1828061184
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003605376128385156,
+ "loss": 1.2722,
+ "theoretical_loss": 3.484883226466704,
+ "tokens_seen": 1828126720
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003605215646940823,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4848727120631535,
+ "tokens_seen": 1828192256
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000360505516549649,
+ "loss": 1.2509,
+ "theoretical_loss": 3.484862198142043,
+ "tokens_seen": 1828257792
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604894684052157,
+ "loss": 1.2392,
+ "theoretical_loss": 3.484851684703332,
+ "tokens_seen": 1828323328
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604734202607824,
+ "loss": 1.299,
+ "theoretical_loss": 3.4848411717469814,
+ "tokens_seen": 1828388864
+ },
+ {
+ "epoch": 0.55,
+ "objective/train/docs_used": 612914,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2961459159851074,
+ "objective/train/theoretical_loss": 3.4848306592729523,
+ "objective/train/tokens_used": 1848914400,
+ "theoretical_loss": 3.4848306592729523,
+ "tokens_seen": 1828454400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604573721163491,
+ "loss": 1.2261,
+ "theoretical_loss": 3.4848306592729523,
+ "tokens_seen": 1828454400
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604413239719158,
+ "loss": 1.2769,
+ "theoretical_loss": 3.4848201472812046,
+ "tokens_seen": 1828519936
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604252758274825,
+ "loss": 1.2933,
+ "theoretical_loss": 3.484809635771699,
+ "tokens_seen": 1828585472
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003604092276830492,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4847991247443963,
+ "tokens_seen": 1828651008
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003603931795386159,
+ "loss": 1.2587,
+ "theoretical_loss": 3.4847886141992572,
+ "tokens_seen": 1828716544
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003603771313941826,
+ "loss": 1.2012,
+ "theoretical_loss": 3.4847781041362422,
+ "tokens_seen": 1828782080
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003603610832497493,
+ "loss": 1.2621,
+ "theoretical_loss": 3.484767594555312,
+ "tokens_seen": 1828847616
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.000360345035105316,
+ "loss": 1.1882,
+ "theoretical_loss": 3.484757085456427,
+ "tokens_seen": 1828913152
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003603289869608827,
+ "loss": 1.3032,
+ "theoretical_loss": 3.484746576839547,
+ "tokens_seen": 1828978688
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003603129388164494,
+ "loss": 1.1647,
+ "theoretical_loss": 3.4847360687046347,
+ "tokens_seen": 1829044224
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003602968906720161,
+ "loss": 1.2247,
+ "theoretical_loss": 3.4847255610516488,
+ "tokens_seen": 1829109760
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003602808425275828,
+ "loss": 1.2256,
+ "theoretical_loss": 3.484715053880551,
+ "tokens_seen": 1829175296
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036026479438314944,
+ "loss": 1.3092,
+ "theoretical_loss": 3.484704547191301,
+ "tokens_seen": 1829240832
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036024874623871614,
+ "loss": 1.1803,
+ "theoretical_loss": 3.484694040983861,
+ "tokens_seen": 1829306368
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036023269809428284,
+ "loss": 1.2245,
+ "theoretical_loss": 3.48468353525819,
+ "tokens_seen": 1829371904
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036021664994984954,
+ "loss": 1.3079,
+ "theoretical_loss": 3.4846730300142497,
+ "tokens_seen": 1829437440
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036020060180541624,
+ "loss": 1.2751,
+ "theoretical_loss": 3.4846625252520003,
+ "tokens_seen": 1829502976
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036018455366098294,
+ "loss": 1.3587,
+ "theoretical_loss": 3.484652020971402,
+ "tokens_seen": 1829568512
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036016850551654964,
+ "loss": 1.2593,
+ "theoretical_loss": 3.4846415171724168,
+ "tokens_seen": 1829634048
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036015245737211634,
+ "loss": 1.0759,
+ "theoretical_loss": 3.484631013855004,
+ "tokens_seen": 1829699584
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036013640922768304,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4846205110191253,
+ "tokens_seen": 1829765120
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036012036108324974,
+ "loss": 1.3465,
+ "theoretical_loss": 3.484610008664741,
+ "tokens_seen": 1829830656
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036010431293881644,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4845995067918114,
+ "tokens_seen": 1829896192
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036008826479438314,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4845890054002973,
+ "tokens_seen": 1829961728
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036007221664994984,
+ "loss": 1.2271,
+ "theoretical_loss": 3.48457850449016,
+ "tokens_seen": 1830027264
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036005616850551654,
+ "loss": 1.175,
+ "theoretical_loss": 3.4845680040613596,
+ "tokens_seen": 1830092800
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036004012036108324,
+ "loss": 1.3492,
+ "theoretical_loss": 3.484557504113857,
+ "tokens_seen": 1830158336
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036002407221664994,
+ "loss": 1.3292,
+ "theoretical_loss": 3.484547004647613,
+ "tokens_seen": 1830223872
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00036000802407221665,
+ "loss": 1.2915,
+ "theoretical_loss": 3.484536505662588,
+ "tokens_seen": 1830289408
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035999197592778335,
+ "loss": 1.3461,
+ "theoretical_loss": 3.484526007158743,
+ "tokens_seen": 1830354944
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035997592778335005,
+ "loss": 1.3139,
+ "theoretical_loss": 3.4845155091360387,
+ "tokens_seen": 1830420480
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035995987963891675,
+ "loss": 1.2514,
+ "theoretical_loss": 3.484505011594436,
+ "tokens_seen": 1830486016
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035994383149448345,
+ "loss": 1.2276,
+ "theoretical_loss": 3.4844945145338952,
+ "tokens_seen": 1830551552
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035992778335005015,
+ "loss": 1.3074,
+ "theoretical_loss": 3.484484017954377,
+ "tokens_seen": 1830617088
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035991173520561685,
+ "loss": 1.3158,
+ "theoretical_loss": 3.4844735218558425,
+ "tokens_seen": 1830682624
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035989568706118355,
+ "loss": 1.2718,
+ "theoretical_loss": 3.4844630262382523,
+ "tokens_seen": 1830748160
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035987963891675025,
+ "loss": 1.2012,
+ "theoretical_loss": 3.4844525311015673,
+ "tokens_seen": 1830813696
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035986359077231695,
+ "loss": 1.33,
+ "theoretical_loss": 3.4844420364457482,
+ "tokens_seen": 1830879232
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035984754262788365,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4844315422707552,
+ "tokens_seen": 1830944768
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035983149448345035,
+ "loss": 1.2533,
+ "theoretical_loss": 3.48442104857655,
+ "tokens_seen": 1831010304
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035981544633901705,
+ "loss": 1.2156,
+ "theoretical_loss": 3.484410555363093,
+ "tokens_seen": 1831075840
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035979939819458375,
+ "loss": 1.2119,
+ "theoretical_loss": 3.4844000626303444,
+ "tokens_seen": 1831141376
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035978335005015045,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4843895703782657,
+ "tokens_seen": 1831206912
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.00035976730190571715,
+ "loss": 1.2062,
+ "theoretical_loss": 3.4843790786068176,
+ "tokens_seen": 1831272448
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003597512537612839,
+ "loss": 1.1707,
+ "theoretical_loss": 3.48436858731596,
+ "tokens_seen": 1831337984
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003597352056168506,
+ "loss": 1.2169,
+ "theoretical_loss": 3.484358096505655,
+ "tokens_seen": 1831403520
+ },
+ {
+ "epoch": 0.55,
+ "learning_rate": 0.0003597191574724173,
+ "loss": 1.4065,
+ "theoretical_loss": 3.4843476061758625,
+ "tokens_seen": 1831469056
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000359703109327984,
+ "loss": 1.3943,
+ "theoretical_loss": 3.484337116326544,
+ "tokens_seen": 1831534592
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003596870611835507,
+ "loss": 1.2586,
+ "theoretical_loss": 3.4843266269576594,
+ "tokens_seen": 1831600128
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003596710130391174,
+ "loss": 1.1794,
+ "theoretical_loss": 3.48431613806917,
+ "tokens_seen": 1831665664
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 614178,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1981984376907349,
+ "objective/train/theoretical_loss": 3.484305649661037,
+ "objective/train/tokens_used": 1852191200,
+ "theoretical_loss": 3.484305649661037,
+ "tokens_seen": 1831731200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003596549648946841,
+ "loss": 1.1518,
+ "theoretical_loss": 3.484305649661037,
+ "tokens_seen": 1831731200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003596389167502508,
+ "loss": 1.19,
+ "theoretical_loss": 3.48429516173322,
+ "tokens_seen": 1831796736
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035962286860581746,
+ "loss": 1.3193,
+ "theoretical_loss": 3.4842846742856812,
+ "tokens_seen": 1831862272
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035960682046138416,
+ "loss": 1.2772,
+ "theoretical_loss": 3.4842741873183813,
+ "tokens_seen": 1831927808
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035959077231695086,
+ "loss": 1.2874,
+ "theoretical_loss": 3.48426370083128,
+ "tokens_seen": 1831993344
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035957472417251756,
+ "loss": 1.3171,
+ "theoretical_loss": 3.4842532148243395,
+ "tokens_seen": 1832058880
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035955867602808426,
+ "loss": 1.3326,
+ "theoretical_loss": 3.4842427292975193,
+ "tokens_seen": 1832124416
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035954262788365096,
+ "loss": 1.2284,
+ "theoretical_loss": 3.484232244250781,
+ "tokens_seen": 1832189952
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035952657973921766,
+ "loss": 1.2879,
+ "theoretical_loss": 3.4842217596840857,
+ "tokens_seen": 1832255488
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035951053159478436,
+ "loss": 1.3435,
+ "theoretical_loss": 3.4842112755973935,
+ "tokens_seen": 1832321024
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035949448345035107,
+ "loss": 1.2232,
+ "theoretical_loss": 3.484200791990666,
+ "tokens_seen": 1832386560
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035947843530591777,
+ "loss": 1.2788,
+ "theoretical_loss": 3.4841903088638637,
+ "tokens_seen": 1832452096
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035946238716148447,
+ "loss": 1.2972,
+ "theoretical_loss": 3.4841798262169474,
+ "tokens_seen": 1832517632
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035944633901705117,
+ "loss": 1.3475,
+ "theoretical_loss": 3.484169344049878,
+ "tokens_seen": 1832583168
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035943029087261787,
+ "loss": 1.3467,
+ "theoretical_loss": 3.4841588623626167,
+ "tokens_seen": 1832648704
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035941424272818457,
+ "loss": 1.29,
+ "theoretical_loss": 3.4841483811551237,
+ "tokens_seen": 1832714240
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035939819458375127,
+ "loss": 1.1779,
+ "theoretical_loss": 3.4841379004273607,
+ "tokens_seen": 1832779776
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035938214643931797,
+ "loss": 1.3041,
+ "theoretical_loss": 3.4841274201792882,
+ "tokens_seen": 1832845312
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035936609829488467,
+ "loss": 1.176,
+ "theoretical_loss": 3.4841169404108667,
+ "tokens_seen": 1832910848
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035935005015045137,
+ "loss": 1.333,
+ "theoretical_loss": 3.484106461122058,
+ "tokens_seen": 1832976384
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035933400200601807,
+ "loss": 1.3407,
+ "theoretical_loss": 3.484095982312822,
+ "tokens_seen": 1833041920
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035931795386158477,
+ "loss": 1.2646,
+ "theoretical_loss": 3.4840855039831204,
+ "tokens_seen": 1833107456
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003593019057171515,
+ "loss": 1.2558,
+ "theoretical_loss": 3.484075026132914,
+ "tokens_seen": 1833172992
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003592858575727182,
+ "loss": 1.2761,
+ "theoretical_loss": 3.4840645487621633,
+ "tokens_seen": 1833238528
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003592698094282849,
+ "loss": 1.248,
+ "theoretical_loss": 3.4840540718708297,
+ "tokens_seen": 1833304064
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003592537612838516,
+ "loss": 1.2802,
+ "theoretical_loss": 3.484043595458873,
+ "tokens_seen": 1833369600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003592377131394183,
+ "loss": 1.3597,
+ "theoretical_loss": 3.484033119526256,
+ "tokens_seen": 1833435136
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000359221664994985,
+ "loss": 1.3109,
+ "theoretical_loss": 3.4840226440729385,
+ "tokens_seen": 1833500672
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003592056168505517,
+ "loss": 1.3115,
+ "theoretical_loss": 3.4840121690988815,
+ "tokens_seen": 1833566208
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591895687061184,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4840016946040455,
+ "tokens_seen": 1833631744
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591735205616851,
+ "loss": 1.3082,
+ "theoretical_loss": 3.4839912205883925,
+ "tokens_seen": 1833697280
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591574724172518,
+ "loss": 1.3108,
+ "theoretical_loss": 3.4839807470518833,
+ "tokens_seen": 1833762816
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591414242728185,
+ "loss": 1.2422,
+ "theoretical_loss": 3.483970273994478,
+ "tokens_seen": 1833828352
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591253761283852,
+ "loss": 1.3497,
+ "theoretical_loss": 3.483959801416138,
+ "tokens_seen": 1833893888
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003591093279839519,
+ "loss": 1.3792,
+ "theoretical_loss": 3.4839493293168244,
+ "tokens_seen": 1833959424
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003590932798395186,
+ "loss": 1.2936,
+ "theoretical_loss": 3.483938857696498,
+ "tokens_seen": 1834024960
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003590772316950853,
+ "loss": 1.3189,
+ "theoretical_loss": 3.48392838655512,
+ "tokens_seen": 1834090496
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000359061183550652,
+ "loss": 1.2818,
+ "theoretical_loss": 3.4839179158926514,
+ "tokens_seen": 1834156032
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003590451354062187,
+ "loss": 1.3387,
+ "theoretical_loss": 3.483907445709053,
+ "tokens_seen": 1834221568
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003590290872617854,
+ "loss": 1.2675,
+ "theoretical_loss": 3.483896976004286,
+ "tokens_seen": 1834287104
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003590130391173521,
+ "loss": 1.3338,
+ "theoretical_loss": 3.483886506778311,
+ "tokens_seen": 1834352640
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589969909729188,
+ "loss": 1.3693,
+ "theoretical_loss": 3.483876038031089,
+ "tokens_seen": 1834418176
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589809428284855,
+ "loss": 1.2749,
+ "theoretical_loss": 3.4838655697625813,
+ "tokens_seen": 1834483712
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589648946840522,
+ "loss": 1.1919,
+ "theoretical_loss": 3.483855101972749,
+ "tokens_seen": 1834549248
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589488465396189,
+ "loss": 1.3126,
+ "theoretical_loss": 3.483844634661553,
+ "tokens_seen": 1834614784
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589327983951856,
+ "loss": 1.3354,
+ "theoretical_loss": 3.483834167828954,
+ "tokens_seen": 1834680320
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003589167502507523,
+ "loss": 1.2303,
+ "theoretical_loss": 3.483823701474914,
+ "tokens_seen": 1834745856
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000358900702106319,
+ "loss": 1.3366,
+ "theoretical_loss": 3.4838132355993925,
+ "tokens_seen": 1834811392
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003588846539618857,
+ "loss": 1.1893,
+ "theoretical_loss": 3.4838027702023515,
+ "tokens_seen": 1834876928
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003588686058174524,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4837923052837523,
+ "tokens_seen": 1834942464
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 615428,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3236737251281738,
+ "objective/train/theoretical_loss": 3.4837818408435552,
+ "objective/train/tokens_used": 1855468000,
+ "theoretical_loss": 3.4837818408435552,
+ "tokens_seen": 1835008000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035885255767301904,
+ "loss": 1.3667,
+ "theoretical_loss": 3.4837818408435552,
+ "tokens_seen": 1835008000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035883650952858574,
+ "loss": 1.2132,
+ "theoretical_loss": 3.483771376881722,
+ "tokens_seen": 1835073536
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035882046138415244,
+ "loss": 1.3665,
+ "theoretical_loss": 3.483760913398213,
+ "tokens_seen": 1835139072
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035880441323971914,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4837504503929893,
+ "tokens_seen": 1835204608
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035878836509528584,
+ "loss": 1.1866,
+ "theoretical_loss": 3.483739987866013,
+ "tokens_seen": 1835270144
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035877231695085254,
+ "loss": 1.218,
+ "theoretical_loss": 3.4837295258172434,
+ "tokens_seen": 1835335680
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035875626880641924,
+ "loss": 1.3062,
+ "theoretical_loss": 3.4837190642466433,
+ "tokens_seen": 1835401216
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035874022066198594,
+ "loss": 1.2915,
+ "theoretical_loss": 3.483708603154173,
+ "tokens_seen": 1835466752
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035872417251755264,
+ "loss": 1.321,
+ "theoretical_loss": 3.4836981425397933,
+ "tokens_seen": 1835532288
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035870812437311934,
+ "loss": 1.1974,
+ "theoretical_loss": 3.483687682403466,
+ "tokens_seen": 1835597824
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035869207622868604,
+ "loss": 1.2398,
+ "theoretical_loss": 3.4836772227451513,
+ "tokens_seen": 1835663360
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035867602808425274,
+ "loss": 1.3121,
+ "theoretical_loss": 3.483666763564811,
+ "tokens_seen": 1835728896
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035865997993981944,
+ "loss": 1.2651,
+ "theoretical_loss": 3.483656304862406,
+ "tokens_seen": 1835794432
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035864393179538614,
+ "loss": 1.3312,
+ "theoretical_loss": 3.4836458466378972,
+ "tokens_seen": 1835859968
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035862788365095285,
+ "loss": 1.3201,
+ "theoretical_loss": 3.483635388891246,
+ "tokens_seen": 1835925504
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035861183550651955,
+ "loss": 1.2883,
+ "theoretical_loss": 3.4836249316224137,
+ "tokens_seen": 1835991040
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035859578736208625,
+ "loss": 1.2329,
+ "theoretical_loss": 3.4836144748313607,
+ "tokens_seen": 1836056576
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035857973921765295,
+ "loss": 1.4045,
+ "theoretical_loss": 3.4836040185180486,
+ "tokens_seen": 1836122112
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035856369107321965,
+ "loss": 1.2736,
+ "theoretical_loss": 3.4835935626824384,
+ "tokens_seen": 1836187648
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003585476429287864,
+ "loss": 1.247,
+ "theoretical_loss": 3.483583107324492,
+ "tokens_seen": 1836253184
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003585315947843531,
+ "loss": 1.2594,
+ "theoretical_loss": 3.4835726524441686,
+ "tokens_seen": 1836318720
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003585155466399198,
+ "loss": 1.2779,
+ "theoretical_loss": 3.4835621980414313,
+ "tokens_seen": 1836384256
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584994984954865,
+ "loss": 1.3443,
+ "theoretical_loss": 3.48355174411624,
+ "tokens_seen": 1836449792
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584834503510532,
+ "loss": 1.2156,
+ "theoretical_loss": 3.4835412906685566,
+ "tokens_seen": 1836515328
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584674022066199,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4835308376983423,
+ "tokens_seen": 1836580864
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584513540621866,
+ "loss": 1.2937,
+ "theoretical_loss": 3.4835203852055576,
+ "tokens_seen": 1836646400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584353059177533,
+ "loss": 1.2619,
+ "theoretical_loss": 3.483509933190164,
+ "tokens_seen": 1836711936
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035841925777332,
+ "loss": 1.2072,
+ "theoretical_loss": 3.4834994816521228,
+ "tokens_seen": 1836777472
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003584032096288867,
+ "loss": 1.2671,
+ "theoretical_loss": 3.483489030591395,
+ "tokens_seen": 1836843008
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583871614844534,
+ "loss": 1.2276,
+ "theoretical_loss": 3.483478580007941,
+ "tokens_seen": 1836908544
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583711133400201,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4834681299017234,
+ "tokens_seen": 1836974080
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583550651955868,
+ "loss": 1.2861,
+ "theoretical_loss": 3.483457680272703,
+ "tokens_seen": 1837039616
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583390170511535,
+ "loss": 1.3095,
+ "theoretical_loss": 3.4834472311208406,
+ "tokens_seen": 1837105152
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583229689067202,
+ "loss": 1.4022,
+ "theoretical_loss": 3.4834367824460974,
+ "tokens_seen": 1837170688
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003583069207622869,
+ "loss": 1.3521,
+ "theoretical_loss": 3.483426334248435,
+ "tokens_seen": 1837236224
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003582908726178536,
+ "loss": 1.3364,
+ "theoretical_loss": 3.4834158865278138,
+ "tokens_seen": 1837301760
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003582748244734203,
+ "loss": 1.2759,
+ "theoretical_loss": 3.483405439284196,
+ "tokens_seen": 1837367296
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000358258776328987,
+ "loss": 1.3857,
+ "theoretical_loss": 3.483394992517542,
+ "tokens_seen": 1837432832
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003582427281845537,
+ "loss": 1.303,
+ "theoretical_loss": 3.4833845462278137,
+ "tokens_seen": 1837498368
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035822668004012036,
+ "loss": 1.3633,
+ "theoretical_loss": 3.483374100414972,
+ "tokens_seen": 1837563904
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035821063189568706,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4833636550789775,
+ "tokens_seen": 1837629440
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035819458375125376,
+ "loss": 1.14,
+ "theoretical_loss": 3.483353210219793,
+ "tokens_seen": 1837694976
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035817853560682046,
+ "loss": 1.2763,
+ "theoretical_loss": 3.483342765837378,
+ "tokens_seen": 1837760512
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035816248746238716,
+ "loss": 1.315,
+ "theoretical_loss": 3.4833323219316945,
+ "tokens_seen": 1837826048
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035814643931795386,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4833218785027036,
+ "tokens_seen": 1837891584
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035813039117352057,
+ "loss": 1.3048,
+ "theoretical_loss": 3.483311435550367,
+ "tokens_seen": 1837957120
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035811434302908727,
+ "loss": 1.226,
+ "theoretical_loss": 3.4833009930746455,
+ "tokens_seen": 1838022656
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035809829488465397,
+ "loss": 1.3317,
+ "theoretical_loss": 3.483290551075501,
+ "tokens_seen": 1838088192
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035808224674022067,
+ "loss": 1.2434,
+ "theoretical_loss": 3.4832801095528936,
+ "tokens_seen": 1838153728
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035806619859578737,
+ "loss": 1.3287,
+ "theoretical_loss": 3.483269668506785,
+ "tokens_seen": 1838219264
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 616389,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1942389011383057,
+ "objective/train/theoretical_loss": 3.483259227937137,
+ "objective/train/tokens_used": 1858744800,
+ "theoretical_loss": 3.483259227937137,
+ "tokens_seen": 1838284800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035805015045135407,
+ "loss": 1.2035,
+ "theoretical_loss": 3.483259227937137,
+ "tokens_seen": 1838284800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035803410230692077,
+ "loss": 1.3313,
+ "theoretical_loss": 3.4832487878439107,
+ "tokens_seen": 1838350336
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035801805416248747,
+ "loss": 1.279,
+ "theoretical_loss": 3.4832383482270672,
+ "tokens_seen": 1838415872
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035800200601805417,
+ "loss": 1.2452,
+ "theoretical_loss": 3.483227909086568,
+ "tokens_seen": 1838481408
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035798595787362087,
+ "loss": 1.2942,
+ "theoretical_loss": 3.4832174704223737,
+ "tokens_seen": 1838546944
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035796990972918757,
+ "loss": 1.2314,
+ "theoretical_loss": 3.4832070322344464,
+ "tokens_seen": 1838612480
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035795386158475427,
+ "loss": 1.2063,
+ "theoretical_loss": 3.483196594522747,
+ "tokens_seen": 1838678016
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000357937813440321,
+ "loss": 1.3389,
+ "theoretical_loss": 3.4831861572872365,
+ "tokens_seen": 1838743552
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003579217652958877,
+ "loss": 1.3447,
+ "theoretical_loss": 3.483175720527877,
+ "tokens_seen": 1838809088
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003579057171514544,
+ "loss": 1.3295,
+ "theoretical_loss": 3.4831652842446292,
+ "tokens_seen": 1838874624
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578896690070211,
+ "loss": 1.2889,
+ "theoretical_loss": 3.483154848437455,
+ "tokens_seen": 1838940160
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578736208625878,
+ "loss": 1.2895,
+ "theoretical_loss": 3.4831444131063147,
+ "tokens_seen": 1839005696
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578575727181545,
+ "loss": 1.251,
+ "theoretical_loss": 3.4831339782511703,
+ "tokens_seen": 1839071232
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578415245737212,
+ "loss": 1.2106,
+ "theoretical_loss": 3.483123543871984,
+ "tokens_seen": 1839136768
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578254764292879,
+ "loss": 1.2716,
+ "theoretical_loss": 3.4831131099687154,
+ "tokens_seen": 1839202304
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003578094282848546,
+ "loss": 1.296,
+ "theoretical_loss": 3.4831026765413267,
+ "tokens_seen": 1839267840
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003577933801404213,
+ "loss": 1.2149,
+ "theoretical_loss": 3.4830922435897786,
+ "tokens_seen": 1839333376
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000357777331995988,
+ "loss": 1.2302,
+ "theoretical_loss": 3.483081811114034,
+ "tokens_seen": 1839398912
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003577612838515547,
+ "loss": 1.3389,
+ "theoretical_loss": 3.483071379114053,
+ "tokens_seen": 1839464448
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003577452357071214,
+ "loss": 1.4062,
+ "theoretical_loss": 3.483060947589797,
+ "tokens_seen": 1839529984
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003577291875626881,
+ "loss": 1.253,
+ "theoretical_loss": 3.483050516541228,
+ "tokens_seen": 1839595520
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003577131394182548,
+ "loss": 1.2078,
+ "theoretical_loss": 3.483040085968306,
+ "tokens_seen": 1839661056
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576970912738215,
+ "loss": 1.2078,
+ "theoretical_loss": 3.483029655870994,
+ "tokens_seen": 1839726592
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576810431293882,
+ "loss": 1.1787,
+ "theoretical_loss": 3.4830192262492528,
+ "tokens_seen": 1839792128
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576649949849549,
+ "loss": 1.2702,
+ "theoretical_loss": 3.483008797103044,
+ "tokens_seen": 1839857664
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576489468405216,
+ "loss": 1.3047,
+ "theoretical_loss": 3.4829983684323276,
+ "tokens_seen": 1839923200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576328986960883,
+ "loss": 1.3219,
+ "theoretical_loss": 3.4829879402370665,
+ "tokens_seen": 1839988736
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000357616850551655,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4829775125172215,
+ "tokens_seen": 1840054272
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003576008024072217,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4829670852727546,
+ "tokens_seen": 1840119808
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575847542627884,
+ "loss": 1.3876,
+ "theoretical_loss": 3.4829566585036265,
+ "tokens_seen": 1840185344
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575687061183551,
+ "loss": 1.2227,
+ "theoretical_loss": 3.4829462322097986,
+ "tokens_seen": 1840250880
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575526579739218,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4829358063912323,
+ "tokens_seen": 1840316416
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575366098294885,
+ "loss": 1.2178,
+ "theoretical_loss": 3.48292538104789,
+ "tokens_seen": 1840381952
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575205616850552,
+ "loss": 1.1665,
+ "theoretical_loss": 3.482914956179732,
+ "tokens_seen": 1840447488
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003575045135406219,
+ "loss": 1.2546,
+ "theoretical_loss": 3.48290453178672,
+ "tokens_seen": 1840513024
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003574884653961886,
+ "loss": 1.3041,
+ "theoretical_loss": 3.482894107868815,
+ "tokens_seen": 1840578560
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003574724172517553,
+ "loss": 1.2707,
+ "theoretical_loss": 3.4828836844259796,
+ "tokens_seen": 1840644096
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035745636910732194,
+ "loss": 1.3621,
+ "theoretical_loss": 3.4828732614581743,
+ "tokens_seen": 1840709632
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035744032096288864,
+ "loss": 1.3637,
+ "theoretical_loss": 3.482862838965361,
+ "tokens_seen": 1840775168
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035742427281845534,
+ "loss": 1.2813,
+ "theoretical_loss": 3.482852416947501,
+ "tokens_seen": 1840840704
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035740822467402204,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4828419954045553,
+ "tokens_seen": 1840906240
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035739217652958874,
+ "loss": 1.2187,
+ "theoretical_loss": 3.482831574336486,
+ "tokens_seen": 1840971776
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035737612838515544,
+ "loss": 1.1802,
+ "theoretical_loss": 3.482821153743254,
+ "tokens_seen": 1841037312
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035736008024072214,
+ "loss": 1.3933,
+ "theoretical_loss": 3.4828107336248215,
+ "tokens_seen": 1841102848
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003573440320962889,
+ "loss": 1.3383,
+ "theoretical_loss": 3.4828003139811496,
+ "tokens_seen": 1841168384
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003573279839518556,
+ "loss": 1.2194,
+ "theoretical_loss": 3.4827898948121994,
+ "tokens_seen": 1841233920
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003573119358074223,
+ "loss": 1.3819,
+ "theoretical_loss": 3.4827794761179325,
+ "tokens_seen": 1841299456
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000357295887662989,
+ "loss": 1.2321,
+ "theoretical_loss": 3.4827690578983113,
+ "tokens_seen": 1841364992
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003572798395185557,
+ "loss": 1.3872,
+ "theoretical_loss": 3.4827586401532957,
+ "tokens_seen": 1841430528
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003572637913741224,
+ "loss": 1.2951,
+ "theoretical_loss": 3.4827482228828486,
+ "tokens_seen": 1841496064
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 617553,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2685306072235107,
+ "objective/train/theoretical_loss": 3.4827378060869307,
+ "objective/train/tokens_used": 1862021600,
+ "theoretical_loss": 3.4827378060869307,
+ "tokens_seen": 1841561600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003572477432296891,
+ "loss": 1.2613,
+ "theoretical_loss": 3.4827378060869307,
+ "tokens_seen": 1841561600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003572316950852558,
+ "loss": 1.2346,
+ "theoretical_loss": 3.482727389765504,
+ "tokens_seen": 1841627136
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003572156469408225,
+ "loss": 1.2575,
+ "theoretical_loss": 3.4827169739185293,
+ "tokens_seen": 1841692672
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571995987963892,
+ "loss": 1.3158,
+ "theoretical_loss": 3.482706558545969,
+ "tokens_seen": 1841758208
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571835506519559,
+ "loss": 1.3707,
+ "theoretical_loss": 3.4826961436477837,
+ "tokens_seen": 1841823744
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571675025075226,
+ "loss": 1.1855,
+ "theoretical_loss": 3.4826857292239355,
+ "tokens_seen": 1841889280
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571514543630893,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4826753152743857,
+ "tokens_seen": 1841954816
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000357135406218656,
+ "loss": 1.2439,
+ "theoretical_loss": 3.4826649017990965,
+ "tokens_seen": 1842020352
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571193580742227,
+ "loss": 1.2445,
+ "theoretical_loss": 3.482654488798028,
+ "tokens_seen": 1842085888
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003571033099297894,
+ "loss": 1.2524,
+ "theoretical_loss": 3.482644076271143,
+ "tokens_seen": 1842151424
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570872617853561,
+ "loss": 1.1124,
+ "theoretical_loss": 3.482633664218403,
+ "tokens_seen": 1842216960
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570712136409228,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4826232526397685,
+ "tokens_seen": 1842282496
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570551654964895,
+ "loss": 1.3294,
+ "theoretical_loss": 3.482612841535202,
+ "tokens_seen": 1842348032
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570391173520562,
+ "loss": 1.2951,
+ "theoretical_loss": 3.482602430904665,
+ "tokens_seen": 1842413568
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570230692076229,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4825920207481182,
+ "tokens_seen": 1842479104
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003570070210631896,
+ "loss": 1.2925,
+ "theoretical_loss": 3.482581611065524,
+ "tokens_seen": 1842544640
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003569909729187563,
+ "loss": 1.4029,
+ "theoretical_loss": 3.482571201856844,
+ "tokens_seen": 1842610176
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000356974924774323,
+ "loss": 1.3022,
+ "theoretical_loss": 3.4825607931220395,
+ "tokens_seen": 1842675712
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003569588766298897,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4825503848610717,
+ "tokens_seen": 1842741248
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003569428284854564,
+ "loss": 1.2332,
+ "theoretical_loss": 3.4825399770739027,
+ "tokens_seen": 1842806784
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003569267803410231,
+ "loss": 1.3129,
+ "theoretical_loss": 3.4825295697604943,
+ "tokens_seen": 1842872320
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003569107321965898,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4825191629208074,
+ "tokens_seen": 1842937856
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003568946840521565,
+ "loss": 1.3705,
+ "theoretical_loss": 3.482508756554804,
+ "tokens_seen": 1843003392
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003568786359077232,
+ "loss": 1.2985,
+ "theoretical_loss": 3.4824983506624454,
+ "tokens_seen": 1843068928
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003568625877632899,
+ "loss": 1.4798,
+ "theoretical_loss": 3.482487945243694,
+ "tokens_seen": 1843134464
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003568465396188566,
+ "loss": 1.2527,
+ "theoretical_loss": 3.48247754029851,
+ "tokens_seen": 1843200000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003568304914744233,
+ "loss": 1.1568,
+ "theoretical_loss": 3.4824671358268566,
+ "tokens_seen": 1843265536
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035681444332998996,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4824567318286945,
+ "tokens_seen": 1843331072
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035679839518555666,
+ "loss": 1.2176,
+ "theoretical_loss": 3.4824463283039853,
+ "tokens_seen": 1843396608
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035678234704112336,
+ "loss": 1.3636,
+ "theoretical_loss": 3.482435925252691,
+ "tokens_seen": 1843462144
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035676629889669007,
+ "loss": 1.2504,
+ "theoretical_loss": 3.482425522674773,
+ "tokens_seen": 1843527680
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035675025075225677,
+ "loss": 1.384,
+ "theoretical_loss": 3.4824151205701925,
+ "tokens_seen": 1843593216
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035673420260782347,
+ "loss": 1.2453,
+ "theoretical_loss": 3.482404718938912,
+ "tokens_seen": 1843658752
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035671815446339017,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4823943177808925,
+ "tokens_seen": 1843724288
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035670210631895687,
+ "loss": 1.1321,
+ "theoretical_loss": 3.4823839170960964,
+ "tokens_seen": 1843789824
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035668605817452357,
+ "loss": 1.3099,
+ "theoretical_loss": 3.4823735168844845,
+ "tokens_seen": 1843855360
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035667001003009027,
+ "loss": 1.2082,
+ "theoretical_loss": 3.4823631171460185,
+ "tokens_seen": 1843920896
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035665396188565697,
+ "loss": 1.273,
+ "theoretical_loss": 3.482352717880661,
+ "tokens_seen": 1843986432
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035663791374122367,
+ "loss": 1.288,
+ "theoretical_loss": 3.4823423190883727,
+ "tokens_seen": 1844051968
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035662186559679037,
+ "loss": 1.3538,
+ "theoretical_loss": 3.4823319207691155,
+ "tokens_seen": 1844117504
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035660581745235707,
+ "loss": 1.3,
+ "theoretical_loss": 3.482321522922851,
+ "tokens_seen": 1844183040
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035658976930792377,
+ "loss": 1.2148,
+ "theoretical_loss": 3.482311125549541,
+ "tokens_seen": 1844248576
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035657372116349047,
+ "loss": 1.3326,
+ "theoretical_loss": 3.4823007286491476,
+ "tokens_seen": 1844314112
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003565576730190572,
+ "loss": 1.2638,
+ "theoretical_loss": 3.482290332221632,
+ "tokens_seen": 1844379648
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003565416248746239,
+ "loss": 1.3407,
+ "theoretical_loss": 3.4822799362669556,
+ "tokens_seen": 1844445184
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003565255767301906,
+ "loss": 1.3225,
+ "theoretical_loss": 3.482269540785081,
+ "tokens_seen": 1844510720
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003565095285857573,
+ "loss": 1.2919,
+ "theoretical_loss": 3.482259145775969,
+ "tokens_seen": 1844576256
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000356493480441324,
+ "loss": 1.3169,
+ "theoretical_loss": 3.482248751239582,
+ "tokens_seen": 1844641792
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003564774322968907,
+ "loss": 1.2356,
+ "theoretical_loss": 3.4822383571758815,
+ "tokens_seen": 1844707328
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003564613841524574,
+ "loss": 1.2596,
+ "theoretical_loss": 3.482227963584829,
+ "tokens_seen": 1844772864
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 618553,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.335506558418274,
+ "objective/train/theoretical_loss": 3.4822175704663865,
+ "objective/train/tokens_used": 1865298400,
+ "theoretical_loss": 3.4822175704663865,
+ "tokens_seen": 1844838400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003564453360080241,
+ "loss": 1.2785,
+ "theoretical_loss": 3.4822175704663865,
+ "tokens_seen": 1844838400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003564292878635908,
+ "loss": 1.2987,
+ "theoretical_loss": 3.4822071778205146,
+ "tokens_seen": 1844903936
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003564132397191575,
+ "loss": 1.2812,
+ "theoretical_loss": 3.4821967856471767,
+ "tokens_seen": 1844969472
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563971915747242,
+ "loss": 1.2096,
+ "theoretical_loss": 3.482186393946334,
+ "tokens_seen": 1845035008
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563811434302909,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4821760027179476,
+ "tokens_seen": 1845100544
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563650952858576,
+ "loss": 1.2864,
+ "theoretical_loss": 3.48216561196198,
+ "tokens_seen": 1845166080
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563490471414243,
+ "loss": 1.3675,
+ "theoretical_loss": 3.4821552216783926,
+ "tokens_seen": 1845231616
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000356332998996991,
+ "loss": 1.2629,
+ "theoretical_loss": 3.482144831867147,
+ "tokens_seen": 1845297152
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563169508525577,
+ "loss": 1.2266,
+ "theoretical_loss": 3.4821344425282055,
+ "tokens_seen": 1845362688
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003563009027081244,
+ "loss": 1.3251,
+ "theoretical_loss": 3.482124053661529,
+ "tokens_seen": 1845428224
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562848545636911,
+ "loss": 1.27,
+ "theoretical_loss": 3.48211366526708,
+ "tokens_seen": 1845493760
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562688064192578,
+ "loss": 1.2866,
+ "theoretical_loss": 3.48210327734482,
+ "tokens_seen": 1845559296
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562527582748245,
+ "loss": 1.1667,
+ "theoretical_loss": 3.4820928898947106,
+ "tokens_seen": 1845624832
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562367101303912,
+ "loss": 1.319,
+ "theoretical_loss": 3.482082502916714,
+ "tokens_seen": 1845690368
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562206619859579,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4820721164107917,
+ "tokens_seen": 1845755904
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003562046138415246,
+ "loss": 1.2869,
+ "theoretical_loss": 3.4820617303769055,
+ "tokens_seen": 1845821440
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003561885656970913,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4820513448150177,
+ "tokens_seen": 1845886976
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000356172517552658,
+ "loss": 1.3266,
+ "theoretical_loss": 3.4820409597250888,
+ "tokens_seen": 1845952512
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003561564694082247,
+ "loss": 1.2993,
+ "theoretical_loss": 3.482030575107082,
+ "tokens_seen": 1846018048
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003561404212637914,
+ "loss": 1.3679,
+ "theoretical_loss": 3.482020190960958,
+ "tokens_seen": 1846083584
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003561243731193581,
+ "loss": 1.2383,
+ "theoretical_loss": 3.482009807286679,
+ "tokens_seen": 1846149120
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003561083249749248,
+ "loss": 1.3407,
+ "theoretical_loss": 3.481999424084208,
+ "tokens_seen": 1846214656
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003560922768304915,
+ "loss": 1.2877,
+ "theoretical_loss": 3.481989041353504,
+ "tokens_seen": 1846280192
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003560762286860582,
+ "loss": 1.2945,
+ "theoretical_loss": 3.481978659094532,
+ "tokens_seen": 1846345728
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003560601805416249,
+ "loss": 1.343,
+ "theoretical_loss": 3.481968277307252,
+ "tokens_seen": 1846411264
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003560441323971916,
+ "loss": 1.2717,
+ "theoretical_loss": 3.481957895991626,
+ "tokens_seen": 1846476800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003560280842527583,
+ "loss": 1.3299,
+ "theoretical_loss": 3.481947515147617,
+ "tokens_seen": 1846542336
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000356012036108325,
+ "loss": 1.3658,
+ "theoretical_loss": 3.4819371347751846,
+ "tokens_seen": 1846607872
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559959879638917,
+ "loss": 1.305,
+ "theoretical_loss": 3.4819267548742925,
+ "tokens_seen": 1846673408
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559799398194584,
+ "loss": 1.3971,
+ "theoretical_loss": 3.481916375444902,
+ "tokens_seen": 1846738944
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559638916750251,
+ "loss": 1.2732,
+ "theoretical_loss": 3.481905996486975,
+ "tokens_seen": 1846804480
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559478435305918,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4818956180004728,
+ "tokens_seen": 1846870016
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559317953861585,
+ "loss": 1.2968,
+ "theoretical_loss": 3.481885239985358,
+ "tokens_seen": 1846935552
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003559157472417252,
+ "loss": 1.2193,
+ "theoretical_loss": 3.4818748624415923,
+ "tokens_seen": 1847001088
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558996990972919,
+ "loss": 1.2138,
+ "theoretical_loss": 3.4818644853691376,
+ "tokens_seen": 1847066624
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558836509528586,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4818541087679558,
+ "tokens_seen": 1847132160
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558676028084253,
+ "loss": 1.3527,
+ "theoretical_loss": 3.481843732638008,
+ "tokens_seen": 1847197696
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000355851554663992,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4818333569792568,
+ "tokens_seen": 1847263232
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558355065195587,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4818229817916646,
+ "tokens_seen": 1847328768
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558194583751254,
+ "loss": 1.189,
+ "theoretical_loss": 3.4818126070751925,
+ "tokens_seen": 1847394304
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003558034102306921,
+ "loss": 1.289,
+ "theoretical_loss": 3.481802232829802,
+ "tokens_seen": 1847459840
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557873620862588,
+ "loss": 1.217,
+ "theoretical_loss": 3.4817918590554564,
+ "tokens_seen": 1847525376
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557713139418255,
+ "loss": 1.3107,
+ "theoretical_loss": 3.481781485752116,
+ "tokens_seen": 1847590912
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557552657973922,
+ "loss": 1.3368,
+ "theoretical_loss": 3.481771112919744,
+ "tokens_seen": 1847656448
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557392176529589,
+ "loss": 1.2753,
+ "theoretical_loss": 3.4817607405583013,
+ "tokens_seen": 1847721984
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557231695085256,
+ "loss": 1.2219,
+ "theoretical_loss": 3.481750368667751,
+ "tokens_seen": 1847787520
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003557071213640923,
+ "loss": 1.2997,
+ "theoretical_loss": 3.481739997248054,
+ "tokens_seen": 1847853056
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000355691073219659,
+ "loss": 1.3898,
+ "theoretical_loss": 3.4817296262991726,
+ "tokens_seen": 1847918592
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003556750250752257,
+ "loss": 1.2225,
+ "theoretical_loss": 3.481719255821069,
+ "tokens_seen": 1847984128
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003556589769307924,
+ "loss": 1.291,
+ "theoretical_loss": 3.4817088858137044,
+ "tokens_seen": 1848049664
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 619773,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3261522054672241,
+ "objective/train/theoretical_loss": 3.4816985162770413,
+ "objective/train/tokens_used": 1868575200,
+ "theoretical_loss": 3.4816985162770413,
+ "tokens_seen": 1848115200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003556429287863591,
+ "loss": 1.2588,
+ "theoretical_loss": 3.4816985162770413,
+ "tokens_seen": 1848115200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003556268806419258,
+ "loss": 1.2595,
+ "theoretical_loss": 3.4816881472110417,
+ "tokens_seen": 1848180736
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003556108324974925,
+ "loss": 1.2753,
+ "theoretical_loss": 3.481677778615667,
+ "tokens_seen": 1848246272
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003555947843530592,
+ "loss": 1.2621,
+ "theoretical_loss": 3.48166741049088,
+ "tokens_seen": 1848311808
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003555787362086259,
+ "loss": 1.167,
+ "theoretical_loss": 3.481657042836642,
+ "tokens_seen": 1848377344
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003555626880641926,
+ "loss": 1.2358,
+ "theoretical_loss": 3.4816466756529145,
+ "tokens_seen": 1848442880
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003555466399197593,
+ "loss": 1.176,
+ "theoretical_loss": 3.481636308939661,
+ "tokens_seen": 1848508416
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000355530591775326,
+ "loss": 1.3824,
+ "theoretical_loss": 3.481625942696842,
+ "tokens_seen": 1848573952
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003555145436308927,
+ "loss": 1.3081,
+ "theoretical_loss": 3.481615576924421,
+ "tokens_seen": 1848639488
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003554984954864594,
+ "loss": 1.3239,
+ "theoretical_loss": 3.481605211622358,
+ "tokens_seen": 1848705024
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003554824473420261,
+ "loss": 1.2946,
+ "theoretical_loss": 3.481594846790616,
+ "tokens_seen": 1848770560
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003554663991975928,
+ "loss": 1.2138,
+ "theoretical_loss": 3.4815844824291577,
+ "tokens_seen": 1848836096
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003554503510531595,
+ "loss": 1.1801,
+ "theoretical_loss": 3.481574118537944,
+ "tokens_seen": 1848901632
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003554343029087262,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4815637551169374,
+ "tokens_seen": 1848967168
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035541825476429286,
+ "loss": 1.467,
+ "theoretical_loss": 3.4815533921660995,
+ "tokens_seen": 1849032704
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035540220661985956,
+ "loss": 1.1877,
+ "theoretical_loss": 3.481543029685393,
+ "tokens_seen": 1849098240
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035538615847542627,
+ "loss": 1.2843,
+ "theoretical_loss": 3.4815326676747795,
+ "tokens_seen": 1849163776
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035537011033099297,
+ "loss": 1.3301,
+ "theoretical_loss": 3.481522306134221,
+ "tokens_seen": 1849229312
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035535406218655967,
+ "loss": 1.3068,
+ "theoretical_loss": 3.481511945063679,
+ "tokens_seen": 1849294848
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035533801404212637,
+ "loss": 1.1361,
+ "theoretical_loss": 3.4815015844631167,
+ "tokens_seen": 1849360384
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035532196589769307,
+ "loss": 1.3086,
+ "theoretical_loss": 3.481491224332496,
+ "tokens_seen": 1849425920
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035530591775325977,
+ "loss": 1.1684,
+ "theoretical_loss": 3.481480864671777,
+ "tokens_seen": 1849491456
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035528986960882647,
+ "loss": 1.3715,
+ "theoretical_loss": 3.4814705054809245,
+ "tokens_seen": 1849556992
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035527382146439317,
+ "loss": 1.322,
+ "theoretical_loss": 3.4814601467598987,
+ "tokens_seen": 1849622528
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035525777331995987,
+ "loss": 1.1257,
+ "theoretical_loss": 3.4814497885086624,
+ "tokens_seen": 1849688064
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035524172517552657,
+ "loss": 1.2464,
+ "theoretical_loss": 3.481439430727177,
+ "tokens_seen": 1849753600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035522567703109327,
+ "loss": 1.1682,
+ "theoretical_loss": 3.4814290734154048,
+ "tokens_seen": 1849819136
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035520962888665997,
+ "loss": 1.2991,
+ "theoretical_loss": 3.4814187165733084,
+ "tokens_seen": 1849884672
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551935807422267,
+ "loss": 1.1955,
+ "theoretical_loss": 3.4814083602008496,
+ "tokens_seen": 1849950208
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551775325977934,
+ "loss": 1.326,
+ "theoretical_loss": 3.48139800429799,
+ "tokens_seen": 1850015744
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551614844533601,
+ "loss": 1.1748,
+ "theoretical_loss": 3.4813876488646924,
+ "tokens_seen": 1850081280
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551454363089268,
+ "loss": 1.3308,
+ "theoretical_loss": 3.481377293900918,
+ "tokens_seen": 1850146816
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551293881644935,
+ "loss": 1.2837,
+ "theoretical_loss": 3.48136693940663,
+ "tokens_seen": 1850212352
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003551133400200602,
+ "loss": 1.1984,
+ "theoretical_loss": 3.481356585381789,
+ "tokens_seen": 1850277888
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550972918756269,
+ "loss": 1.3163,
+ "theoretical_loss": 3.481346231826359,
+ "tokens_seen": 1850343424
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550812437311936,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4813358787403006,
+ "tokens_seen": 1850408960
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550651955867603,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4813255261235763,
+ "tokens_seen": 1850474496
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000355049147442327,
+ "loss": 1.2722,
+ "theoretical_loss": 3.481315173976148,
+ "tokens_seen": 1850540032
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550330992978937,
+ "loss": 1.136,
+ "theoretical_loss": 3.4813048222979783,
+ "tokens_seen": 1850605568
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550170511534604,
+ "loss": 1.2749,
+ "theoretical_loss": 3.4812944710890292,
+ "tokens_seen": 1850671104
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003550010030090271,
+ "loss": 1.3021,
+ "theoretical_loss": 3.4812841203492626,
+ "tokens_seen": 1850736640
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003549849548645938,
+ "loss": 1.2612,
+ "theoretical_loss": 3.4812737700786407,
+ "tokens_seen": 1850802176
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003549689067201605,
+ "loss": 1.273,
+ "theoretical_loss": 3.4812634202771258,
+ "tokens_seen": 1850867712
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035495285857572724,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4812530709446796,
+ "tokens_seen": 1850933248
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035493681043129394,
+ "loss": 1.2257,
+ "theoretical_loss": 3.481242722081265,
+ "tokens_seen": 1850998784
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035492076228686064,
+ "loss": 1.2262,
+ "theoretical_loss": 3.481232373686843,
+ "tokens_seen": 1851064320
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035490471414242734,
+ "loss": 1.2075,
+ "theoretical_loss": 3.481222025761377,
+ "tokens_seen": 1851129856
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035488866599799404,
+ "loss": 1.2839,
+ "theoretical_loss": 3.4812116783048275,
+ "tokens_seen": 1851195392
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035487261785356074,
+ "loss": 1.1574,
+ "theoretical_loss": 3.4812013313171586,
+ "tokens_seen": 1851260928
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035485656970912744,
+ "loss": 1.1607,
+ "theoretical_loss": 3.4811909847983316,
+ "tokens_seen": 1851326464
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 620645,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.138369083404541,
+ "objective/train/theoretical_loss": 3.4811806387483077,
+ "objective/train/tokens_used": 1871852000,
+ "theoretical_loss": 3.4811806387483077,
+ "tokens_seen": 1851392000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035484052156469414,
+ "loss": 1.2111,
+ "theoretical_loss": 3.4811806387483077,
+ "tokens_seen": 1851392000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035482447342026084,
+ "loss": 1.2684,
+ "theoretical_loss": 3.4811702931670507,
+ "tokens_seen": 1851457536
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035480842527582754,
+ "loss": 1.2848,
+ "theoretical_loss": 3.481159948054522,
+ "tokens_seen": 1851523072
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003547923771313942,
+ "loss": 1.185,
+ "theoretical_loss": 3.4811496034106835,
+ "tokens_seen": 1851588608
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003547763289869609,
+ "loss": 1.227,
+ "theoretical_loss": 3.4811392592354977,
+ "tokens_seen": 1851654144
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003547602808425276,
+ "loss": 1.1717,
+ "theoretical_loss": 3.4811289155289273,
+ "tokens_seen": 1851719680
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003547442326980943,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4811185722909332,
+ "tokens_seen": 1851785216
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000354728184553661,
+ "loss": 1.3275,
+ "theoretical_loss": 3.481108229521479,
+ "tokens_seen": 1851850752
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003547121364092277,
+ "loss": 1.3113,
+ "theoretical_loss": 3.4810978872205256,
+ "tokens_seen": 1851916288
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546960882647944,
+ "loss": 1.2839,
+ "theoretical_loss": 3.4810875453880366,
+ "tokens_seen": 1851981824
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546800401203611,
+ "loss": 1.1094,
+ "theoretical_loss": 3.481077204023973,
+ "tokens_seen": 1852047360
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546639919759278,
+ "loss": 1.2206,
+ "theoretical_loss": 3.481066863128297,
+ "tokens_seen": 1852112896
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546479438314945,
+ "loss": 1.1729,
+ "theoretical_loss": 3.481056522700972,
+ "tokens_seen": 1852178432
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546318956870612,
+ "loss": 1.2434,
+ "theoretical_loss": 3.4810461827419594,
+ "tokens_seen": 1852243968
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003546158475426279,
+ "loss": 1.1684,
+ "theoretical_loss": 3.481035843251221,
+ "tokens_seen": 1852309504
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545997993981946,
+ "loss": 1.1849,
+ "theoretical_loss": 3.48102550422872,
+ "tokens_seen": 1852375040
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545837512537613,
+ "loss": 1.1773,
+ "theoretical_loss": 3.481015165674418,
+ "tokens_seen": 1852440576
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000354567703109328,
+ "loss": 1.2843,
+ "theoretical_loss": 3.4810048275882775,
+ "tokens_seen": 1852506112
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545516549648947,
+ "loss": 1.1171,
+ "theoretical_loss": 3.4809944899702603,
+ "tokens_seen": 1852571648
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545356068204614,
+ "loss": 1.2333,
+ "theoretical_loss": 3.480984152820329,
+ "tokens_seen": 1852637184
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545195586760281,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4809738161384463,
+ "tokens_seen": 1852702720
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003545035105315948,
+ "loss": 1.3383,
+ "theoretical_loss": 3.4809634799245734,
+ "tokens_seen": 1852768256
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003544874623871615,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4809531441786734,
+ "tokens_seen": 1852833792
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003544714142427282,
+ "loss": 1.3029,
+ "theoretical_loss": 3.4809428089007084,
+ "tokens_seen": 1852899328
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003544553660982949,
+ "loss": 1.2386,
+ "theoretical_loss": 3.4809324740906407,
+ "tokens_seen": 1852964864
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003544393179538616,
+ "loss": 1.2548,
+ "theoretical_loss": 3.480922139748432,
+ "tokens_seen": 1853030400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003544232698094283,
+ "loss": 1.3125,
+ "theoretical_loss": 3.480911805874045,
+ "tokens_seen": 1853095936
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000354407221664995,
+ "loss": 1.3323,
+ "theoretical_loss": 3.4809014724674423,
+ "tokens_seen": 1853161472
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543911735205617,
+ "loss": 1.3229,
+ "theoretical_loss": 3.4808911395285858,
+ "tokens_seen": 1853227008
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543751253761284,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4808808070574377,
+ "tokens_seen": 1853292544
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543590772316951,
+ "loss": 1.2726,
+ "theoretical_loss": 3.48087047505396,
+ "tokens_seen": 1853358080
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543430290872618,
+ "loss": 1.2289,
+ "theoretical_loss": 3.4808601435181163,
+ "tokens_seen": 1853423616
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543269809428285,
+ "loss": 1.1585,
+ "theoretical_loss": 3.480849812449868,
+ "tokens_seen": 1853489152
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003543109327983952,
+ "loss": 1.1852,
+ "theoretical_loss": 3.480839481849177,
+ "tokens_seen": 1853554688
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003542948846539619,
+ "loss": 1.2325,
+ "theoretical_loss": 3.480829151716006,
+ "tokens_seen": 1853620224
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003542788365095286,
+ "loss": 1.3029,
+ "theoretical_loss": 3.480818822050318,
+ "tokens_seen": 1853685760
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003542627883650953,
+ "loss": 1.2026,
+ "theoretical_loss": 3.480808492852074,
+ "tokens_seen": 1853751296
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000354246740220662,
+ "loss": 1.3862,
+ "theoretical_loss": 3.4807981641212375,
+ "tokens_seen": 1853816832
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003542306920762287,
+ "loss": 1.2485,
+ "theoretical_loss": 3.48078783585777,
+ "tokens_seen": 1853882368
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003542146439317954,
+ "loss": 1.1972,
+ "theoretical_loss": 3.4807775080616343,
+ "tokens_seen": 1853947904
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541985957873621,
+ "loss": 1.1841,
+ "theoretical_loss": 3.4807671807327925,
+ "tokens_seen": 1854013440
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541825476429288,
+ "loss": 1.272,
+ "theoretical_loss": 3.480756853871207,
+ "tokens_seen": 1854078976
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541664994984955,
+ "loss": 1.521,
+ "theoretical_loss": 3.48074652747684,
+ "tokens_seen": 1854144512
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541504513540622,
+ "loss": 1.2621,
+ "theoretical_loss": 3.4807362015496546,
+ "tokens_seen": 1854210048
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541344032096289,
+ "loss": 1.3607,
+ "theoretical_loss": 3.480725876089612,
+ "tokens_seen": 1854275584
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541183550651956,
+ "loss": 1.2133,
+ "theoretical_loss": 3.480715551096676,
+ "tokens_seen": 1854341120
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003541023069207623,
+ "loss": 1.2679,
+ "theoretical_loss": 3.480705226570807,
+ "tokens_seen": 1854406656
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000354086258776329,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4806949025119693,
+ "tokens_seen": 1854472192
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003540702106318957,
+ "loss": 1.33,
+ "theoretical_loss": 3.480684578920124,
+ "tokens_seen": 1854537728
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003540541624874624,
+ "loss": 1.2586,
+ "theoretical_loss": 3.4806742557952344,
+ "tokens_seen": 1854603264
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 621939,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3597110509872437,
+ "objective/train/theoretical_loss": 3.4806639331372615,
+ "objective/train/tokens_used": 1875128800,
+ "theoretical_loss": 3.4806639331372615,
+ "tokens_seen": 1854668800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003540381143430291,
+ "loss": 1.2847,
+ "theoretical_loss": 3.4806639331372615,
+ "tokens_seen": 1854668800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035402206619859577,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4806536109461694,
+ "tokens_seen": 1854734336
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035400601805416247,
+ "loss": 1.1308,
+ "theoretical_loss": 3.4806432892219195,
+ "tokens_seen": 1854799872
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035398996990972917,
+ "loss": 1.1318,
+ "theoretical_loss": 3.4806329679644743,
+ "tokens_seen": 1854865408
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035397392176529587,
+ "loss": 1.3945,
+ "theoretical_loss": 3.480622647173796,
+ "tokens_seen": 1854930944
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035395787362086257,
+ "loss": 1.1835,
+ "theoretical_loss": 3.4806123268498474,
+ "tokens_seen": 1854996480
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035394182547642927,
+ "loss": 1.2976,
+ "theoretical_loss": 3.480602006992591,
+ "tokens_seen": 1855062016
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035392577733199597,
+ "loss": 1.4108,
+ "theoretical_loss": 3.480591687601989,
+ "tokens_seen": 1855127552
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035390972918756267,
+ "loss": 1.2807,
+ "theoretical_loss": 3.4805813686780036,
+ "tokens_seen": 1855193088
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035389368104312937,
+ "loss": 1.1922,
+ "theoretical_loss": 3.4805710502205973,
+ "tokens_seen": 1855258624
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035387763289869607,
+ "loss": 1.3072,
+ "theoretical_loss": 3.4805607322297325,
+ "tokens_seen": 1855324160
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035386158475426277,
+ "loss": 1.2357,
+ "theoretical_loss": 3.4805504147053723,
+ "tokens_seen": 1855389696
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035384553660982947,
+ "loss": 1.3204,
+ "theoretical_loss": 3.480540097647478,
+ "tokens_seen": 1855455232
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035382948846539617,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4805297810560134,
+ "tokens_seen": 1855520768
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003538134403209629,
+ "loss": 1.2275,
+ "theoretical_loss": 3.480519464930939,
+ "tokens_seen": 1855586304
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003537973921765296,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4805091492722195,
+ "tokens_seen": 1855651840
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003537813440320963,
+ "loss": 1.2677,
+ "theoretical_loss": 3.4804988340798158,
+ "tokens_seen": 1855717376
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000353765295887663,
+ "loss": 1.1013,
+ "theoretical_loss": 3.480488519353691,
+ "tokens_seen": 1855782912
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035374924774322973,
+ "loss": 1.1932,
+ "theoretical_loss": 3.480478205093807,
+ "tokens_seen": 1855848448
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035373319959879643,
+ "loss": 1.3038,
+ "theoretical_loss": 3.480467891300127,
+ "tokens_seen": 1855913984
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035371715145436313,
+ "loss": 1.3431,
+ "theoretical_loss": 3.4804575779726123,
+ "tokens_seen": 1855979520
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035370110330992983,
+ "loss": 1.1593,
+ "theoretical_loss": 3.480447265111227,
+ "tokens_seen": 1856045056
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035368505516549653,
+ "loss": 1.2414,
+ "theoretical_loss": 3.480436952715933,
+ "tokens_seen": 1856110592
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035366900702106323,
+ "loss": 1.2544,
+ "theoretical_loss": 3.4804266407866917,
+ "tokens_seen": 1856176128
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035365295887662993,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4804163293234667,
+ "tokens_seen": 1856241664
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035363691073219663,
+ "loss": 1.2375,
+ "theoretical_loss": 3.48040601832622,
+ "tokens_seen": 1856307200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035362086258776334,
+ "loss": 1.3539,
+ "theoretical_loss": 3.480395707794915,
+ "tokens_seen": 1856372736
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035360481444333004,
+ "loss": 1.1772,
+ "theoretical_loss": 3.4803853977295125,
+ "tokens_seen": 1856438272
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035358876629889674,
+ "loss": 1.3211,
+ "theoretical_loss": 3.4803750881299766,
+ "tokens_seen": 1856503808
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035357271815446344,
+ "loss": 1.1062,
+ "theoretical_loss": 3.480364778996269,
+ "tokens_seen": 1856569344
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035355667001003014,
+ "loss": 1.2429,
+ "theoretical_loss": 3.4803544703283524,
+ "tokens_seen": 1856634880
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035354062186559684,
+ "loss": 1.3015,
+ "theoretical_loss": 3.4803441621261895,
+ "tokens_seen": 1856700416
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035352457372116354,
+ "loss": 1.1434,
+ "theoretical_loss": 3.4803338543897424,
+ "tokens_seen": 1856765952
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035350852557673024,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4803235471189735,
+ "tokens_seen": 1856831488
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035349247743229694,
+ "loss": 1.3286,
+ "theoretical_loss": 3.4803132403138464,
+ "tokens_seen": 1856897024
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035347642928786364,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4803029339743228,
+ "tokens_seen": 1856962560
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035346038114343034,
+ "loss": 1.1949,
+ "theoretical_loss": 3.480292628100365,
+ "tokens_seen": 1857028096
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035344433299899704,
+ "loss": 1.2342,
+ "theoretical_loss": 3.480282322691936,
+ "tokens_seen": 1857093632
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035342828485456374,
+ "loss": 1.2792,
+ "theoretical_loss": 3.480272017748998,
+ "tokens_seen": 1857159168
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035341223671013044,
+ "loss": 1.2919,
+ "theoretical_loss": 3.4802617132715143,
+ "tokens_seen": 1857224704
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035339618856569714,
+ "loss": 1.3163,
+ "theoretical_loss": 3.4802514092594468,
+ "tokens_seen": 1857290240
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003533801404212638,
+ "loss": 1.1526,
+ "theoretical_loss": 3.4802411057127576,
+ "tokens_seen": 1857355776
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003533640922768305,
+ "loss": 1.3376,
+ "theoretical_loss": 3.4802308026314104,
+ "tokens_seen": 1857421312
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003533480441323972,
+ "loss": 1.2372,
+ "theoretical_loss": 3.480220500015367,
+ "tokens_seen": 1857486848
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003533319959879639,
+ "loss": 1.1392,
+ "theoretical_loss": 3.48021019786459,
+ "tokens_seen": 1857552384
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003533159478435306,
+ "loss": 1.2869,
+ "theoretical_loss": 3.480199896179043,
+ "tokens_seen": 1857617920
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532998996990973,
+ "loss": 1.0969,
+ "theoretical_loss": 3.480189594958687,
+ "tokens_seen": 1857683456
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000353283851554664,
+ "loss": 1.2201,
+ "theoretical_loss": 3.4801792942034853,
+ "tokens_seen": 1857748992
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532678034102307,
+ "loss": 1.2125,
+ "theoretical_loss": 3.480168993913401,
+ "tokens_seen": 1857814528
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532517552657974,
+ "loss": 1.2873,
+ "theoretical_loss": 3.480158694088396,
+ "tokens_seen": 1857880064
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 622851,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1701698303222656,
+ "objective/train/theoretical_loss": 3.480148394728433,
+ "objective/train/tokens_used": 1878405600,
+ "theoretical_loss": 3.480148394728433,
+ "tokens_seen": 1857945600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532357071213641,
+ "loss": 1.2884,
+ "theoretical_loss": 3.480148394728433,
+ "tokens_seen": 1857945600
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532196589769308,
+ "loss": 1.2617,
+ "theoretical_loss": 3.480138095833474,
+ "tokens_seen": 1858011136
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003532036108324975,
+ "loss": 1.2999,
+ "theoretical_loss": 3.480127797403483,
+ "tokens_seen": 1858076672
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003531875626880642,
+ "loss": 1.3064,
+ "theoretical_loss": 3.480117499438422,
+ "tokens_seen": 1858142208
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003531715145436309,
+ "loss": 1.313,
+ "theoretical_loss": 3.480107201938254,
+ "tokens_seen": 1858207744
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003531554663991976,
+ "loss": 1.2608,
+ "theoretical_loss": 3.48009690490294,
+ "tokens_seen": 1858273280
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003531394182547643,
+ "loss": 1.2381,
+ "theoretical_loss": 3.4800866083324444,
+ "tokens_seen": 1858338816
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000353123370110331,
+ "loss": 1.2377,
+ "theoretical_loss": 3.4800763122267293,
+ "tokens_seen": 1858404352
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003531073219658977,
+ "loss": 1.2618,
+ "theoretical_loss": 3.480066016585757,
+ "tokens_seen": 1858469888
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530912738214644,
+ "loss": 1.2694,
+ "theoretical_loss": 3.4800557214094905,
+ "tokens_seen": 1858535424
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530752256770311,
+ "loss": 1.1255,
+ "theoretical_loss": 3.4800454266978917,
+ "tokens_seen": 1858600960
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530591775325978,
+ "loss": 1.2449,
+ "theoretical_loss": 3.480035132450925,
+ "tokens_seen": 1858666496
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530431293881645,
+ "loss": 1.2354,
+ "theoretical_loss": 3.480024838668551,
+ "tokens_seen": 1858732032
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530270812437312,
+ "loss": 1.3882,
+ "theoretical_loss": 3.480014545350733,
+ "tokens_seen": 1858797568
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003530110330992979,
+ "loss": 1.364,
+ "theoretical_loss": 3.4800042524974346,
+ "tokens_seen": 1858863104
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003529949849548646,
+ "loss": 1.2397,
+ "theoretical_loss": 3.479993960108618,
+ "tokens_seen": 1858928640
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003529789368104313,
+ "loss": 1.246,
+ "theoretical_loss": 3.4799836681842446,
+ "tokens_seen": 1858994176
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000352962888665998,
+ "loss": 1.1804,
+ "theoretical_loss": 3.4799733767242786,
+ "tokens_seen": 1859059712
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003529468405215647,
+ "loss": 1.2561,
+ "theoretical_loss": 3.4799630857286825,
+ "tokens_seen": 1859125248
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003529307923771314,
+ "loss": 1.2491,
+ "theoretical_loss": 3.479952795197418,
+ "tokens_seen": 1859190784
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003529147442326981,
+ "loss": 1.2596,
+ "theoretical_loss": 3.479942505130449,
+ "tokens_seen": 1859256320
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528986960882648,
+ "loss": 1.2859,
+ "theoretical_loss": 3.4799322155277377,
+ "tokens_seen": 1859321856
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528826479438315,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4799219263892462,
+ "tokens_seen": 1859387392
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528665997993982,
+ "loss": 1.2556,
+ "theoretical_loss": 3.479911637714938,
+ "tokens_seen": 1859452928
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528505516549649,
+ "loss": 1.326,
+ "theoretical_loss": 3.4799013495047753,
+ "tokens_seen": 1859518464
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528345035105316,
+ "loss": 1.2371,
+ "theoretical_loss": 3.479891061758721,
+ "tokens_seen": 1859584000
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003528184553660983,
+ "loss": 1.2856,
+ "theoretical_loss": 3.479880774476738,
+ "tokens_seen": 1859649536
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000352802407221665,
+ "loss": 1.2215,
+ "theoretical_loss": 3.479870487658789,
+ "tokens_seen": 1859715072
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527863590772317,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4798602013048363,
+ "tokens_seen": 1859780608
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527703109327984,
+ "loss": 1.2099,
+ "theoretical_loss": 3.479849915414843,
+ "tokens_seen": 1859846144
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527542627883651,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4798396299887715,
+ "tokens_seen": 1859911680
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527382146439318,
+ "loss": 1.2559,
+ "theoretical_loss": 3.4798293450265847,
+ "tokens_seen": 1859977216
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527221664994985,
+ "loss": 1.2405,
+ "theoretical_loss": 3.4798190605282455,
+ "tokens_seen": 1860042752
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003527061183550652,
+ "loss": 1.207,
+ "theoretical_loss": 3.4798087764937167,
+ "tokens_seen": 1860108288
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003526900702106319,
+ "loss": 1.27,
+ "theoretical_loss": 3.4797984929229604,
+ "tokens_seen": 1860173824
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003526740220661986,
+ "loss": 1.2256,
+ "theoretical_loss": 3.47978820981594,
+ "tokens_seen": 1860239360
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003526579739217653,
+ "loss": 1.2341,
+ "theoretical_loss": 3.479777927172618,
+ "tokens_seen": 1860304896
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000352641925777332,
+ "loss": 1.3621,
+ "theoretical_loss": 3.479767644992957,
+ "tokens_seen": 1860370432
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003526258776328987,
+ "loss": 1.1453,
+ "theoretical_loss": 3.4797573632769203,
+ "tokens_seen": 1860435968
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035260982948846537,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4797470820244705,
+ "tokens_seen": 1860501504
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035259378134403207,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4797368012355694,
+ "tokens_seen": 1860567040
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035257773319959877,
+ "loss": 1.298,
+ "theoretical_loss": 3.479726520910181,
+ "tokens_seen": 1860632576
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035256168505516547,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4797162410482683,
+ "tokens_seen": 1860698112
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003525456369107322,
+ "loss": 1.2671,
+ "theoretical_loss": 3.4797059616497927,
+ "tokens_seen": 1860763648
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003525295887662989,
+ "loss": 1.1978,
+ "theoretical_loss": 3.4796956827147176,
+ "tokens_seen": 1860829184
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003525135406218656,
+ "loss": 1.1468,
+ "theoretical_loss": 3.479685404243006,
+ "tokens_seen": 1860894720
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003524974924774323,
+ "loss": 1.2522,
+ "theoretical_loss": 3.479675126234621,
+ "tokens_seen": 1860960256
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000352481444332999,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4796648486895245,
+ "tokens_seen": 1861025792
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003524653961885657,
+ "loss": 1.1668,
+ "theoretical_loss": 3.4796545716076803,
+ "tokens_seen": 1861091328
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035244934804413243,
+ "loss": 1.1882,
+ "theoretical_loss": 3.4796442949890505,
+ "tokens_seen": 1861156864
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 624035,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2446659803390503,
+ "objective/train/theoretical_loss": 3.4796340188335977,
+ "objective/train/tokens_used": 1881682400,
+ "theoretical_loss": 3.4796340188335977,
+ "tokens_seen": 1861222400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035243329989969913,
+ "loss": 1.1809,
+ "theoretical_loss": 3.4796340188335977,
+ "tokens_seen": 1861222400
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035241725175526583,
+ "loss": 1.1594,
+ "theoretical_loss": 3.4796237431412855,
+ "tokens_seen": 1861287936
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035240120361083253,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4796134679120767,
+ "tokens_seen": 1861353472
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035238515546639923,
+ "loss": 1.4019,
+ "theoretical_loss": 3.479603193145933,
+ "tokens_seen": 1861419008
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035236910732196593,
+ "loss": 1.3315,
+ "theoretical_loss": 3.4795929188428185,
+ "tokens_seen": 1861484544
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035235305917753263,
+ "loss": 1.2562,
+ "theoretical_loss": 3.4795826450026954,
+ "tokens_seen": 1861550080
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035233701103309933,
+ "loss": 1.2403,
+ "theoretical_loss": 3.479572371625527,
+ "tokens_seen": 1861615616
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035232096288866603,
+ "loss": 1.1111,
+ "theoretical_loss": 3.479562098711275,
+ "tokens_seen": 1861681152
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035230491474423273,
+ "loss": 1.2603,
+ "theoretical_loss": 3.479551826259904,
+ "tokens_seen": 1861746688
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035228886659979943,
+ "loss": 1.2011,
+ "theoretical_loss": 3.4795415542713757,
+ "tokens_seen": 1861812224
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035227281845536613,
+ "loss": 1.2842,
+ "theoretical_loss": 3.479531282745653,
+ "tokens_seen": 1861877760
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035225677031093283,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4795210116826985,
+ "tokens_seen": 1861943296
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035224072216649954,
+ "loss": 1.2592,
+ "theoretical_loss": 3.4795107410824757,
+ "tokens_seen": 1862008832
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035222467402206624,
+ "loss": 1.202,
+ "theoretical_loss": 3.4795004709449477,
+ "tokens_seen": 1862074368
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035220862587763294,
+ "loss": 1.3344,
+ "theoretical_loss": 3.479490201270077,
+ "tokens_seen": 1862139904
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035219257773319964,
+ "loss": 1.1294,
+ "theoretical_loss": 3.479479932057826,
+ "tokens_seen": 1862205440
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035217652958876634,
+ "loss": 1.3079,
+ "theoretical_loss": 3.479469663308158,
+ "tokens_seen": 1862270976
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035216048144433304,
+ "loss": 1.3409,
+ "theoretical_loss": 3.4794593950210357,
+ "tokens_seen": 1862336512
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035214443329989974,
+ "loss": 1.1971,
+ "theoretical_loss": 3.479449127196423,
+ "tokens_seen": 1862402048
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035212838515546644,
+ "loss": 1.1932,
+ "theoretical_loss": 3.479438859834281,
+ "tokens_seen": 1862467584
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035211233701103314,
+ "loss": 1.204,
+ "theoretical_loss": 3.479428592934574,
+ "tokens_seen": 1862533120
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035209628886659984,
+ "loss": 1.1678,
+ "theoretical_loss": 3.4794183264972647,
+ "tokens_seen": 1862598656
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035208024072216654,
+ "loss": 1.3539,
+ "theoretical_loss": 3.4794080605223154,
+ "tokens_seen": 1862664192
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035206419257773324,
+ "loss": 1.3683,
+ "theoretical_loss": 3.4793977950096893,
+ "tokens_seen": 1862729728
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035204814443329994,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4793875299593493,
+ "tokens_seen": 1862795264
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035203209628886664,
+ "loss": 1.2677,
+ "theoretical_loss": 3.4793772653712587,
+ "tokens_seen": 1862860800
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035201604814443334,
+ "loss": 1.2411,
+ "theoretical_loss": 3.47936700124538,
+ "tokens_seen": 1862926336
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.00035200000000000005,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4793567375816767,
+ "tokens_seen": 1862991872
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519839518555667,
+ "loss": 1.2394,
+ "theoretical_loss": 3.479346474380111,
+ "tokens_seen": 1863057408
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519679037111334,
+ "loss": 1.245,
+ "theoretical_loss": 3.4793362116406454,
+ "tokens_seen": 1863122944
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519518555667001,
+ "loss": 1.2949,
+ "theoretical_loss": 3.4793259493632442,
+ "tokens_seen": 1863188480
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519358074222668,
+ "loss": 1.3093,
+ "theoretical_loss": 3.47931568754787,
+ "tokens_seen": 1863254016
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519197592778335,
+ "loss": 1.3423,
+ "theoretical_loss": 3.479305426194485,
+ "tokens_seen": 1863319552
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003519037111334002,
+ "loss": 1.3179,
+ "theoretical_loss": 3.4792951653030526,
+ "tokens_seen": 1863385088
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003518876629889669,
+ "loss": 1.3124,
+ "theoretical_loss": 3.4792849048735364,
+ "tokens_seen": 1863450624
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003518716148445336,
+ "loss": 1.2778,
+ "theoretical_loss": 3.479274644905898,
+ "tokens_seen": 1863516160
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003518555667001003,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4792643854001017,
+ "tokens_seen": 1863581696
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000351839518555667,
+ "loss": 1.3245,
+ "theoretical_loss": 3.4792541263561096,
+ "tokens_seen": 1863647232
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003518234704112337,
+ "loss": 1.2322,
+ "theoretical_loss": 3.4792438677738846,
+ "tokens_seen": 1863712768
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003518074222668004,
+ "loss": 1.1972,
+ "theoretical_loss": 3.4792336096533907,
+ "tokens_seen": 1863778304
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517913741223671,
+ "loss": 1.1831,
+ "theoretical_loss": 3.479223351994589,
+ "tokens_seen": 1863843840
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517753259779338,
+ "loss": 1.2943,
+ "theoretical_loss": 3.479213094797445,
+ "tokens_seen": 1863909376
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517592778335005,
+ "loss": 1.3168,
+ "theoretical_loss": 3.4792028380619198,
+ "tokens_seen": 1863974912
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517432296890672,
+ "loss": 1.1766,
+ "theoretical_loss": 3.479192581787977,
+ "tokens_seen": 1864040448
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517271815446339,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4791823259755796,
+ "tokens_seen": 1864105984
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003517111334002006,
+ "loss": 1.302,
+ "theoretical_loss": 3.479172070624691,
+ "tokens_seen": 1864171520
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003516950852557673,
+ "loss": 1.2858,
+ "theoretical_loss": 3.4791618157352735,
+ "tokens_seen": 1864237056
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.000351679037111334,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4791515613072903,
+ "tokens_seen": 1864302592
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003516629889669007,
+ "loss": 1.3262,
+ "theoretical_loss": 3.4791413073407043,
+ "tokens_seen": 1864368128
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003516469408224674,
+ "loss": 1.2562,
+ "theoretical_loss": 3.479131053835479,
+ "tokens_seen": 1864433664
+ },
+ {
+ "epoch": 0.56,
+ "objective/train/docs_used": 625017,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2231545448303223,
+ "objective/train/theoretical_loss": 3.479120800791577,
+ "objective/train/tokens_used": 1884959200,
+ "theoretical_loss": 3.479120800791577,
+ "tokens_seen": 1864499200
+ },
+ {
+ "epoch": 0.56,
+ "learning_rate": 0.0003516308926780341,
+ "loss": 1.2109,
+ "theoretical_loss": 3.479120800791577,
+ "tokens_seen": 1864499200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003516148445336008,
+ "loss": 1.2546,
+ "theoretical_loss": 3.479110548208962,
+ "tokens_seen": 1864564736
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515987963891675,
+ "loss": 1.2542,
+ "theoretical_loss": 3.479100296087596,
+ "tokens_seen": 1864630272
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515827482447342,
+ "loss": 1.1893,
+ "theoretical_loss": 3.479090044427443,
+ "tokens_seen": 1864695808
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515667001003009,
+ "loss": 1.2244,
+ "theoretical_loss": 3.4790797932284656,
+ "tokens_seen": 1864761344
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515506519558676,
+ "loss": 1.3097,
+ "theoretical_loss": 3.4790695424906266,
+ "tokens_seen": 1864826880
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515346038114343,
+ "loss": 1.1156,
+ "theoretical_loss": 3.479059292213889,
+ "tokens_seen": 1864892416
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000351518555667001,
+ "loss": 1.2433,
+ "theoretical_loss": 3.479049042398217,
+ "tokens_seen": 1864957952
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003515025075225677,
+ "loss": 1.2961,
+ "theoretical_loss": 3.479038793043572,
+ "tokens_seen": 1865023488
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514864593781344,
+ "loss": 1.1828,
+ "theoretical_loss": 3.4790285441499185,
+ "tokens_seen": 1865089024
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514704112337011,
+ "loss": 1.1852,
+ "theoretical_loss": 3.4790182957172187,
+ "tokens_seen": 1865154560
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514543630892678,
+ "loss": 1.2386,
+ "theoretical_loss": 3.479008047745436,
+ "tokens_seen": 1865220096
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514383149448345,
+ "loss": 1.3009,
+ "theoretical_loss": 3.4789978002345334,
+ "tokens_seen": 1865285632
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514222668004012,
+ "loss": 1.309,
+ "theoretical_loss": 3.4789875531844743,
+ "tokens_seen": 1865351168
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003514062186559679,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4789773065952208,
+ "tokens_seen": 1865416704
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003513901705115346,
+ "loss": 1.2765,
+ "theoretical_loss": 3.4789670604667373,
+ "tokens_seen": 1865482240
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003513741223671013,
+ "loss": 1.2994,
+ "theoretical_loss": 3.4789568147989858,
+ "tokens_seen": 1865547776
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000351358074222668,
+ "loss": 1.3446,
+ "theoretical_loss": 3.47894656959193,
+ "tokens_seen": 1865613312
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003513420260782347,
+ "loss": 1.173,
+ "theoretical_loss": 3.478936324845533,
+ "tokens_seen": 1865678848
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003513259779338014,
+ "loss": 1.3353,
+ "theoretical_loss": 3.478926080559758,
+ "tokens_seen": 1865744384
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003513099297893681,
+ "loss": 1.2428,
+ "theoretical_loss": 3.4789158367345676,
+ "tokens_seen": 1865809920
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512938816449348,
+ "loss": 1.2132,
+ "theoretical_loss": 3.478905593369925,
+ "tokens_seen": 1865875456
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512778335005015,
+ "loss": 1.1979,
+ "theoretical_loss": 3.478895350465794,
+ "tokens_seen": 1865940992
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512617853560682,
+ "loss": 1.3416,
+ "theoretical_loss": 3.478885108022137,
+ "tokens_seen": 1866006528
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512457372116349,
+ "loss": 1.4879,
+ "theoretical_loss": 3.4788748660389173,
+ "tokens_seen": 1866072064
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512296890672016,
+ "loss": 1.3334,
+ "theoretical_loss": 3.478864624516098,
+ "tokens_seen": 1866137600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003512136409227683,
+ "loss": 1.2625,
+ "theoretical_loss": 3.4788543834536423,
+ "tokens_seen": 1866203136
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000351197592778335,
+ "loss": 1.1902,
+ "theoretical_loss": 3.4788441428515133,
+ "tokens_seen": 1866268672
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511815446339017,
+ "loss": 1.1718,
+ "theoretical_loss": 3.478833902709675,
+ "tokens_seen": 1866334208
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511654964894684,
+ "loss": 1.3965,
+ "theoretical_loss": 3.4788236630280887,
+ "tokens_seen": 1866399744
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511494483450351,
+ "loss": 1.2303,
+ "theoretical_loss": 3.478813423806719,
+ "tokens_seen": 1866465280
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511334002006018,
+ "loss": 1.1672,
+ "theoretical_loss": 3.478803185045529,
+ "tokens_seen": 1866530816
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511173520561685,
+ "loss": 1.2076,
+ "theoretical_loss": 3.4787929467444814,
+ "tokens_seen": 1866596352
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003511013039117352,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4787827089035397,
+ "tokens_seen": 1866661888
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003510852557673019,
+ "loss": 1.2007,
+ "theoretical_loss": 3.4787724715226664,
+ "tokens_seen": 1866727424
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035106920762286863,
+ "loss": 1.1515,
+ "theoretical_loss": 3.4787622346018257,
+ "tokens_seen": 1866792960
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035105315947843533,
+ "loss": 1.2986,
+ "theoretical_loss": 3.47875199814098,
+ "tokens_seen": 1866858496
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035103711133400203,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4787417621400927,
+ "tokens_seen": 1866924032
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035102106318956873,
+ "loss": 1.3908,
+ "theoretical_loss": 3.4787315265991268,
+ "tokens_seen": 1866989568
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035100501504513543,
+ "loss": 1.2961,
+ "theoretical_loss": 3.4787212915180463,
+ "tokens_seen": 1867055104
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035098896690070213,
+ "loss": 1.1641,
+ "theoretical_loss": 3.478711056896813,
+ "tokens_seen": 1867120640
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035097291875626883,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4787008227353917,
+ "tokens_seen": 1867186176
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035095687061183553,
+ "loss": 1.3288,
+ "theoretical_loss": 3.478690589033744,
+ "tokens_seen": 1867251712
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035094082246740223,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4786803557918344,
+ "tokens_seen": 1867317248
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035092477432296893,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4786701230096257,
+ "tokens_seen": 1867382784
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035090872617853563,
+ "loss": 1.2213,
+ "theoretical_loss": 3.478659890687081,
+ "tokens_seen": 1867448320
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035089267803410233,
+ "loss": 1.3573,
+ "theoretical_loss": 3.478649658824163,
+ "tokens_seen": 1867513856
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035087662988966904,
+ "loss": 1.2319,
+ "theoretical_loss": 3.478639427420836,
+ "tokens_seen": 1867579392
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035086058174523574,
+ "loss": 1.2222,
+ "theoretical_loss": 3.478629196477063,
+ "tokens_seen": 1867644928
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035084453360080244,
+ "loss": 1.3038,
+ "theoretical_loss": 3.4786189659928057,
+ "tokens_seen": 1867710464
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 626305,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3878669738769531,
+ "objective/train/theoretical_loss": 3.47860873596803,
+ "objective/train/tokens_used": 1888236000,
+ "theoretical_loss": 3.47860873596803,
+ "tokens_seen": 1867776000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035082848545636914,
+ "loss": 1.3074,
+ "theoretical_loss": 3.47860873596803,
+ "tokens_seen": 1867776000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035081243731193584,
+ "loss": 1.2039,
+ "theoretical_loss": 3.478598506402697,
+ "tokens_seen": 1867841536
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035079638916750254,
+ "loss": 1.3566,
+ "theoretical_loss": 3.4785882772967707,
+ "tokens_seen": 1867907072
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035078034102306924,
+ "loss": 1.3228,
+ "theoretical_loss": 3.4785780486502142,
+ "tokens_seen": 1867972608
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035076429287863594,
+ "loss": 1.2506,
+ "theoretical_loss": 3.4785678204629917,
+ "tokens_seen": 1868038144
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035074824473420264,
+ "loss": 1.3418,
+ "theoretical_loss": 3.478557592735065,
+ "tokens_seen": 1868103680
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035073219658976934,
+ "loss": 1.2865,
+ "theoretical_loss": 3.478547365466398,
+ "tokens_seen": 1868169216
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035071614844533604,
+ "loss": 1.239,
+ "theoretical_loss": 3.478537138656954,
+ "tokens_seen": 1868234752
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035070010030090274,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4785269123066964,
+ "tokens_seen": 1868300288
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035068405215646944,
+ "loss": 1.1591,
+ "theoretical_loss": 3.4785166864155883,
+ "tokens_seen": 1868365824
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035066800401203614,
+ "loss": 1.2985,
+ "theoretical_loss": 3.478506460983593,
+ "tokens_seen": 1868431360
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035065195586760284,
+ "loss": 1.2762,
+ "theoretical_loss": 3.478496236010673,
+ "tokens_seen": 1868496896
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035063590772316954,
+ "loss": 1.1663,
+ "theoretical_loss": 3.4784860114967935,
+ "tokens_seen": 1868562432
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035061985957873625,
+ "loss": 1.2888,
+ "theoretical_loss": 3.478475787441916,
+ "tokens_seen": 1868627968
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035060381143430295,
+ "loss": 1.2325,
+ "theoretical_loss": 3.478465563846005,
+ "tokens_seen": 1868693504
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035058776328986965,
+ "loss": 1.3075,
+ "theoretical_loss": 3.478455340709023,
+ "tokens_seen": 1868759040
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003505717151454363,
+ "loss": 1.2535,
+ "theoretical_loss": 3.4784451180309333,
+ "tokens_seen": 1868824576
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000350555667001003,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4784348958116995,
+ "tokens_seen": 1868890112
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003505396188565697,
+ "loss": 1.2416,
+ "theoretical_loss": 3.4784246740512854,
+ "tokens_seen": 1868955648
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003505235707121364,
+ "loss": 1.2366,
+ "theoretical_loss": 3.4784144527496537,
+ "tokens_seen": 1869021184
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003505075225677031,
+ "loss": 1.3414,
+ "theoretical_loss": 3.4784042319067674,
+ "tokens_seen": 1869086720
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504914744232698,
+ "loss": 1.1661,
+ "theoretical_loss": 3.4783940115225906,
+ "tokens_seen": 1869152256
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504754262788365,
+ "loss": 1.1315,
+ "theoretical_loss": 3.478383791597086,
+ "tokens_seen": 1869217792
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504593781344032,
+ "loss": 1.2743,
+ "theoretical_loss": 3.4783735721302174,
+ "tokens_seen": 1869283328
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504433299899699,
+ "loss": 1.2474,
+ "theoretical_loss": 3.4783633531219484,
+ "tokens_seen": 1869348864
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504272818455366,
+ "loss": 1.2987,
+ "theoretical_loss": 3.478353134572241,
+ "tokens_seen": 1869414400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003504112337011033,
+ "loss": 1.3324,
+ "theoretical_loss": 3.4783429164810604,
+ "tokens_seen": 1869479936
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035039518555667,
+ "loss": 1.2863,
+ "theoretical_loss": 3.4783326988483685,
+ "tokens_seen": 1869545472
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003503791374122367,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4783224816741294,
+ "tokens_seen": 1869611008
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003503630892678034,
+ "loss": 1.2684,
+ "theoretical_loss": 3.478312264958306,
+ "tokens_seen": 1869676544
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003503470411233701,
+ "loss": 1.3193,
+ "theoretical_loss": 3.478302048700862,
+ "tokens_seen": 1869742080
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003503309929789368,
+ "loss": 1.3287,
+ "theoretical_loss": 3.4782918329017605,
+ "tokens_seen": 1869807616
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003503149448345035,
+ "loss": 1.3643,
+ "theoretical_loss": 3.478281617560965,
+ "tokens_seen": 1869873152
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502988966900702,
+ "loss": 1.3551,
+ "theoretical_loss": 3.4782714026784394,
+ "tokens_seen": 1869938688
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502828485456369,
+ "loss": 1.2436,
+ "theoretical_loss": 3.4782611882541463,
+ "tokens_seen": 1870004224
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502668004012036,
+ "loss": 1.3334,
+ "theoretical_loss": 3.4782509742880494,
+ "tokens_seen": 1870069760
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502507522567703,
+ "loss": 1.3374,
+ "theoretical_loss": 3.478240760780112,
+ "tokens_seen": 1870135296
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000350234704112337,
+ "loss": 1.2015,
+ "theoretical_loss": 3.4782305477302975,
+ "tokens_seen": 1870200832
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502186559679037,
+ "loss": 1.2048,
+ "theoretical_loss": 3.4782203351385697,
+ "tokens_seen": 1870266368
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003502026078234704,
+ "loss": 1.2261,
+ "theoretical_loss": 3.4782101230048914,
+ "tokens_seen": 1870331904
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003501865596790371,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4781999113292263,
+ "tokens_seen": 1870397440
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003501705115346038,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4781897001115376,
+ "tokens_seen": 1870462976
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003501544633901705,
+ "loss": 1.3354,
+ "theoretical_loss": 3.478179489351789,
+ "tokens_seen": 1870528512
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035013841524573726,
+ "loss": 1.2014,
+ "theoretical_loss": 3.4781692790499443,
+ "tokens_seen": 1870594048
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035012236710130397,
+ "loss": 1.3876,
+ "theoretical_loss": 3.478159069205966,
+ "tokens_seen": 1870659584
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035010631895687067,
+ "loss": 1.2212,
+ "theoretical_loss": 3.478148859819818,
+ "tokens_seen": 1870725120
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035009027081243737,
+ "loss": 1.2918,
+ "theoretical_loss": 3.478138650891464,
+ "tokens_seen": 1870790656
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035007422266800407,
+ "loss": 1.3017,
+ "theoretical_loss": 3.4781284424208665,
+ "tokens_seen": 1870856192
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035005817452357077,
+ "loss": 1.1694,
+ "theoretical_loss": 3.47811823440799,
+ "tokens_seen": 1870921728
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035004212637913747,
+ "loss": 1.23,
+ "theoretical_loss": 3.478108026852797,
+ "tokens_seen": 1870987264
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 627508,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2596744298934937,
+ "objective/train/theoretical_loss": 3.4780978197552526,
+ "objective/train/tokens_used": 1891512800,
+ "theoretical_loss": 3.4780978197552526,
+ "tokens_seen": 1871052800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035002607823470417,
+ "loss": 1.2767,
+ "theoretical_loss": 3.4780978197552526,
+ "tokens_seen": 1871052800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00035001003009027087,
+ "loss": 1.1887,
+ "theoretical_loss": 3.4780876131153184,
+ "tokens_seen": 1871118336
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034999398194583757,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4780774069329583,
+ "tokens_seen": 1871183872
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034997793380140427,
+ "loss": 1.2167,
+ "theoretical_loss": 3.4780672012081366,
+ "tokens_seen": 1871249408
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034996188565697097,
+ "loss": 1.307,
+ "theoretical_loss": 3.478056995940816,
+ "tokens_seen": 1871314944
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003499458375125376,
+ "loss": 1.2567,
+ "theoretical_loss": 3.4780467911309603,
+ "tokens_seen": 1871380480
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003499297893681043,
+ "loss": 1.2335,
+ "theoretical_loss": 3.478036586778533,
+ "tokens_seen": 1871446016
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000349913741223671,
+ "loss": 1.1905,
+ "theoretical_loss": 3.478026382883497,
+ "tokens_seen": 1871511552
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498976930792377,
+ "loss": 1.3345,
+ "theoretical_loss": 3.4780161794458166,
+ "tokens_seen": 1871577088
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498816449348044,
+ "loss": 1.2942,
+ "theoretical_loss": 3.4780059764654547,
+ "tokens_seen": 1871642624
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498655967903711,
+ "loss": 1.2912,
+ "theoretical_loss": 3.477995773942375,
+ "tokens_seen": 1871708160
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498495486459378,
+ "loss": 1.2644,
+ "theoretical_loss": 3.477985571876541,
+ "tokens_seen": 1871773696
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498335005015045,
+ "loss": 1.3195,
+ "theoretical_loss": 3.477975370267916,
+ "tokens_seen": 1871839232
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498174523570712,
+ "loss": 1.3754,
+ "theoretical_loss": 3.477965169116464,
+ "tokens_seen": 1871904768
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003498014042126379,
+ "loss": 1.4017,
+ "theoretical_loss": 3.4779549684221482,
+ "tokens_seen": 1871970304
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003497853560682046,
+ "loss": 1.223,
+ "theoretical_loss": 3.477944768184932,
+ "tokens_seen": 1872035840
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003497693079237713,
+ "loss": 1.3074,
+ "theoretical_loss": 3.477934568404779,
+ "tokens_seen": 1872101376
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000349753259779338,
+ "loss": 1.2329,
+ "theoretical_loss": 3.477924369081653,
+ "tokens_seen": 1872166912
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003497372116349047,
+ "loss": 1.335,
+ "theoretical_loss": 3.477914170215517,
+ "tokens_seen": 1872232448
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003497211634904714,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4779039718063345,
+ "tokens_seen": 1872297984
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034970511534603813,
+ "loss": 1.3578,
+ "theoretical_loss": 3.4778937738540696,
+ "tokens_seen": 1872363520
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034968906720160483,
+ "loss": 1.2558,
+ "theoretical_loss": 3.477883576358686,
+ "tokens_seen": 1872429056
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034967301905717153,
+ "loss": 1.2519,
+ "theoretical_loss": 3.4778733793201466,
+ "tokens_seen": 1872494592
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034965697091273823,
+ "loss": 1.2095,
+ "theoretical_loss": 3.4778631827384148,
+ "tokens_seen": 1872560128
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034964092276830493,
+ "loss": 1.341,
+ "theoretical_loss": 3.477852986613455,
+ "tokens_seen": 1872625664
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034962487462387163,
+ "loss": 1.3427,
+ "theoretical_loss": 3.47784279094523,
+ "tokens_seen": 1872691200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034960882647943833,
+ "loss": 1.3541,
+ "theoretical_loss": 3.4778325957337035,
+ "tokens_seen": 1872756736
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034959277833500503,
+ "loss": 1.2472,
+ "theoretical_loss": 3.477822400978839,
+ "tokens_seen": 1872822272
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034957673019057173,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4778122066806008,
+ "tokens_seen": 1872887808
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034956068204613843,
+ "loss": 1.1452,
+ "theoretical_loss": 3.4778020128389517,
+ "tokens_seen": 1872953344
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034954463390170513,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4777918194538557,
+ "tokens_seen": 1873018880
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034952858575727183,
+ "loss": 1.1957,
+ "theoretical_loss": 3.477781626525276,
+ "tokens_seen": 1873084416
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034951253761283853,
+ "loss": 1.2944,
+ "theoretical_loss": 3.4777714340531762,
+ "tokens_seen": 1873149952
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034949648946840524,
+ "loss": 1.2395,
+ "theoretical_loss": 3.47776124203752,
+ "tokens_seen": 1873215488
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034948044132397194,
+ "loss": 1.2769,
+ "theoretical_loss": 3.477751050478271,
+ "tokens_seen": 1873281024
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034946439317953864,
+ "loss": 1.2623,
+ "theoretical_loss": 3.477740859375393,
+ "tokens_seen": 1873346560
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034944834503510534,
+ "loss": 1.3131,
+ "theoretical_loss": 3.4777306687288494,
+ "tokens_seen": 1873412096
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034943229689067204,
+ "loss": 1.2359,
+ "theoretical_loss": 3.477720478538604,
+ "tokens_seen": 1873477632
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034941624874623874,
+ "loss": 1.2749,
+ "theoretical_loss": 3.4777102888046203,
+ "tokens_seen": 1873543168
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034940020060180544,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4777000995268614,
+ "tokens_seen": 1873608704
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034938415245737214,
+ "loss": 1.296,
+ "theoretical_loss": 3.477689910705292,
+ "tokens_seen": 1873674240
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034936810431293884,
+ "loss": 1.2472,
+ "theoretical_loss": 3.4776797223398743,
+ "tokens_seen": 1873739776
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034935205616850554,
+ "loss": 1.3447,
+ "theoretical_loss": 3.4776695344305733,
+ "tokens_seen": 1873805312
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034933600802407224,
+ "loss": 1.3039,
+ "theoretical_loss": 3.477659346977352,
+ "tokens_seen": 1873870848
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034931995987963894,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4776491599801735,
+ "tokens_seen": 1873936384
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034930391173520564,
+ "loss": 1.2527,
+ "theoretical_loss": 3.477638973439003,
+ "tokens_seen": 1874001920
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034928786359077234,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4776287873538023,
+ "tokens_seen": 1874067456
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034927181544633904,
+ "loss": 1.3203,
+ "theoretical_loss": 3.4776186017245365,
+ "tokens_seen": 1874132992
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034925576730190575,
+ "loss": 1.1955,
+ "theoretical_loss": 3.4776084165511683,
+ "tokens_seen": 1874198528
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034923971915747245,
+ "loss": 1.2311,
+ "theoretical_loss": 3.4775982318336616,
+ "tokens_seen": 1874264064
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 628392,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2881674766540527,
+ "objective/train/theoretical_loss": 3.47758804757198,
+ "objective/train/tokens_used": 1894789600,
+ "theoretical_loss": 3.47758804757198,
+ "tokens_seen": 1874329600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034922367101303915,
+ "loss": 1.2322,
+ "theoretical_loss": 3.47758804757198,
+ "tokens_seen": 1874329600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034920762286860585,
+ "loss": 1.2592,
+ "theoretical_loss": 3.477577863766088,
+ "tokens_seen": 1874395136
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034919157472417255,
+ "loss": 1.3634,
+ "theoretical_loss": 3.477567680415948,
+ "tokens_seen": 1874460672
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003491755265797392,
+ "loss": 1.2855,
+ "theoretical_loss": 3.477557497521525,
+ "tokens_seen": 1874526208
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003491594784353059,
+ "loss": 1.2787,
+ "theoretical_loss": 3.477547315082781,
+ "tokens_seen": 1874591744
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003491434302908726,
+ "loss": 1.2652,
+ "theoretical_loss": 3.4775371330996814,
+ "tokens_seen": 1874657280
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003491273821464393,
+ "loss": 1.3414,
+ "theoretical_loss": 3.4775269515721887,
+ "tokens_seen": 1874722816
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000349111334002006,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4775167705002668,
+ "tokens_seen": 1874788352
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490952858575727,
+ "loss": 1.3128,
+ "theoretical_loss": 3.47750658988388,
+ "tokens_seen": 1874853888
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490792377131394,
+ "loss": 1.2863,
+ "theoretical_loss": 3.4774964097229915,
+ "tokens_seen": 1874919424
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490631895687061,
+ "loss": 1.3146,
+ "theoretical_loss": 3.477486230017565,
+ "tokens_seen": 1874984960
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490471414242728,
+ "loss": 1.263,
+ "theoretical_loss": 3.477476050767564,
+ "tokens_seen": 1875050496
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490310932798395,
+ "loss": 1.3015,
+ "theoretical_loss": 3.477465871972953,
+ "tokens_seen": 1875116032
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003490150451354062,
+ "loss": 1.2515,
+ "theoretical_loss": 3.477455693633695,
+ "tokens_seen": 1875181568
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003489989969909729,
+ "loss": 1.3155,
+ "theoretical_loss": 3.4774455157497535,
+ "tokens_seen": 1875247104
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003489829488465396,
+ "loss": 1.2219,
+ "theoretical_loss": 3.477435338321093,
+ "tokens_seen": 1875312640
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003489669007021063,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4774251613476768,
+ "tokens_seen": 1875378176
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000348950852557673,
+ "loss": 1.316,
+ "theoretical_loss": 3.477414984829469,
+ "tokens_seen": 1875443712
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034893480441323976,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4774048087664324,
+ "tokens_seen": 1875509248
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034891875626880646,
+ "loss": 1.3149,
+ "theoretical_loss": 3.477394633158532,
+ "tokens_seen": 1875574784
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034890270812437316,
+ "loss": 1.2952,
+ "theoretical_loss": 3.4773844580057305,
+ "tokens_seen": 1875640320
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034888665997993986,
+ "loss": 1.2977,
+ "theoretical_loss": 3.4773742833079924,
+ "tokens_seen": 1875705856
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034887061183550656,
+ "loss": 1.2053,
+ "theoretical_loss": 3.477364109065281,
+ "tokens_seen": 1875771392
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034885456369107326,
+ "loss": 1.3631,
+ "theoretical_loss": 3.4773539352775593,
+ "tokens_seen": 1875836928
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034883851554663996,
+ "loss": 1.3677,
+ "theoretical_loss": 3.477343761944793,
+ "tokens_seen": 1875902464
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034882246740220666,
+ "loss": 1.3194,
+ "theoretical_loss": 3.4773335890669443,
+ "tokens_seen": 1875968000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034880641925777336,
+ "loss": 1.2171,
+ "theoretical_loss": 3.4773234166439773,
+ "tokens_seen": 1876033536
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034879037111334006,
+ "loss": 1.2656,
+ "theoretical_loss": 3.477313244675856,
+ "tokens_seen": 1876099072
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034877432296890676,
+ "loss": 1.2395,
+ "theoretical_loss": 3.4773030731625445,
+ "tokens_seen": 1876164608
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034875827482447346,
+ "loss": 1.3173,
+ "theoretical_loss": 3.477292902104006,
+ "tokens_seen": 1876230144
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034874222668004017,
+ "loss": 1.3607,
+ "theoretical_loss": 3.477282731500204,
+ "tokens_seen": 1876295680
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034872617853560687,
+ "loss": 1.2865,
+ "theoretical_loss": 3.477272561351103,
+ "tokens_seen": 1876361216
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034871013039117357,
+ "loss": 1.3007,
+ "theoretical_loss": 3.4772623916566667,
+ "tokens_seen": 1876426752
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034869408224674027,
+ "loss": 1.2685,
+ "theoretical_loss": 3.4772522224168583,
+ "tokens_seen": 1876492288
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034867803410230697,
+ "loss": 1.333,
+ "theoretical_loss": 3.4772420536316426,
+ "tokens_seen": 1876557824
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034866198595787367,
+ "loss": 1.3772,
+ "theoretical_loss": 3.4772318853009825,
+ "tokens_seen": 1876623360
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034864593781344037,
+ "loss": 1.1597,
+ "theoretical_loss": 3.477221717424842,
+ "tokens_seen": 1876688896
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034862988966900707,
+ "loss": 1.2317,
+ "theoretical_loss": 3.477211550003185,
+ "tokens_seen": 1876754432
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034861384152457377,
+ "loss": 1.2302,
+ "theoretical_loss": 3.4772013830359754,
+ "tokens_seen": 1876819968
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034859779338014047,
+ "loss": 1.2797,
+ "theoretical_loss": 3.4771912165231775,
+ "tokens_seen": 1876885504
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034858174523570717,
+ "loss": 1.2687,
+ "theoretical_loss": 3.477181050464754,
+ "tokens_seen": 1876951040
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034856569709127387,
+ "loss": 1.3463,
+ "theoretical_loss": 3.4771708848606693,
+ "tokens_seen": 1877016576
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003485496489468405,
+ "loss": 1.308,
+ "theoretical_loss": 3.477160719710888,
+ "tokens_seen": 1877082112
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003485336008024072,
+ "loss": 1.3132,
+ "theoretical_loss": 3.4771505550153723,
+ "tokens_seen": 1877147648
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003485175526579739,
+ "loss": 1.2976,
+ "theoretical_loss": 3.477140390774087,
+ "tokens_seen": 1877213184
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003485015045135406,
+ "loss": 1.3134,
+ "theoretical_loss": 3.477130226986996,
+ "tokens_seen": 1877278720
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003484854563691073,
+ "loss": 1.257,
+ "theoretical_loss": 3.4771200636540636,
+ "tokens_seen": 1877344256
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000348469408224674,
+ "loss": 1.2723,
+ "theoretical_loss": 3.4771099007752526,
+ "tokens_seen": 1877409792
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003484533600802407,
+ "loss": 1.3667,
+ "theoretical_loss": 3.4770997383505273,
+ "tokens_seen": 1877475328
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003484373119358074,
+ "loss": 1.2546,
+ "theoretical_loss": 3.4770895763798517,
+ "tokens_seen": 1877540864
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 629623,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.36664879322052,
+ "objective/train/theoretical_loss": 3.47707941486319,
+ "objective/train/tokens_used": 1898066400,
+ "theoretical_loss": 3.47707941486319,
+ "tokens_seen": 1877606400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003484212637913741,
+ "loss": 1.3118,
+ "theoretical_loss": 3.47707941486319,
+ "tokens_seen": 1877606400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003484052156469408,
+ "loss": 1.2411,
+ "theoretical_loss": 3.4770692538005052,
+ "tokens_seen": 1877671936
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003483891675025075,
+ "loss": 1.2114,
+ "theoretical_loss": 3.4770590931917615,
+ "tokens_seen": 1877737472
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003483731193580742,
+ "loss": 1.2308,
+ "theoretical_loss": 3.477048933036923,
+ "tokens_seen": 1877803008
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003483570712136409,
+ "loss": 1.3513,
+ "theoretical_loss": 3.477038773335954,
+ "tokens_seen": 1877868544
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034834102306920763,
+ "loss": 1.2517,
+ "theoretical_loss": 3.477028614088817,
+ "tokens_seen": 1877934080
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034832497492477433,
+ "loss": 1.3276,
+ "theoretical_loss": 3.477018455295477,
+ "tokens_seen": 1877999616
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034830892678034103,
+ "loss": 1.2847,
+ "theoretical_loss": 3.4770082969558977,
+ "tokens_seen": 1878065152
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034829287863590773,
+ "loss": 1.304,
+ "theoretical_loss": 3.4769981390700435,
+ "tokens_seen": 1878130688
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034827683049147443,
+ "loss": 1.3372,
+ "theoretical_loss": 3.476987981637877,
+ "tokens_seen": 1878196224
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034826078234704113,
+ "loss": 1.322,
+ "theoretical_loss": 3.4769778246593632,
+ "tokens_seen": 1878261760
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034824473420260783,
+ "loss": 1.1953,
+ "theoretical_loss": 3.4769676681344652,
+ "tokens_seen": 1878327296
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034822868605817453,
+ "loss": 1.2815,
+ "theoretical_loss": 3.476957512063148,
+ "tokens_seen": 1878392832
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034821263791374123,
+ "loss": 1.2241,
+ "theoretical_loss": 3.476947356445375,
+ "tokens_seen": 1878458368
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034819658976930793,
+ "loss": 1.1834,
+ "theoretical_loss": 3.4769372012811095,
+ "tokens_seen": 1878523904
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034818054162487463,
+ "loss": 1.3603,
+ "theoretical_loss": 3.4769270465703164,
+ "tokens_seen": 1878589440
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034816449348044133,
+ "loss": 1.3119,
+ "theoretical_loss": 3.4769168923129588,
+ "tokens_seen": 1878654976
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034814844533600803,
+ "loss": 1.2568,
+ "theoretical_loss": 3.476906738509001,
+ "tokens_seen": 1878720512
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034813239719157474,
+ "loss": 1.3474,
+ "theoretical_loss": 3.476896585158407,
+ "tokens_seen": 1878786048
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034811634904714144,
+ "loss": 1.2602,
+ "theoretical_loss": 3.476886432261141,
+ "tokens_seen": 1878851584
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034810030090270814,
+ "loss": 1.264,
+ "theoretical_loss": 3.4768762798171666,
+ "tokens_seen": 1878917120
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034808425275827484,
+ "loss": 1.2258,
+ "theoretical_loss": 3.4768661278264474,
+ "tokens_seen": 1878982656
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034806820461384154,
+ "loss": 1.239,
+ "theoretical_loss": 3.4768559762889484,
+ "tokens_seen": 1879048192
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034805215646940824,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4768458252046326,
+ "tokens_seen": 1879113728
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034803610832497494,
+ "loss": 1.3197,
+ "theoretical_loss": 3.476835674573464,
+ "tokens_seen": 1879179264
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034802006018054164,
+ "loss": 1.3999,
+ "theoretical_loss": 3.476825524395407,
+ "tokens_seen": 1879244800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034800401203610834,
+ "loss": 1.2971,
+ "theoretical_loss": 3.4768153746704256,
+ "tokens_seen": 1879310336
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034798796389167504,
+ "loss": 1.3798,
+ "theoretical_loss": 3.4768052253984836,
+ "tokens_seen": 1879375872
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034797191574724174,
+ "loss": 1.2581,
+ "theoretical_loss": 3.4767950765795455,
+ "tokens_seen": 1879441408
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034795586760280844,
+ "loss": 1.2281,
+ "theoretical_loss": 3.476784928213574,
+ "tokens_seen": 1879506944
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034793981945837514,
+ "loss": 1.2776,
+ "theoretical_loss": 3.4767747803005338,
+ "tokens_seen": 1879572480
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034792377131394184,
+ "loss": 1.3545,
+ "theoretical_loss": 3.476764632840389,
+ "tokens_seen": 1879638016
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034790772316950854,
+ "loss": 1.2976,
+ "theoretical_loss": 3.476754485833104,
+ "tokens_seen": 1879703552
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034789167502507524,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4767443392786426,
+ "tokens_seen": 1879769088
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034787562688064195,
+ "loss": 1.3606,
+ "theoretical_loss": 3.4767341931769677,
+ "tokens_seen": 1879834624
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034785957873620865,
+ "loss": 1.2684,
+ "theoretical_loss": 3.4767240475280445,
+ "tokens_seen": 1879900160
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034784353059177535,
+ "loss": 1.2395,
+ "theoretical_loss": 3.476713902331837,
+ "tokens_seen": 1879965696
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034782748244734205,
+ "loss": 1.2505,
+ "theoretical_loss": 3.476703757588308,
+ "tokens_seen": 1880031232
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034781143430290875,
+ "loss": 1.308,
+ "theoretical_loss": 3.4766936132974235,
+ "tokens_seen": 1880096768
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034779538615847545,
+ "loss": 1.2212,
+ "theoretical_loss": 3.476683469459146,
+ "tokens_seen": 1880162304
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003477793380140421,
+ "loss": 1.2989,
+ "theoretical_loss": 3.47667332607344,
+ "tokens_seen": 1880227840
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003477632898696088,
+ "loss": 1.2363,
+ "theoretical_loss": 3.47666318314027,
+ "tokens_seen": 1880293376
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034774724172517555,
+ "loss": 1.3316,
+ "theoretical_loss": 3.476653040659599,
+ "tokens_seen": 1880358912
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034773119358074225,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4766428986313915,
+ "tokens_seen": 1880424448
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034771514543630895,
+ "loss": 1.4198,
+ "theoretical_loss": 3.476632757055612,
+ "tokens_seen": 1880489984
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034769909729187565,
+ "loss": 1.3403,
+ "theoretical_loss": 3.476622615932224,
+ "tokens_seen": 1880555520
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034768304914744235,
+ "loss": 1.3469,
+ "theoretical_loss": 3.476612475261192,
+ "tokens_seen": 1880621056
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034766700100300905,
+ "loss": 1.2963,
+ "theoretical_loss": 3.4766023350424797,
+ "tokens_seen": 1880686592
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034765095285857575,
+ "loss": 1.254,
+ "theoretical_loss": 3.476592195276051,
+ "tokens_seen": 1880752128
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034763490471414246,
+ "loss": 1.2565,
+ "theoretical_loss": 3.4765820559618703,
+ "tokens_seen": 1880817664
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 630537,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3395367860794067,
+ "objective/train/theoretical_loss": 3.476571917099902,
+ "objective/train/tokens_used": 1901343200,
+ "theoretical_loss": 3.476571917099902,
+ "tokens_seen": 1880883200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034761885656970916,
+ "loss": 1.2829,
+ "theoretical_loss": 3.476571917099902,
+ "tokens_seen": 1880883200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034760280842527586,
+ "loss": 1.2508,
+ "theoretical_loss": 3.47656177869011,
+ "tokens_seen": 1880948736
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034758676028084256,
+ "loss": 1.3241,
+ "theoretical_loss": 3.4765516407324575,
+ "tokens_seen": 1881014272
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034757071213640926,
+ "loss": 1.2649,
+ "theoretical_loss": 3.476541503226909,
+ "tokens_seen": 1881079808
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034755466399197596,
+ "loss": 1.2795,
+ "theoretical_loss": 3.47653136617343,
+ "tokens_seen": 1881145344
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034753861584754266,
+ "loss": 1.3853,
+ "theoretical_loss": 3.4765212295719827,
+ "tokens_seen": 1881210880
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034752256770310936,
+ "loss": 1.2557,
+ "theoretical_loss": 3.4765110934225323,
+ "tokens_seen": 1881276416
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034750651955867606,
+ "loss": 1.2602,
+ "theoretical_loss": 3.476500957725042,
+ "tokens_seen": 1881341952
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034749047141424276,
+ "loss": 1.2826,
+ "theoretical_loss": 3.4764908224794766,
+ "tokens_seen": 1881407488
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034747442326980946,
+ "loss": 1.2976,
+ "theoretical_loss": 3.4764806876858003,
+ "tokens_seen": 1881473024
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034745837512537616,
+ "loss": 1.2569,
+ "theoretical_loss": 3.476470553343977,
+ "tokens_seen": 1881538560
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034744232698094286,
+ "loss": 1.3674,
+ "theoretical_loss": 3.4764604194539706,
+ "tokens_seen": 1881604096
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034742627883650956,
+ "loss": 1.2636,
+ "theoretical_loss": 3.476450286015745,
+ "tokens_seen": 1881669632
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034741023069207626,
+ "loss": 1.3229,
+ "theoretical_loss": 3.4764401530292655,
+ "tokens_seen": 1881735168
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034739418254764296,
+ "loss": 1.1782,
+ "theoretical_loss": 3.476430020494495,
+ "tokens_seen": 1881800704
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034737813440320967,
+ "loss": 1.237,
+ "theoretical_loss": 3.476419888411398,
+ "tokens_seen": 1881866240
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034736208625877637,
+ "loss": 1.2918,
+ "theoretical_loss": 3.4764097567799395,
+ "tokens_seen": 1881931776
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034734603811434307,
+ "loss": 1.244,
+ "theoretical_loss": 3.4763996256000818,
+ "tokens_seen": 1881997312
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034732998996990977,
+ "loss": 1.305,
+ "theoretical_loss": 3.476389494871791,
+ "tokens_seen": 1882062848
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034731394182547647,
+ "loss": 1.1724,
+ "theoretical_loss": 3.47637936459503,
+ "tokens_seen": 1882128384
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034729789368104317,
+ "loss": 1.1619,
+ "theoretical_loss": 3.476369234769763,
+ "tokens_seen": 1882193920
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034728184553660987,
+ "loss": 1.2412,
+ "theoretical_loss": 3.4763591053959546,
+ "tokens_seen": 1882259456
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034726579739217657,
+ "loss": 1.2919,
+ "theoretical_loss": 3.4763489764735693,
+ "tokens_seen": 1882324992
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034724974924774327,
+ "loss": 1.1972,
+ "theoretical_loss": 3.4763388480025705,
+ "tokens_seen": 1882390528
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034723370110330997,
+ "loss": 1.2253,
+ "theoretical_loss": 3.4763287199829227,
+ "tokens_seen": 1882456064
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034721765295887667,
+ "loss": 1.3869,
+ "theoretical_loss": 3.47631859241459,
+ "tokens_seen": 1882521600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034720160481444337,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4763084652975365,
+ "tokens_seen": 1882587136
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034718555667001007,
+ "loss": 1.3585,
+ "theoretical_loss": 3.4762983386317265,
+ "tokens_seen": 1882652672
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003471695085255768,
+ "loss": 1.3123,
+ "theoretical_loss": 3.476288212417124,
+ "tokens_seen": 1882718208
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003471534603811435,
+ "loss": 1.3165,
+ "theoretical_loss": 3.476278086653694,
+ "tokens_seen": 1882783744
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003471374122367101,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4762679613414003,
+ "tokens_seen": 1882849280
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003471213640922768,
+ "loss": 1.2957,
+ "theoretical_loss": 3.4762578364802064,
+ "tokens_seen": 1882914816
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003471053159478435,
+ "loss": 1.3283,
+ "theoretical_loss": 3.4762477120700765,
+ "tokens_seen": 1882980352
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003470892678034102,
+ "loss": 1.4514,
+ "theoretical_loss": 3.476237588110976,
+ "tokens_seen": 1883045888
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003470732196589769,
+ "loss": 1.3323,
+ "theoretical_loss": 3.476227464602868,
+ "tokens_seen": 1883111424
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003470571715145436,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4762173415457176,
+ "tokens_seen": 1883176960
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003470411233701103,
+ "loss": 1.2769,
+ "theoretical_loss": 3.4762072189394884,
+ "tokens_seen": 1883242496
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000347025075225677,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4761970967841442,
+ "tokens_seen": 1883308032
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003470090270812437,
+ "loss": 1.315,
+ "theoretical_loss": 3.47618697507965,
+ "tokens_seen": 1883373568
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003469929789368104,
+ "loss": 1.176,
+ "theoretical_loss": 3.47617685382597,
+ "tokens_seen": 1883439104
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003469769307923771,
+ "loss": 1.3379,
+ "theoretical_loss": 3.476166733023068,
+ "tokens_seen": 1883504640
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034696088264794383,
+ "loss": 1.3245,
+ "theoretical_loss": 3.476156612670909,
+ "tokens_seen": 1883570176
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034694483450351053,
+ "loss": 1.2619,
+ "theoretical_loss": 3.4761464927694563,
+ "tokens_seen": 1883635712
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034692878635907723,
+ "loss": 1.2807,
+ "theoretical_loss": 3.476136373318675,
+ "tokens_seen": 1883701248
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034691273821464393,
+ "loss": 1.418,
+ "theoretical_loss": 3.4761262543185287,
+ "tokens_seen": 1883766784
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034689669007021063,
+ "loss": 1.2756,
+ "theoretical_loss": 3.476116135768982,
+ "tokens_seen": 1883832320
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034688064192577733,
+ "loss": 1.3639,
+ "theoretical_loss": 3.476106017669999,
+ "tokens_seen": 1883897856
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034686459378134403,
+ "loss": 1.2701,
+ "theoretical_loss": 3.476095900021544,
+ "tokens_seen": 1883963392
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034684854563691073,
+ "loss": 1.214,
+ "theoretical_loss": 3.4760857828235814,
+ "tokens_seen": 1884028928
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034683249749247743,
+ "loss": 1.4156,
+ "theoretical_loss": 3.476075666076075,
+ "tokens_seen": 1884094464
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 631766,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2157416343688965,
+ "objective/train/theoretical_loss": 3.4760655497789896,
+ "objective/train/tokens_used": 1904620000,
+ "theoretical_loss": 3.4760655497789896,
+ "tokens_seen": 1884160000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034681644934804413,
+ "loss": 1.1797,
+ "theoretical_loss": 3.4760655497789896,
+ "tokens_seen": 1884160000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034680040120361083,
+ "loss": 1.2719,
+ "theoretical_loss": 3.4760554339322893,
+ "tokens_seen": 1884225536
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034678435305917753,
+ "loss": 1.2336,
+ "theoretical_loss": 3.476045318535938,
+ "tokens_seen": 1884291072
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034676830491474424,
+ "loss": 1.3025,
+ "theoretical_loss": 3.476035203589901,
+ "tokens_seen": 1884356608
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034675225677031094,
+ "loss": 1.2623,
+ "theoretical_loss": 3.4760250890941418,
+ "tokens_seen": 1884422144
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034673620862587764,
+ "loss": 1.4002,
+ "theoretical_loss": 3.476014975048625,
+ "tokens_seen": 1884487680
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034672016048144434,
+ "loss": 1.3848,
+ "theoretical_loss": 3.4760048614533146,
+ "tokens_seen": 1884553216
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034670411233701104,
+ "loss": 1.3453,
+ "theoretical_loss": 3.4759947483081755,
+ "tokens_seen": 1884618752
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034668806419257774,
+ "loss": 1.2601,
+ "theoretical_loss": 3.475984635613171,
+ "tokens_seen": 1884684288
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034667201604814444,
+ "loss": 1.3033,
+ "theoretical_loss": 3.475974523368266,
+ "tokens_seen": 1884749824
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034665596790371114,
+ "loss": 1.302,
+ "theoretical_loss": 3.4759644115734254,
+ "tokens_seen": 1884815360
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034663991975927784,
+ "loss": 1.3104,
+ "theoretical_loss": 3.4759543002286124,
+ "tokens_seen": 1884880896
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034662387161484454,
+ "loss": 1.2814,
+ "theoretical_loss": 3.4759441893337923,
+ "tokens_seen": 1884946432
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034660782347041124,
+ "loss": 1.3162,
+ "theoretical_loss": 3.475934078888929,
+ "tokens_seen": 1885011968
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034659177532597794,
+ "loss": 1.2729,
+ "theoretical_loss": 3.4759239688939862,
+ "tokens_seen": 1885077504
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034657572718154464,
+ "loss": 1.3725,
+ "theoretical_loss": 3.475913859348929,
+ "tokens_seen": 1885143040
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034655967903711134,
+ "loss": 1.3049,
+ "theoretical_loss": 3.475903750253722,
+ "tokens_seen": 1885208576
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003465436308926781,
+ "loss": 1.3137,
+ "theoretical_loss": 3.4758936416083293,
+ "tokens_seen": 1885274112
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003465275827482448,
+ "loss": 1.2189,
+ "theoretical_loss": 3.4758835334127145,
+ "tokens_seen": 1885339648
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034651153460381145,
+ "loss": 1.3198,
+ "theoretical_loss": 3.475873425666843,
+ "tokens_seen": 1885405184
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034649548645937815,
+ "loss": 1.2949,
+ "theoretical_loss": 3.4758633183706786,
+ "tokens_seen": 1885470720
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034647943831494485,
+ "loss": 1.3653,
+ "theoretical_loss": 3.4758532115241856,
+ "tokens_seen": 1885536256
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034646339017051155,
+ "loss": 1.1303,
+ "theoretical_loss": 3.4758431051273293,
+ "tokens_seen": 1885601792
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034644734202607825,
+ "loss": 1.2017,
+ "theoretical_loss": 3.4758329991800725,
+ "tokens_seen": 1885667328
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034643129388164495,
+ "loss": 1.1794,
+ "theoretical_loss": 3.4758228936823805,
+ "tokens_seen": 1885732864
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034641524573721165,
+ "loss": 1.3534,
+ "theoretical_loss": 3.475812788634218,
+ "tokens_seen": 1885798400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034639919759277835,
+ "loss": 1.3524,
+ "theoretical_loss": 3.475802684035548,
+ "tokens_seen": 1885863936
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034638314944834505,
+ "loss": 1.1679,
+ "theoretical_loss": 3.475792579886337,
+ "tokens_seen": 1885929472
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034636710130391175,
+ "loss": 1.3444,
+ "theoretical_loss": 3.4757824761865477,
+ "tokens_seen": 1885995008
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034635105315947845,
+ "loss": 1.237,
+ "theoretical_loss": 3.475772372936145,
+ "tokens_seen": 1886060544
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034633500501504515,
+ "loss": 1.3152,
+ "theoretical_loss": 3.4757622701350934,
+ "tokens_seen": 1886126080
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034631895687061185,
+ "loss": 1.2485,
+ "theoretical_loss": 3.475752167783357,
+ "tokens_seen": 1886191616
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034630290872617855,
+ "loss": 1.242,
+ "theoretical_loss": 3.4757420658809006,
+ "tokens_seen": 1886257152
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034628686058174525,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4757319644276885,
+ "tokens_seen": 1886322688
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034627081243731195,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4757218634236846,
+ "tokens_seen": 1886388224
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034625476429287866,
+ "loss": 1.1945,
+ "theoretical_loss": 3.4757117628688543,
+ "tokens_seen": 1886453760
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034623871614844536,
+ "loss": 1.2792,
+ "theoretical_loss": 3.4757016627631616,
+ "tokens_seen": 1886519296
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034622266800401206,
+ "loss": 1.2926,
+ "theoretical_loss": 3.4756915631065706,
+ "tokens_seen": 1886584832
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034620661985957876,
+ "loss": 1.1737,
+ "theoretical_loss": 3.4756814638990456,
+ "tokens_seen": 1886650368
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034619057171514546,
+ "loss": 1.2689,
+ "theoretical_loss": 3.4756713651405517,
+ "tokens_seen": 1886715904
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034617452357071216,
+ "loss": 1.3873,
+ "theoretical_loss": 3.475661266831053,
+ "tokens_seen": 1886781440
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034615847542627886,
+ "loss": 1.2714,
+ "theoretical_loss": 3.475651168970514,
+ "tokens_seen": 1886846976
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034614242728184556,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4756410715588992,
+ "tokens_seen": 1886912512
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034612637913741226,
+ "loss": 1.311,
+ "theoretical_loss": 3.475630974596173,
+ "tokens_seen": 1886978048
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034611033099297896,
+ "loss": 1.2887,
+ "theoretical_loss": 3.4756208780822995,
+ "tokens_seen": 1887043584
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034609428284854566,
+ "loss": 1.143,
+ "theoretical_loss": 3.4756107820172435,
+ "tokens_seen": 1887109120
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034607823470411236,
+ "loss": 1.3259,
+ "theoretical_loss": 3.47560068640097,
+ "tokens_seen": 1887174656
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034606218655967906,
+ "loss": 1.2593,
+ "theoretical_loss": 3.475590591233442,
+ "tokens_seen": 1887240192
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034604613841524576,
+ "loss": 1.3756,
+ "theoretical_loss": 3.4755804965146258,
+ "tokens_seen": 1887305728
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034603009027081246,
+ "loss": 1.1932,
+ "theoretical_loss": 3.475570402244484,
+ "tokens_seen": 1887371264
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 632651,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3401379585266113,
+ "objective/train/theoretical_loss": 3.475560308422983,
+ "objective/train/tokens_used": 1907896800,
+ "theoretical_loss": 3.475560308422983,
+ "tokens_seen": 1887436800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034601404212637916,
+ "loss": 1.3055,
+ "theoretical_loss": 3.475560308422983,
+ "tokens_seen": 1887436800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034599799398194587,
+ "loss": 1.2112,
+ "theoretical_loss": 3.4755502150500863,
+ "tokens_seen": 1887502336
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034598194583751257,
+ "loss": 1.2654,
+ "theoretical_loss": 3.475540122125758,
+ "tokens_seen": 1887567872
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034596589769307927,
+ "loss": 1.2081,
+ "theoretical_loss": 3.4755300296499625,
+ "tokens_seen": 1887633408
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034594984954864597,
+ "loss": 1.3585,
+ "theoretical_loss": 3.4755199376226655,
+ "tokens_seen": 1887698944
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034593380140421267,
+ "loss": 1.2937,
+ "theoretical_loss": 3.4755098460438307,
+ "tokens_seen": 1887764480
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034591775325977937,
+ "loss": 1.209,
+ "theoretical_loss": 3.4754997549134226,
+ "tokens_seen": 1887830016
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034590170511534607,
+ "loss": 1.2338,
+ "theoretical_loss": 3.475489664231406,
+ "tokens_seen": 1887895552
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034588565697091277,
+ "loss": 1.2702,
+ "theoretical_loss": 3.475479573997745,
+ "tokens_seen": 1887961088
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034586960882647947,
+ "loss": 1.3039,
+ "theoretical_loss": 3.475469484212404,
+ "tokens_seen": 1888026624
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034585356068204617,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4754593948753483,
+ "tokens_seen": 1888092160
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034583751253761287,
+ "loss": 1.1778,
+ "theoretical_loss": 3.475449305986542,
+ "tokens_seen": 1888157696
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034582146439317957,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4754392175459494,
+ "tokens_seen": 1888223232
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003458054162487463,
+ "loss": 1.206,
+ "theoretical_loss": 3.475429129553535,
+ "tokens_seen": 1888288768
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000345789368104313,
+ "loss": 1.264,
+ "theoretical_loss": 3.475419042009264,
+ "tokens_seen": 1888354304
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003457733199598797,
+ "loss": 1.2604,
+ "theoretical_loss": 3.475408954913101,
+ "tokens_seen": 1888419840
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003457572718154464,
+ "loss": 1.2237,
+ "theoretical_loss": 3.4753988682650094,
+ "tokens_seen": 1888485376
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000345741223671013,
+ "loss": 1.2801,
+ "theoretical_loss": 3.4753887820649547,
+ "tokens_seen": 1888550912
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003457251755265797,
+ "loss": 1.2739,
+ "theoretical_loss": 3.4753786963129008,
+ "tokens_seen": 1888616448
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003457091273821464,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4753686110088124,
+ "tokens_seen": 1888681984
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456930792377131,
+ "loss": 1.2537,
+ "theoretical_loss": 3.4753585261526547,
+ "tokens_seen": 1888747520
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456770310932798,
+ "loss": 1.2924,
+ "theoretical_loss": 3.4753484417443916,
+ "tokens_seen": 1888813056
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456609829488465,
+ "loss": 1.4141,
+ "theoretical_loss": 3.475338357783988,
+ "tokens_seen": 1888878592
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456449348044132,
+ "loss": 1.3914,
+ "theoretical_loss": 3.475328274271409,
+ "tokens_seen": 1888944128
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456288866599799,
+ "loss": 1.3289,
+ "theoretical_loss": 3.475318191206618,
+ "tokens_seen": 1889009664
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003456128385155466,
+ "loss": 1.169,
+ "theoretical_loss": 3.47530810858958,
+ "tokens_seen": 1889075200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034559679037111333,
+ "loss": 1.2749,
+ "theoretical_loss": 3.4752980264202598,
+ "tokens_seen": 1889140736
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034558074222668003,
+ "loss": 1.3492,
+ "theoretical_loss": 3.4752879446986222,
+ "tokens_seen": 1889206272
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034556469408224673,
+ "loss": 1.3428,
+ "theoretical_loss": 3.475277863424631,
+ "tokens_seen": 1889271808
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034554864593781343,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4752677825982516,
+ "tokens_seen": 1889337344
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034553259779338013,
+ "loss": 1.2268,
+ "theoretical_loss": 3.475257702219448,
+ "tokens_seen": 1889402880
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034551654964894683,
+ "loss": 1.266,
+ "theoretical_loss": 3.475247622288186,
+ "tokens_seen": 1889468416
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034550050150451353,
+ "loss": 1.3253,
+ "theoretical_loss": 3.4752375428044284,
+ "tokens_seen": 1889533952
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034548445336008023,
+ "loss": 1.2741,
+ "theoretical_loss": 3.475227463768141,
+ "tokens_seen": 1889599488
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034546840521564693,
+ "loss": 1.2559,
+ "theoretical_loss": 3.4752173851792882,
+ "tokens_seen": 1889665024
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034545235707121363,
+ "loss": 1.1909,
+ "theoretical_loss": 3.4752073070378344,
+ "tokens_seen": 1889730560
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034543630892678033,
+ "loss": 1.3639,
+ "theoretical_loss": 3.4751972293437445,
+ "tokens_seen": 1889796096
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034542026078234703,
+ "loss": 1.2577,
+ "theoretical_loss": 3.475187152096983,
+ "tokens_seen": 1889861632
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034540421263791373,
+ "loss": 1.2106,
+ "theoretical_loss": 3.4751770752975144,
+ "tokens_seen": 1889927168
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034538816449348044,
+ "loss": 1.2934,
+ "theoretical_loss": 3.475166998945304,
+ "tokens_seen": 1889992704
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034537211634904714,
+ "loss": 1.2602,
+ "theoretical_loss": 3.4751569230403154,
+ "tokens_seen": 1890058240
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034535606820461384,
+ "loss": 1.3608,
+ "theoretical_loss": 3.475146847582514,
+ "tokens_seen": 1890123776
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003453400200601806,
+ "loss": 1.2963,
+ "theoretical_loss": 3.4751367725718643,
+ "tokens_seen": 1890189312
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003453239719157473,
+ "loss": 1.398,
+ "theoretical_loss": 3.4751266980083306,
+ "tokens_seen": 1890254848
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000345307923771314,
+ "loss": 1.1694,
+ "theoretical_loss": 3.4751166238918776,
+ "tokens_seen": 1890320384
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452918756268807,
+ "loss": 1.2798,
+ "theoretical_loss": 3.4751065502224705,
+ "tokens_seen": 1890385920
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452758274824474,
+ "loss": 1.2935,
+ "theoretical_loss": 3.475096477000074,
+ "tokens_seen": 1890451456
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452597793380141,
+ "loss": 1.2158,
+ "theoretical_loss": 3.475086404224652,
+ "tokens_seen": 1890516992
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452437311935808,
+ "loss": 1.2658,
+ "theoretical_loss": 3.4750763318961697,
+ "tokens_seen": 1890582528
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452276830491475,
+ "loss": 1.238,
+ "theoretical_loss": 3.4750662600145916,
+ "tokens_seen": 1890648064
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 633930,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3091156482696533,
+ "objective/train/theoretical_loss": 3.4750561885798827,
+ "objective/train/tokens_used": 1911173600,
+ "theoretical_loss": 3.4750561885798827,
+ "tokens_seen": 1890713600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003452116349047142,
+ "loss": 1.3026,
+ "theoretical_loss": 3.4750561885798827,
+ "tokens_seen": 1890713600
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003451955867602809,
+ "loss": 1.3184,
+ "theoretical_loss": 3.475046117592007,
+ "tokens_seen": 1890779136
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003451795386158476,
+ "loss": 1.3288,
+ "theoretical_loss": 3.47503604705093,
+ "tokens_seen": 1890844672
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003451634904714143,
+ "loss": 1.2822,
+ "theoretical_loss": 3.4750259769566156,
+ "tokens_seen": 1890910208
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000345147442326981,
+ "loss": 1.17,
+ "theoretical_loss": 3.4750159073090297,
+ "tokens_seen": 1890975744
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003451313941825477,
+ "loss": 1.1888,
+ "theoretical_loss": 3.4750058381081357,
+ "tokens_seen": 1891041280
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034511534603811435,
+ "loss": 1.2295,
+ "theoretical_loss": 3.4749957693538986,
+ "tokens_seen": 1891106816
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034509929789368105,
+ "loss": 1.3152,
+ "theoretical_loss": 3.4749857010462835,
+ "tokens_seen": 1891172352
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034508324974924775,
+ "loss": 1.295,
+ "theoretical_loss": 3.474975633185255,
+ "tokens_seen": 1891237888
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034506720160481445,
+ "loss": 1.2137,
+ "theoretical_loss": 3.4749655657707783,
+ "tokens_seen": 1891303424
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034505115346038115,
+ "loss": 1.2384,
+ "theoretical_loss": 3.474955498802817,
+ "tokens_seen": 1891368960
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034503510531594785,
+ "loss": 1.3459,
+ "theoretical_loss": 3.4749454322813365,
+ "tokens_seen": 1891434496
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034501905717151455,
+ "loss": 1.3558,
+ "theoretical_loss": 3.474935366206301,
+ "tokens_seen": 1891500032
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034500300902708125,
+ "loss": 1.2696,
+ "theoretical_loss": 3.4749253005776763,
+ "tokens_seen": 1891565568
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034498696088264795,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4749152353954265,
+ "tokens_seen": 1891631104
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034497091273821465,
+ "loss": 1.3241,
+ "theoretical_loss": 3.4749051706595164,
+ "tokens_seen": 1891696640
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034495486459378135,
+ "loss": 1.2803,
+ "theoretical_loss": 3.4748951063699103,
+ "tokens_seen": 1891762176
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034493881644934805,
+ "loss": 1.2139,
+ "theoretical_loss": 3.474885042526574,
+ "tokens_seen": 1891827712
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034492276830491475,
+ "loss": 1.3661,
+ "theoretical_loss": 3.4748749791294706,
+ "tokens_seen": 1891893248
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034490672016048145,
+ "loss": 1.2944,
+ "theoretical_loss": 3.474864916178566,
+ "tokens_seen": 1891958784
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034489067201604816,
+ "loss": 1.31,
+ "theoretical_loss": 3.4748548536738255,
+ "tokens_seen": 1892024320
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034487462387161486,
+ "loss": 1.3213,
+ "theoretical_loss": 3.474844791615213,
+ "tokens_seen": 1892089856
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034485857572718156,
+ "loss": 1.2517,
+ "theoretical_loss": 3.4748347300026934,
+ "tokens_seen": 1892155392
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034484252758274826,
+ "loss": 1.3111,
+ "theoretical_loss": 3.474824668836231,
+ "tokens_seen": 1892220928
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034482647943831496,
+ "loss": 1.3167,
+ "theoretical_loss": 3.474814608115792,
+ "tokens_seen": 1892286464
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034481043129388166,
+ "loss": 1.2661,
+ "theoretical_loss": 3.474804547841339,
+ "tokens_seen": 1892352000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034479438314944836,
+ "loss": 1.3023,
+ "theoretical_loss": 3.474794488012839,
+ "tokens_seen": 1892417536
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034477833500501506,
+ "loss": 1.2426,
+ "theoretical_loss": 3.4747844286302563,
+ "tokens_seen": 1892483072
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034476228686058176,
+ "loss": 1.2107,
+ "theoretical_loss": 3.4747743696935545,
+ "tokens_seen": 1892548608
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034474623871614846,
+ "loss": 1.3656,
+ "theoretical_loss": 3.4747643112026996,
+ "tokens_seen": 1892614144
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034473019057171516,
+ "loss": 1.3224,
+ "theoretical_loss": 3.4747542531576556,
+ "tokens_seen": 1892679680
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034471414242728186,
+ "loss": 1.3857,
+ "theoretical_loss": 3.4747441955583875,
+ "tokens_seen": 1892745216
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034469809428284856,
+ "loss": 1.3303,
+ "theoretical_loss": 3.4747341384048607,
+ "tokens_seen": 1892810752
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034468204613841526,
+ "loss": 1.2032,
+ "theoretical_loss": 3.474724081697039,
+ "tokens_seen": 1892876288
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034466599799398196,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4747140254348885,
+ "tokens_seen": 1892941824
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034464994984954866,
+ "loss": 1.2251,
+ "theoretical_loss": 3.4747039696183726,
+ "tokens_seen": 1893007360
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034463390170511537,
+ "loss": 1.1538,
+ "theoretical_loss": 3.4746939142474575,
+ "tokens_seen": 1893072896
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034461785356068207,
+ "loss": 1.2843,
+ "theoretical_loss": 3.4746838593221065,
+ "tokens_seen": 1893138432
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034460180541624877,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4746738048422863,
+ "tokens_seen": 1893203968
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034458575727181547,
+ "loss": 1.3885,
+ "theoretical_loss": 3.4746637508079603,
+ "tokens_seen": 1893269504
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034456970912738217,
+ "loss": 1.3518,
+ "theoretical_loss": 3.4746536972190936,
+ "tokens_seen": 1893335040
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034455366098294887,
+ "loss": 1.3294,
+ "theoretical_loss": 3.4746436440756514,
+ "tokens_seen": 1893400576
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034453761283851557,
+ "loss": 1.2387,
+ "theoretical_loss": 3.4746335913775983,
+ "tokens_seen": 1893466112
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034452156469408227,
+ "loss": 1.3303,
+ "theoretical_loss": 3.474623539124899,
+ "tokens_seen": 1893531648
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034450551654964897,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4746134873175194,
+ "tokens_seen": 1893597184
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034448946840521567,
+ "loss": 1.3185,
+ "theoretical_loss": 3.474603435955423,
+ "tokens_seen": 1893662720
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034447342026078237,
+ "loss": 1.3028,
+ "theoretical_loss": 3.474593385038575,
+ "tokens_seen": 1893728256
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034445737211634907,
+ "loss": 1.3192,
+ "theoretical_loss": 3.474583334566941,
+ "tokens_seen": 1893793792
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034444132397191577,
+ "loss": 1.342,
+ "theoretical_loss": 3.474573284540485,
+ "tokens_seen": 1893859328
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003444252758274825,
+ "loss": 1.1973,
+ "theoretical_loss": 3.474563234959172,
+ "tokens_seen": 1893924864
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 635239,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4342564344406128,
+ "objective/train/theoretical_loss": 3.474553185822967,
+ "objective/train/tokens_used": 1914450400,
+ "theoretical_loss": 3.474553185822967,
+ "tokens_seen": 1893990400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003444092276830492,
+ "loss": 1.3198,
+ "theoretical_loss": 3.474553185822967,
+ "tokens_seen": 1893990400
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443931795386159,
+ "loss": 1.2737,
+ "theoretical_loss": 3.474543137131836,
+ "tokens_seen": 1894055936
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443771313941826,
+ "loss": 1.318,
+ "theoretical_loss": 3.474533088885742,
+ "tokens_seen": 1894121472
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443610832497493,
+ "loss": 1.2231,
+ "theoretical_loss": 3.474523041084651,
+ "tokens_seen": 1894187008
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443450351053159,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4745129937285277,
+ "tokens_seen": 1894252544
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443289869608826,
+ "loss": 1.3682,
+ "theoretical_loss": 3.4745029468173367,
+ "tokens_seen": 1894318080
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003443129388164493,
+ "loss": 1.314,
+ "theoretical_loss": 3.4744929003510436,
+ "tokens_seen": 1894383616
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000344296890672016,
+ "loss": 1.2886,
+ "theoretical_loss": 3.474482854329613,
+ "tokens_seen": 1894449152
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003442808425275827,
+ "loss": 1.3542,
+ "theoretical_loss": 3.474472808753009,
+ "tokens_seen": 1894514688
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003442647943831494,
+ "loss": 1.1662,
+ "theoretical_loss": 3.474462763621198,
+ "tokens_seen": 1894580224
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003442487462387161,
+ "loss": 1.2887,
+ "theoretical_loss": 3.4744527189341436,
+ "tokens_seen": 1894645760
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003442326980942828,
+ "loss": 1.2968,
+ "theoretical_loss": 3.4744426746918116,
+ "tokens_seen": 1894711296
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034421664994984953,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4744326308941664,
+ "tokens_seen": 1894776832
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034420060180541623,
+ "loss": 1.1862,
+ "theoretical_loss": 3.474422587541173,
+ "tokens_seen": 1894842368
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034418455366098293,
+ "loss": 1.2971,
+ "theoretical_loss": 3.4744125446327967,
+ "tokens_seen": 1894907904
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034416850551654963,
+ "loss": 1.3183,
+ "theoretical_loss": 3.474402502169002,
+ "tokens_seen": 1894973440
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034415245737211633,
+ "loss": 1.1865,
+ "theoretical_loss": 3.4743924601497542,
+ "tokens_seen": 1895038976
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003441364092276831,
+ "loss": 1.274,
+ "theoretical_loss": 3.474382418575018,
+ "tokens_seen": 1895104512
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003441203610832498,
+ "loss": 1.2753,
+ "theoretical_loss": 3.474372377444759,
+ "tokens_seen": 1895170048
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003441043129388165,
+ "loss": 1.1921,
+ "theoretical_loss": 3.474362336758941,
+ "tokens_seen": 1895235584
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003440882647943832,
+ "loss": 1.2538,
+ "theoretical_loss": 3.4743522965175293,
+ "tokens_seen": 1895301120
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003440722166499499,
+ "loss": 1.2322,
+ "theoretical_loss": 3.47434225672049,
+ "tokens_seen": 1895366656
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003440561685055166,
+ "loss": 1.2942,
+ "theoretical_loss": 3.4743322173677864,
+ "tokens_seen": 1895432192
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003440401203610833,
+ "loss": 1.2598,
+ "theoretical_loss": 3.4743221784593845,
+ "tokens_seen": 1895497728
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034402407221665,
+ "loss": 1.2917,
+ "theoretical_loss": 3.4743121399952495,
+ "tokens_seen": 1895563264
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003440080240722167,
+ "loss": 1.3415,
+ "theoretical_loss": 3.4743021019753453,
+ "tokens_seen": 1895628800
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439919759277834,
+ "loss": 1.2572,
+ "theoretical_loss": 3.474292064399638,
+ "tokens_seen": 1895694336
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439759277833501,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4742820272680914,
+ "tokens_seen": 1895759872
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439598796389168,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4742719905806716,
+ "tokens_seen": 1895825408
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439438314944835,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4742619543373436,
+ "tokens_seen": 1895890944
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439277833500502,
+ "loss": 1.3106,
+ "theoretical_loss": 3.4742519185380716,
+ "tokens_seen": 1895956480
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003439117352056169,
+ "loss": 1.2808,
+ "theoretical_loss": 3.4742418831828212,
+ "tokens_seen": 1896022016
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003438956870611836,
+ "loss": 1.3894,
+ "theoretical_loss": 3.4742318482715566,
+ "tokens_seen": 1896087552
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003438796389167503,
+ "loss": 1.333,
+ "theoretical_loss": 3.474221813804244,
+ "tokens_seen": 1896153088
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.000343863590772317,
+ "loss": 1.2456,
+ "theoretical_loss": 3.4742117797808474,
+ "tokens_seen": 1896218624
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003438475426278837,
+ "loss": 1.2233,
+ "theoretical_loss": 3.4742017462013326,
+ "tokens_seen": 1896284160
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003438314944834504,
+ "loss": 1.2812,
+ "theoretical_loss": 3.4741917130656645,
+ "tokens_seen": 1896349696
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003438154463390171,
+ "loss": 1.3476,
+ "theoretical_loss": 3.4741816803738077,
+ "tokens_seen": 1896415232
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437993981945838,
+ "loss": 1.3916,
+ "theoretical_loss": 3.4741716481257274,
+ "tokens_seen": 1896480768
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437833500501505,
+ "loss": 1.3828,
+ "theoretical_loss": 3.474161616321388,
+ "tokens_seen": 1896546304
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437673019057172,
+ "loss": 1.2785,
+ "theoretical_loss": 3.4741515849607563,
+ "tokens_seen": 1896611840
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437512537612839,
+ "loss": 1.2298,
+ "theoretical_loss": 3.474141554043795,
+ "tokens_seen": 1896677376
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437352056168506,
+ "loss": 1.3062,
+ "theoretical_loss": 3.4741315235704717,
+ "tokens_seen": 1896742912
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.0003437191574724173,
+ "loss": 1.2785,
+ "theoretical_loss": 3.47412149354075,
+ "tokens_seen": 1896808448
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034370310932798395,
+ "loss": 1.3084,
+ "theoretical_loss": 3.4741114639545945,
+ "tokens_seen": 1896873984
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034368706118355065,
+ "loss": 1.2558,
+ "theoretical_loss": 3.474101434811971,
+ "tokens_seen": 1896939520
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034367101303911735,
+ "loss": 1.2023,
+ "theoretical_loss": 3.4740914061128443,
+ "tokens_seen": 1897005056
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034365496489468405,
+ "loss": 1.3064,
+ "theoretical_loss": 3.47408137785718,
+ "tokens_seen": 1897070592
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034363891675025075,
+ "loss": 1.2492,
+ "theoretical_loss": 3.4740713500449427,
+ "tokens_seen": 1897136128
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034362286860581745,
+ "loss": 1.1774,
+ "theoretical_loss": 3.4740613226760972,
+ "tokens_seen": 1897201664
+ },
+ {
+ "epoch": 0.57,
+ "objective/train/docs_used": 636243,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.247380018234253,
+ "objective/train/theoretical_loss": 3.474051295750609,
+ "objective/train/tokens_used": 1917727200,
+ "theoretical_loss": 3.474051295750609,
+ "tokens_seen": 1897267200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034360682046138415,
+ "loss": 1.2707,
+ "theoretical_loss": 3.474051295750609,
+ "tokens_seen": 1897267200
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034359077231695085,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4740412692684437,
+ "tokens_seen": 1897332736
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034357472417251755,
+ "loss": 1.2457,
+ "theoretical_loss": 3.4740312432295655,
+ "tokens_seen": 1897398272
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 0.00034355867602808425,
+ "loss": 1.2609,
+ "theoretical_loss": 3.47402121763394,
+ "tokens_seen": 1897463808
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034354262788365095,
+ "loss": 1.3008,
+ "theoretical_loss": 3.474011192481531,
+ "tokens_seen": 1897529344
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034352657973921765,
+ "loss": 1.2505,
+ "theoretical_loss": 3.4740011677723057,
+ "tokens_seen": 1897594880
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034351053159478436,
+ "loss": 1.1974,
+ "theoretical_loss": 3.473991143506228,
+ "tokens_seen": 1897660416
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034349448345035106,
+ "loss": 1.3438,
+ "theoretical_loss": 3.4739811196832626,
+ "tokens_seen": 1897725952
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034347843530591776,
+ "loss": 1.3501,
+ "theoretical_loss": 3.473971096303376,
+ "tokens_seen": 1897791488
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034346238716148446,
+ "loss": 1.2011,
+ "theoretical_loss": 3.4739610733665325,
+ "tokens_seen": 1897857024
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034344633901705116,
+ "loss": 1.3031,
+ "theoretical_loss": 3.4739510508726967,
+ "tokens_seen": 1897922560
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034343029087261786,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4739410288218346,
+ "tokens_seen": 1897988096
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034341424272818456,
+ "loss": 1.3224,
+ "theoretical_loss": 3.473931007213911,
+ "tokens_seen": 1898053632
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034339819458375126,
+ "loss": 1.3595,
+ "theoretical_loss": 3.473920986048891,
+ "tokens_seen": 1898119168
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034338214643931796,
+ "loss": 1.3369,
+ "theoretical_loss": 3.4739109653267395,
+ "tokens_seen": 1898184704
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034336609829488466,
+ "loss": 1.2238,
+ "theoretical_loss": 3.473900945047422,
+ "tokens_seen": 1898250240
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034335005015045136,
+ "loss": 1.3535,
+ "theoretical_loss": 3.4738909252109043,
+ "tokens_seen": 1898315776
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034333400200601806,
+ "loss": 1.3003,
+ "theoretical_loss": 3.4738809058171496,
+ "tokens_seen": 1898381312
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034331795386158476,
+ "loss": 1.3546,
+ "theoretical_loss": 3.473870886866125,
+ "tokens_seen": 1898446848
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034330190571715146,
+ "loss": 1.2407,
+ "theoretical_loss": 3.4738608683577947,
+ "tokens_seen": 1898512384
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034328585757271816,
+ "loss": 1.3227,
+ "theoretical_loss": 3.473850850292124,
+ "tokens_seen": 1898577920
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034326980942828487,
+ "loss": 1.2954,
+ "theoretical_loss": 3.473840832669078,
+ "tokens_seen": 1898643456
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034325376128385157,
+ "loss": 1.1862,
+ "theoretical_loss": 3.473830815488622,
+ "tokens_seen": 1898708992
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034323771313941827,
+ "loss": 1.3211,
+ "theoretical_loss": 3.4738207987507215,
+ "tokens_seen": 1898774528
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034322166499498497,
+ "loss": 1.3274,
+ "theoretical_loss": 3.473810782455341,
+ "tokens_seen": 1898840064
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034320561685055167,
+ "loss": 1.265,
+ "theoretical_loss": 3.473800766602446,
+ "tokens_seen": 1898905600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034318956870611837,
+ "loss": 1.3108,
+ "theoretical_loss": 3.473790751192002,
+ "tokens_seen": 1898971136
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034317352056168507,
+ "loss": 1.2596,
+ "theoretical_loss": 3.4737807362239734,
+ "tokens_seen": 1899036672
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034315747241725177,
+ "loss": 1.3156,
+ "theoretical_loss": 3.473770721698326,
+ "tokens_seen": 1899102208
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034314142427281847,
+ "loss": 1.2043,
+ "theoretical_loss": 3.473760707615025,
+ "tokens_seen": 1899167744
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034312537612838517,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4737506939740355,
+ "tokens_seen": 1899233280
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034310932798395187,
+ "loss": 1.2747,
+ "theoretical_loss": 3.4737406807753226,
+ "tokens_seen": 1899298816
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034309327983951857,
+ "loss": 1.3753,
+ "theoretical_loss": 3.4737306680188516,
+ "tokens_seen": 1899364352
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034307723169508527,
+ "loss": 1.3495,
+ "theoretical_loss": 3.4737206557045877,
+ "tokens_seen": 1899429888
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000343061183550652,
+ "loss": 1.2598,
+ "theoretical_loss": 3.473710643832496,
+ "tokens_seen": 1899495424
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003430451354062187,
+ "loss": 1.1686,
+ "theoretical_loss": 3.4737006324025415,
+ "tokens_seen": 1899560960
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003430290872617854,
+ "loss": 1.2165,
+ "theoretical_loss": 3.47369062141469,
+ "tokens_seen": 1899626496
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003430130391173521,
+ "loss": 1.248,
+ "theoretical_loss": 3.4736806108689064,
+ "tokens_seen": 1899692032
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429969909729188,
+ "loss": 1.2529,
+ "theoretical_loss": 3.473670600765156,
+ "tokens_seen": 1899757568
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429809428284855,
+ "loss": 1.1621,
+ "theoretical_loss": 3.473660591103404,
+ "tokens_seen": 1899823104
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429648946840522,
+ "loss": 1.3636,
+ "theoretical_loss": 3.4736505818836156,
+ "tokens_seen": 1899888640
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429488465396189,
+ "loss": 1.3198,
+ "theoretical_loss": 3.4736405731057562,
+ "tokens_seen": 1899954176
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429327983951856,
+ "loss": 1.2778,
+ "theoretical_loss": 3.473630564769791,
+ "tokens_seen": 1900019712
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003429167502507523,
+ "loss": 1.3051,
+ "theoretical_loss": 3.473620556875685,
+ "tokens_seen": 1900085248
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000342900702106319,
+ "loss": 1.2498,
+ "theoretical_loss": 3.4736105494234035,
+ "tokens_seen": 1900150784
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428846539618857,
+ "loss": 1.1991,
+ "theoretical_loss": 3.473600542412912,
+ "tokens_seen": 1900216320
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428686058174524,
+ "loss": 1.3323,
+ "theoretical_loss": 3.473590535844176,
+ "tokens_seen": 1900281856
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428525576730191,
+ "loss": 1.2312,
+ "theoretical_loss": 3.47358052971716,
+ "tokens_seen": 1900347392
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428365095285858,
+ "loss": 1.2788,
+ "theoretical_loss": 3.47357052403183,
+ "tokens_seen": 1900412928
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428204613841525,
+ "loss": 1.284,
+ "theoretical_loss": 3.473560518788151,
+ "tokens_seen": 1900478464
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 637457,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2931511402130127,
+ "objective/train/theoretical_loss": 3.473550513986088,
+ "objective/train/tokens_used": 1921004000,
+ "theoretical_loss": 3.473550513986088,
+ "tokens_seen": 1900544000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003428044132397192,
+ "loss": 1.3519,
+ "theoretical_loss": 3.473550513986088,
+ "tokens_seen": 1900544000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003427883650952859,
+ "loss": 1.4324,
+ "theoretical_loss": 3.4735405096256065,
+ "tokens_seen": 1900609536
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003427723169508526,
+ "loss": 1.3575,
+ "theoretical_loss": 3.473530505706672,
+ "tokens_seen": 1900675072
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003427562688064193,
+ "loss": 1.203,
+ "theoretical_loss": 3.4735205022292495,
+ "tokens_seen": 1900740608
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000342740220661986,
+ "loss": 1.2644,
+ "theoretical_loss": 3.473510499193304,
+ "tokens_seen": 1900806144
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003427241725175527,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4735004965988017,
+ "tokens_seen": 1900871680
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003427081243731194,
+ "loss": 1.3334,
+ "theoretical_loss": 3.4734904944457075,
+ "tokens_seen": 1900937216
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426920762286861,
+ "loss": 1.2499,
+ "theoretical_loss": 3.473480492733986,
+ "tokens_seen": 1901002752
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426760280842528,
+ "loss": 1.4125,
+ "theoretical_loss": 3.4734704914636034,
+ "tokens_seen": 1901068288
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426599799398195,
+ "loss": 1.2694,
+ "theoretical_loss": 3.4734604906345248,
+ "tokens_seen": 1901133824
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426439317953862,
+ "loss": 1.3579,
+ "theoretical_loss": 3.4734504902467154,
+ "tokens_seen": 1901199360
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426278836509529,
+ "loss": 1.3578,
+ "theoretical_loss": 3.47344049030014,
+ "tokens_seen": 1901264896
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003426118355065196,
+ "loss": 1.1838,
+ "theoretical_loss": 3.4734304907947653,
+ "tokens_seen": 1901330432
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003425957873620863,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4734204917305553,
+ "tokens_seen": 1901395968
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000342579739217653,
+ "loss": 1.2379,
+ "theoretical_loss": 3.473410493107476,
+ "tokens_seen": 1901461504
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003425636910732197,
+ "loss": 1.2518,
+ "theoretical_loss": 3.4734004949254924,
+ "tokens_seen": 1901527040
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003425476429287864,
+ "loss": 1.4048,
+ "theoretical_loss": 3.47339049718457,
+ "tokens_seen": 1901592576
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003425315947843531,
+ "loss": 1.2727,
+ "theoretical_loss": 3.4733804998846747,
+ "tokens_seen": 1901658112
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003425155466399198,
+ "loss": 1.2697,
+ "theoretical_loss": 3.4733705030257704,
+ "tokens_seen": 1901723648
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424994984954865,
+ "loss": 1.2625,
+ "theoretical_loss": 3.4733605066078237,
+ "tokens_seen": 1901789184
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424834503510532,
+ "loss": 1.1387,
+ "theoretical_loss": 3.4733505106308,
+ "tokens_seen": 1901854720
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424674022066199,
+ "loss": 1.35,
+ "theoretical_loss": 3.473340515094664,
+ "tokens_seen": 1901920256
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424513540621866,
+ "loss": 1.1771,
+ "theoretical_loss": 3.473330519999381,
+ "tokens_seen": 1901985792
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424353059177533,
+ "loss": 1.2577,
+ "theoretical_loss": 3.473320525344917,
+ "tokens_seen": 1902051328
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034241925777332,
+ "loss": 1.3879,
+ "theoretical_loss": 3.473310531131237,
+ "tokens_seen": 1902116864
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003424032096288867,
+ "loss": 1.2248,
+ "theoretical_loss": 3.4733005373583064,
+ "tokens_seen": 1902182400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003423871614844534,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4732905440260904,
+ "tokens_seen": 1902247936
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003423711133400201,
+ "loss": 1.1326,
+ "theoretical_loss": 3.473280551134555,
+ "tokens_seen": 1902313472
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003423550651955868,
+ "loss": 1.2457,
+ "theoretical_loss": 3.473270558683665,
+ "tokens_seen": 1902379008
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003423390170511535,
+ "loss": 1.2991,
+ "theoretical_loss": 3.473260566673386,
+ "tokens_seen": 1902444544
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003423229689067202,
+ "loss": 1.3326,
+ "theoretical_loss": 3.473250575103683,
+ "tokens_seen": 1902510080
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034230692076228685,
+ "loss": 1.2935,
+ "theoretical_loss": 3.473240583974522,
+ "tokens_seen": 1902575616
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034229087261785355,
+ "loss": 1.2272,
+ "theoretical_loss": 3.4732305932858685,
+ "tokens_seen": 1902641152
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034227482447342025,
+ "loss": 1.2277,
+ "theoretical_loss": 3.473220603037687,
+ "tokens_seen": 1902706688
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034225877632898695,
+ "loss": 1.203,
+ "theoretical_loss": 3.473210613229944,
+ "tokens_seen": 1902772224
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034224272818455365,
+ "loss": 1.3093,
+ "theoretical_loss": 3.473200623862604,
+ "tokens_seen": 1902837760
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034222668004012035,
+ "loss": 1.2145,
+ "theoretical_loss": 3.473190634935633,
+ "tokens_seen": 1902903296
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034221063189568705,
+ "loss": 1.2696,
+ "theoretical_loss": 3.4731806464489963,
+ "tokens_seen": 1902968832
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034219458375125375,
+ "loss": 1.2956,
+ "theoretical_loss": 3.473170658402659,
+ "tokens_seen": 1903034368
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034217853560682045,
+ "loss": 1.3289,
+ "theoretical_loss": 3.473160670796587,
+ "tokens_seen": 1903099904
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034216248746238715,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4731506836307453,
+ "tokens_seen": 1903165440
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034214643931795386,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4731406969050997,
+ "tokens_seen": 1903230976
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034213039117352056,
+ "loss": 1.2736,
+ "theoretical_loss": 3.473130710619616,
+ "tokens_seen": 1903296512
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034211434302908726,
+ "loss": 1.2114,
+ "theoretical_loss": 3.473120724774258,
+ "tokens_seen": 1903362048
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034209829488465396,
+ "loss": 1.2031,
+ "theoretical_loss": 3.4731107393689933,
+ "tokens_seen": 1903427584
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034208224674022066,
+ "loss": 1.297,
+ "theoretical_loss": 3.473100754403786,
+ "tokens_seen": 1903493120
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034206619859578736,
+ "loss": 1.3535,
+ "theoretical_loss": 3.4730907698786018,
+ "tokens_seen": 1903558656
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034205015045135406,
+ "loss": 1.2116,
+ "theoretical_loss": 3.4730807857934063,
+ "tokens_seen": 1903624192
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034203410230692076,
+ "loss": 1.336,
+ "theoretical_loss": 3.473070802148165,
+ "tokens_seen": 1903689728
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034201805416248746,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4730608189428436,
+ "tokens_seen": 1903755264
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 638270,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2350528240203857,
+ "objective/train/theoretical_loss": 3.4730508361774066,
+ "objective/train/tokens_used": 1924280800,
+ "theoretical_loss": 3.4730508361774066,
+ "tokens_seen": 1903820800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034200200601805416,
+ "loss": 1.2642,
+ "theoretical_loss": 3.4730508361774066,
+ "tokens_seen": 1903820800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034198595787362086,
+ "loss": 1.1995,
+ "theoretical_loss": 3.4730408538518205,
+ "tokens_seen": 1903886336
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034196990972918756,
+ "loss": 1.2251,
+ "theoretical_loss": 3.47303087196605,
+ "tokens_seen": 1903951872
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034195386158475426,
+ "loss": 1.2938,
+ "theoretical_loss": 3.473020890520062,
+ "tokens_seen": 1904017408
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034193781344032096,
+ "loss": 1.2223,
+ "theoretical_loss": 3.47301090951382,
+ "tokens_seen": 1904082944
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034192176529588766,
+ "loss": 1.3467,
+ "theoretical_loss": 3.4730009289472914,
+ "tokens_seen": 1904148480
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034190571715145436,
+ "loss": 1.3448,
+ "theoretical_loss": 3.4729909488204402,
+ "tokens_seen": 1904214016
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034188966900702107,
+ "loss": 1.2684,
+ "theoretical_loss": 3.472980969133232,
+ "tokens_seen": 1904279552
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034187362086258777,
+ "loss": 1.2732,
+ "theoretical_loss": 3.4729709898856336,
+ "tokens_seen": 1904345088
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034185757271815447,
+ "loss": 1.3134,
+ "theoretical_loss": 3.4729610110776097,
+ "tokens_seen": 1904410624
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034184152457372117,
+ "loss": 1.277,
+ "theoretical_loss": 3.4729510327091253,
+ "tokens_seen": 1904476160
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034182547642928787,
+ "loss": 1.3276,
+ "theoretical_loss": 3.4729410547801463,
+ "tokens_seen": 1904541696
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034180942828485457,
+ "loss": 1.2706,
+ "theoretical_loss": 3.4729310772906388,
+ "tokens_seen": 1904607232
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034179338014042127,
+ "loss": 1.2142,
+ "theoretical_loss": 3.472921100240568,
+ "tokens_seen": 1904672768
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034177733199598797,
+ "loss": 1.3635,
+ "theoretical_loss": 3.472911123629899,
+ "tokens_seen": 1904738304
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034176128385155467,
+ "loss": 1.2714,
+ "theoretical_loss": 3.4729011474585976,
+ "tokens_seen": 1904803840
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034174523570712137,
+ "loss": 1.3285,
+ "theoretical_loss": 3.4728911717266295,
+ "tokens_seen": 1904869376
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003417291875626881,
+ "loss": 1.3742,
+ "theoretical_loss": 3.47288119643396,
+ "tokens_seen": 1904934912
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003417131394182548,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4728712215805544,
+ "tokens_seen": 1905000448
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034169709127382153,
+ "loss": 1.4007,
+ "theoretical_loss": 3.472861247166379,
+ "tokens_seen": 1905065984
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034168104312938823,
+ "loss": 1.3081,
+ "theoretical_loss": 3.4728512731913987,
+ "tokens_seen": 1905131520
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003416649949849549,
+ "loss": 1.3103,
+ "theoretical_loss": 3.4728412996555797,
+ "tokens_seen": 1905197056
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003416489468405216,
+ "loss": 1.2175,
+ "theoretical_loss": 3.472831326558887,
+ "tokens_seen": 1905262592
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003416328986960883,
+ "loss": 1.2998,
+ "theoretical_loss": 3.4728213539012858,
+ "tokens_seen": 1905328128
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000341616850551655,
+ "loss": 1.3657,
+ "theoretical_loss": 3.472811381682743,
+ "tokens_seen": 1905393664
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003416008024072217,
+ "loss": 1.1165,
+ "theoretical_loss": 3.472801409903223,
+ "tokens_seen": 1905459200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415847542627884,
+ "loss": 1.2717,
+ "theoretical_loss": 3.472791438562691,
+ "tokens_seen": 1905524736
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415687061183551,
+ "loss": 1.346,
+ "theoretical_loss": 3.472781467661114,
+ "tokens_seen": 1905590272
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415526579739218,
+ "loss": 1.3452,
+ "theoretical_loss": 3.472771497198457,
+ "tokens_seen": 1905655808
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415366098294885,
+ "loss": 1.3512,
+ "theoretical_loss": 3.472761527174685,
+ "tokens_seen": 1905721344
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415205616850552,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4727515575897643,
+ "tokens_seen": 1905786880
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003415045135406219,
+ "loss": 1.3,
+ "theoretical_loss": 3.47274158844366,
+ "tokens_seen": 1905852416
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003414884653961886,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4727316197363383,
+ "tokens_seen": 1905917952
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003414724172517553,
+ "loss": 1.2441,
+ "theoretical_loss": 3.472721651467764,
+ "tokens_seen": 1905983488
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000341456369107322,
+ "loss": 1.3087,
+ "theoretical_loss": 3.472711683637904,
+ "tokens_seen": 1906049024
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003414403209628887,
+ "loss": 1.2881,
+ "theoretical_loss": 3.472701716246722,
+ "tokens_seen": 1906114560
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003414242728184554,
+ "loss": 1.2606,
+ "theoretical_loss": 3.4726917492941856,
+ "tokens_seen": 1906180096
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003414082246740221,
+ "loss": 1.3042,
+ "theoretical_loss": 3.472681782780259,
+ "tokens_seen": 1906245632
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413921765295888,
+ "loss": 1.2833,
+ "theoretical_loss": 3.472671816704908,
+ "tokens_seen": 1906311168
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413761283851555,
+ "loss": 1.3791,
+ "theoretical_loss": 3.472661851068099,
+ "tokens_seen": 1906376704
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413600802407222,
+ "loss": 1.3409,
+ "theoretical_loss": 3.472651885869797,
+ "tokens_seen": 1906442240
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413440320962889,
+ "loss": 1.2872,
+ "theoretical_loss": 3.4726419211099677,
+ "tokens_seen": 1906507776
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413279839518556,
+ "loss": 1.2778,
+ "theoretical_loss": 3.472631956788577,
+ "tokens_seen": 1906573312
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003413119358074223,
+ "loss": 1.266,
+ "theoretical_loss": 3.47262199290559,
+ "tokens_seen": 1906638848
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000341295887662989,
+ "loss": 1.369,
+ "theoretical_loss": 3.472612029460973,
+ "tokens_seen": 1906704384
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003412798395185557,
+ "loss": 1.2039,
+ "theoretical_loss": 3.4726020664546913,
+ "tokens_seen": 1906769920
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003412637913741224,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4725921038867105,
+ "tokens_seen": 1906835456
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003412477432296891,
+ "loss": 1.2744,
+ "theoretical_loss": 3.4725821417569964,
+ "tokens_seen": 1906900992
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003412316950852558,
+ "loss": 1.3848,
+ "theoretical_loss": 3.472572180065515,
+ "tokens_seen": 1906966528
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003412156469408225,
+ "loss": 1.2444,
+ "theoretical_loss": 3.472562218812231,
+ "tokens_seen": 1907032064
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 639529,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.321748971939087,
+ "objective/train/theoretical_loss": 3.4725522579971106,
+ "objective/train/tokens_used": 1927557600,
+ "theoretical_loss": 3.4725522579971106,
+ "tokens_seen": 1907097600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411995987963892,
+ "loss": 1.3124,
+ "theoretical_loss": 3.4725522579971106,
+ "tokens_seen": 1907097600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411835506519559,
+ "loss": 1.3336,
+ "theoretical_loss": 3.47254229762012,
+ "tokens_seen": 1907163136
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411675025075226,
+ "loss": 1.2696,
+ "theoretical_loss": 3.4725323376812236,
+ "tokens_seen": 1907228672
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411514543630893,
+ "loss": 1.1987,
+ "theoretical_loss": 3.472522378180389,
+ "tokens_seen": 1907294208
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000341135406218656,
+ "loss": 1.3674,
+ "theoretical_loss": 3.47251241911758,
+ "tokens_seen": 1907359744
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411193580742227,
+ "loss": 1.3064,
+ "theoretical_loss": 3.472502460492763,
+ "tokens_seen": 1907425280
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003411033099297894,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4724925023059043,
+ "tokens_seen": 1907490816
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410872617853561,
+ "loss": 1.3269,
+ "theoretical_loss": 3.4724825445569687,
+ "tokens_seen": 1907556352
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410712136409228,
+ "loss": 1.3637,
+ "theoretical_loss": 3.472472587245922,
+ "tokens_seen": 1907621888
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410551654964895,
+ "loss": 1.3615,
+ "theoretical_loss": 3.47246263037273,
+ "tokens_seen": 1907687424
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410391173520562,
+ "loss": 1.2879,
+ "theoretical_loss": 3.472452673937359,
+ "tokens_seen": 1907752960
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410230692076229,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4724427179397734,
+ "tokens_seen": 1907818496
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003410070210631896,
+ "loss": 1.3401,
+ "theoretical_loss": 3.472432762379941,
+ "tokens_seen": 1907884032
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003409909729187563,
+ "loss": 1.3418,
+ "theoretical_loss": 3.4724228072578254,
+ "tokens_seen": 1907949568
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000340974924774323,
+ "loss": 1.2865,
+ "theoretical_loss": 3.4724128525733935,
+ "tokens_seen": 1908015104
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003409588766298897,
+ "loss": 1.3213,
+ "theoretical_loss": 3.47240289832661,
+ "tokens_seen": 1908080640
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003409428284854564,
+ "loss": 1.2519,
+ "theoretical_loss": 3.472392944517442,
+ "tokens_seen": 1908146176
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003409267803410231,
+ "loss": 1.3364,
+ "theoretical_loss": 3.472382991145855,
+ "tokens_seen": 1908211712
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003409107321965898,
+ "loss": 1.2792,
+ "theoretical_loss": 3.4723730382118134,
+ "tokens_seen": 1908277248
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034089468405215645,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4723630857152843,
+ "tokens_seen": 1908342784
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034087863590772315,
+ "loss": 1.253,
+ "theoretical_loss": 3.4723531336562328,
+ "tokens_seen": 1908408320
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034086258776328985,
+ "loss": 1.1897,
+ "theoretical_loss": 3.472343182034625,
+ "tokens_seen": 1908473856
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034084653961885655,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4723332308504267,
+ "tokens_seen": 1908539392
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034083049147442325,
+ "loss": 1.3273,
+ "theoretical_loss": 3.472323280103603,
+ "tokens_seen": 1908604928
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034081444332998995,
+ "loss": 1.3726,
+ "theoretical_loss": 3.4723133297941198,
+ "tokens_seen": 1908670464
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034079839518555665,
+ "loss": 1.1786,
+ "theoretical_loss": 3.472303379921944,
+ "tokens_seen": 1908736000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034078234704112335,
+ "loss": 1.3043,
+ "theoretical_loss": 3.47229343048704,
+ "tokens_seen": 1908801536
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034076629889669006,
+ "loss": 1.376,
+ "theoretical_loss": 3.472283481489374,
+ "tokens_seen": 1908867072
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034075025075225676,
+ "loss": 1.342,
+ "theoretical_loss": 3.472273532928912,
+ "tokens_seen": 1908932608
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034073420260782346,
+ "loss": 1.2983,
+ "theoretical_loss": 3.47226358480562,
+ "tokens_seen": 1908998144
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034071815446339016,
+ "loss": 1.241,
+ "theoretical_loss": 3.472253637119463,
+ "tokens_seen": 1909063680
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034070210631895686,
+ "loss": 1.3002,
+ "theoretical_loss": 3.4722436898704077,
+ "tokens_seen": 1909129216
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034068605817452356,
+ "loss": 1.2919,
+ "theoretical_loss": 3.4722337430584194,
+ "tokens_seen": 1909194752
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034067001003009026,
+ "loss": 1.3391,
+ "theoretical_loss": 3.4722237966834633,
+ "tokens_seen": 1909260288
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034065396188565696,
+ "loss": 1.3131,
+ "theoretical_loss": 3.472213850745506,
+ "tokens_seen": 1909325824
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034063791374122366,
+ "loss": 1.2926,
+ "theoretical_loss": 3.4722039052445135,
+ "tokens_seen": 1909391360
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034062186559679036,
+ "loss": 1.3557,
+ "theoretical_loss": 3.4721939601804506,
+ "tokens_seen": 1909456896
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034060581745235706,
+ "loss": 1.3762,
+ "theoretical_loss": 3.4721840155532844,
+ "tokens_seen": 1909522432
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034058976930792376,
+ "loss": 1.2973,
+ "theoretical_loss": 3.4721740713629794,
+ "tokens_seen": 1909587968
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034057372116349046,
+ "loss": 1.3813,
+ "theoretical_loss": 3.4721641276095023,
+ "tokens_seen": 1909653504
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034055767301905716,
+ "loss": 1.2666,
+ "theoretical_loss": 3.472154184292819,
+ "tokens_seen": 1909719040
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034054162487462386,
+ "loss": 1.1721,
+ "theoretical_loss": 3.4721442414128942,
+ "tokens_seen": 1909784576
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003405255767301906,
+ "loss": 1.3085,
+ "theoretical_loss": 3.4721342989696953,
+ "tokens_seen": 1909850112
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003405095285857573,
+ "loss": 1.2772,
+ "theoretical_loss": 3.472124356963187,
+ "tokens_seen": 1909915648
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000340493480441324,
+ "loss": 1.2309,
+ "theoretical_loss": 3.4721144153933357,
+ "tokens_seen": 1909981184
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003404774322968907,
+ "loss": 1.292,
+ "theoretical_loss": 3.4721044742601066,
+ "tokens_seen": 1910046720
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003404613841524574,
+ "loss": 1.2997,
+ "theoretical_loss": 3.4720945335634665,
+ "tokens_seen": 1910112256
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003404453360080241,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4720845933033804,
+ "tokens_seen": 1910177792
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003404292878635908,
+ "loss": 1.2563,
+ "theoretical_loss": 3.4720746534798144,
+ "tokens_seen": 1910243328
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003404132397191575,
+ "loss": 1.2148,
+ "theoretical_loss": 3.4720647140927348,
+ "tokens_seen": 1910308864
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 640490,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3794310092926025,
+ "objective/train/theoretical_loss": 3.472054775142107,
+ "objective/train/tokens_used": 1930834400,
+ "theoretical_loss": 3.472054775142107,
+ "tokens_seen": 1910374400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003403971915747242,
+ "loss": 1.2561,
+ "theoretical_loss": 3.472054775142107,
+ "tokens_seen": 1910374400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003403811434302909,
+ "loss": 1.3067,
+ "theoretical_loss": 3.4720448366278966,
+ "tokens_seen": 1910439936
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003403650952858576,
+ "loss": 1.2907,
+ "theoretical_loss": 3.47203489855007,
+ "tokens_seen": 1910505472
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003403490471414243,
+ "loss": 1.2845,
+ "theoretical_loss": 3.472024960908593,
+ "tokens_seen": 1910571008
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000340332998996991,
+ "loss": 1.2744,
+ "theoretical_loss": 3.4720150237034315,
+ "tokens_seen": 1910636544
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034031695085255773,
+ "loss": 1.4365,
+ "theoretical_loss": 3.4720050869345513,
+ "tokens_seen": 1910702080
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034030090270812443,
+ "loss": 1.264,
+ "theoretical_loss": 3.4719951506019178,
+ "tokens_seen": 1910767616
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00034028485456369113,
+ "loss": 1.2367,
+ "theoretical_loss": 3.471985214705497,
+ "tokens_seen": 1910833152
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003402688064192578,
+ "loss": 1.3216,
+ "theoretical_loss": 3.471975279245256,
+ "tokens_seen": 1910898688
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003402527582748245,
+ "loss": 1.2689,
+ "theoretical_loss": 3.471965344221159,
+ "tokens_seen": 1910964224
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003402367101303912,
+ "loss": 1.2882,
+ "theoretical_loss": 3.4719554096331735,
+ "tokens_seen": 1911029760
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003402206619859579,
+ "loss": 1.3626,
+ "theoretical_loss": 3.471945475481264,
+ "tokens_seen": 1911095296
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003402046138415246,
+ "loss": 1.2561,
+ "theoretical_loss": 3.471935541765397,
+ "tokens_seen": 1911160832
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003401885656970913,
+ "loss": 1.3408,
+ "theoretical_loss": 3.471925608485539,
+ "tokens_seen": 1911226368
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000340172517552658,
+ "loss": 1.2421,
+ "theoretical_loss": 3.4719156756416547,
+ "tokens_seen": 1911291904
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003401564694082247,
+ "loss": 1.2793,
+ "theoretical_loss": 3.471905743233711,
+ "tokens_seen": 1911357440
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003401404212637914,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4718958112616733,
+ "tokens_seen": 1911422976
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003401243731193581,
+ "loss": 1.3562,
+ "theoretical_loss": 3.471885879725508,
+ "tokens_seen": 1911488512
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003401083249749248,
+ "loss": 1.3743,
+ "theoretical_loss": 3.47187594862518,
+ "tokens_seen": 1911554048
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003400922768304915,
+ "loss": 1.2788,
+ "theoretical_loss": 3.4718660179606564,
+ "tokens_seen": 1911619584
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003400762286860582,
+ "loss": 1.2732,
+ "theoretical_loss": 3.4718560877319025,
+ "tokens_seen": 1911685120
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003400601805416249,
+ "loss": 1.2543,
+ "theoretical_loss": 3.471846157938885,
+ "tokens_seen": 1911750656
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003400441323971916,
+ "loss": 1.3305,
+ "theoretical_loss": 3.4718362285815685,
+ "tokens_seen": 1911816192
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003400280842527583,
+ "loss": 1.1654,
+ "theoretical_loss": 3.47182629965992,
+ "tokens_seen": 1911881728
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000340012036108325,
+ "loss": 1.2997,
+ "theoretical_loss": 3.4718163711739054,
+ "tokens_seen": 1911947264
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399959879638917,
+ "loss": 1.2786,
+ "theoretical_loss": 3.4718064431234903,
+ "tokens_seen": 1912012800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399799398194584,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4717965155086405,
+ "tokens_seen": 1912078336
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399638916750251,
+ "loss": 1.2186,
+ "theoretical_loss": 3.471786588329323,
+ "tokens_seen": 1912143872
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399478435305918,
+ "loss": 1.366,
+ "theoretical_loss": 3.471776661585502,
+ "tokens_seen": 1912209408
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399317953861585,
+ "loss": 1.2173,
+ "theoretical_loss": 3.471766735277145,
+ "tokens_seen": 1912274944
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003399157472417252,
+ "loss": 1.177,
+ "theoretical_loss": 3.4717568094042175,
+ "tokens_seen": 1912340480
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398996990972919,
+ "loss": 1.3297,
+ "theoretical_loss": 3.471746883966685,
+ "tokens_seen": 1912406016
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398836509528586,
+ "loss": 1.3362,
+ "theoretical_loss": 3.471736958964515,
+ "tokens_seen": 1912471552
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398676028084253,
+ "loss": 1.2236,
+ "theoretical_loss": 3.471727034397671,
+ "tokens_seen": 1912537088
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000339851554663992,
+ "loss": 1.314,
+ "theoretical_loss": 3.4717171102661215,
+ "tokens_seen": 1912602624
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398355065195587,
+ "loss": 1.2637,
+ "theoretical_loss": 3.471707186569831,
+ "tokens_seen": 1912668160
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398194583751254,
+ "loss": 1.1917,
+ "theoretical_loss": 3.4716972633087657,
+ "tokens_seen": 1912733696
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003398034102306921,
+ "loss": 1.1985,
+ "theoretical_loss": 3.471687340482892,
+ "tokens_seen": 1912799232
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397873620862588,
+ "loss": 1.2209,
+ "theoretical_loss": 3.4716774180921752,
+ "tokens_seen": 1912864768
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397713139418255,
+ "loss": 1.303,
+ "theoretical_loss": 3.4716674961365825,
+ "tokens_seen": 1912930304
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397552657973922,
+ "loss": 1.3331,
+ "theoretical_loss": 3.4716575746160787,
+ "tokens_seen": 1912995840
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397392176529589,
+ "loss": 1.2819,
+ "theoretical_loss": 3.4716476535306304,
+ "tokens_seen": 1913061376
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397231695085256,
+ "loss": 1.2822,
+ "theoretical_loss": 3.471637732880204,
+ "tokens_seen": 1913126912
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003397071213640923,
+ "loss": 1.3185,
+ "theoretical_loss": 3.4716278126647646,
+ "tokens_seen": 1913192448
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000339691073219659,
+ "loss": 1.2822,
+ "theoretical_loss": 3.4716178928842787,
+ "tokens_seen": 1913257984
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003396750250752257,
+ "loss": 1.289,
+ "theoretical_loss": 3.4716079735387124,
+ "tokens_seen": 1913323520
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003396589769307924,
+ "loss": 1.2929,
+ "theoretical_loss": 3.471598054628031,
+ "tokens_seen": 1913389056
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003396429287863591,
+ "loss": 1.4238,
+ "theoretical_loss": 3.471588136152202,
+ "tokens_seen": 1913454592
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003396268806419258,
+ "loss": 1.3002,
+ "theoretical_loss": 3.471578218111191,
+ "tokens_seen": 1913520128
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003396108324974925,
+ "loss": 1.3253,
+ "theoretical_loss": 3.471568300504963,
+ "tokens_seen": 1913585664
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 641663,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2823185920715332,
+ "objective/train/theoretical_loss": 3.471558383333485,
+ "objective/train/tokens_used": 1934111200,
+ "theoretical_loss": 3.471558383333485,
+ "tokens_seen": 1913651200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003395947843530592,
+ "loss": 1.3023,
+ "theoretical_loss": 3.471558383333485,
+ "tokens_seen": 1913651200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003395787362086259,
+ "loss": 1.2016,
+ "theoretical_loss": 3.471548466596723,
+ "tokens_seen": 1913716736
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003395626880641926,
+ "loss": 1.3205,
+ "theoretical_loss": 3.4715385502946425,
+ "tokens_seen": 1913782272
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003395466399197593,
+ "loss": 1.2488,
+ "theoretical_loss": 3.47152863442721,
+ "tokens_seen": 1913847808
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000339530591775326,
+ "loss": 1.4029,
+ "theoretical_loss": 3.4715187189943917,
+ "tokens_seen": 1913913344
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003395145436308927,
+ "loss": 1.3087,
+ "theoretical_loss": 3.4715088039961532,
+ "tokens_seen": 1913978880
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033949849548645935,
+ "loss": 1.2972,
+ "theoretical_loss": 3.4714988894324605,
+ "tokens_seen": 1914044416
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033948244734202605,
+ "loss": 1.3646,
+ "theoretical_loss": 3.4714889753032807,
+ "tokens_seen": 1914109952
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033946639919759275,
+ "loss": 1.3149,
+ "theoretical_loss": 3.471479061608579,
+ "tokens_seen": 1914175488
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033945035105315945,
+ "loss": 1.3548,
+ "theoretical_loss": 3.4714691483483215,
+ "tokens_seen": 1914241024
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033943430290872615,
+ "loss": 1.3736,
+ "theoretical_loss": 3.4714592355224747,
+ "tokens_seen": 1914306560
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033941825476429285,
+ "loss": 1.2342,
+ "theoretical_loss": 3.4714493231310044,
+ "tokens_seen": 1914372096
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033940220661985956,
+ "loss": 1.3392,
+ "theoretical_loss": 3.4714394111738764,
+ "tokens_seen": 1914437632
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033938615847542626,
+ "loss": 1.2515,
+ "theoretical_loss": 3.471429499651057,
+ "tokens_seen": 1914503168
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033937011033099296,
+ "loss": 1.3416,
+ "theoretical_loss": 3.4714195885625134,
+ "tokens_seen": 1914568704
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033935406218655966,
+ "loss": 1.2121,
+ "theoretical_loss": 3.47140967790821,
+ "tokens_seen": 1914634240
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003393380140421264,
+ "loss": 1.3935,
+ "theoretical_loss": 3.471399767688114,
+ "tokens_seen": 1914699776
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003393219658976931,
+ "loss": 1.1634,
+ "theoretical_loss": 3.471389857902191,
+ "tokens_seen": 1914765312
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003393059177532598,
+ "loss": 1.2524,
+ "theoretical_loss": 3.471379948550407,
+ "tokens_seen": 1914830848
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003392898696088265,
+ "loss": 1.1771,
+ "theoretical_loss": 3.471370039632729,
+ "tokens_seen": 1914896384
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003392738214643932,
+ "loss": 1.3699,
+ "theoretical_loss": 3.4713601311491225,
+ "tokens_seen": 1914961920
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003392577733199599,
+ "loss": 1.3535,
+ "theoretical_loss": 3.471350223099553,
+ "tokens_seen": 1915027456
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003392417251755266,
+ "loss": 1.2448,
+ "theoretical_loss": 3.471340315483988,
+ "tokens_seen": 1915092992
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003392256770310933,
+ "loss": 1.2565,
+ "theoretical_loss": 3.471330408302393,
+ "tokens_seen": 1915158528
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033920962888666,
+ "loss": 1.2465,
+ "theoretical_loss": 3.471320501554734,
+ "tokens_seen": 1915224064
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391935807422267,
+ "loss": 1.3965,
+ "theoretical_loss": 3.4713105952409773,
+ "tokens_seen": 1915289600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391775325977934,
+ "loss": 1.3081,
+ "theoretical_loss": 3.4713006893610885,
+ "tokens_seen": 1915355136
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391614844533601,
+ "loss": 1.2785,
+ "theoretical_loss": 3.471290783915035,
+ "tokens_seen": 1915420672
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391454363089268,
+ "loss": 1.2244,
+ "theoretical_loss": 3.471280878902782,
+ "tokens_seen": 1915486208
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391293881644935,
+ "loss": 1.2579,
+ "theoretical_loss": 3.471270974324295,
+ "tokens_seen": 1915551744
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003391133400200602,
+ "loss": 1.2709,
+ "theoretical_loss": 3.471261070179542,
+ "tokens_seen": 1915617280
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390972918756269,
+ "loss": 1.3413,
+ "theoretical_loss": 3.4712511664684875,
+ "tokens_seen": 1915682816
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390812437311936,
+ "loss": 1.2658,
+ "theoretical_loss": 3.471241263191099,
+ "tokens_seen": 1915748352
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390651955867603,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4712313603473417,
+ "tokens_seen": 1915813888
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000339049147442327,
+ "loss": 1.2708,
+ "theoretical_loss": 3.4712214579371823,
+ "tokens_seen": 1915879424
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390330992978937,
+ "loss": 1.3324,
+ "theoretical_loss": 3.471211555960587,
+ "tokens_seen": 1915944960
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390170511534604,
+ "loss": 1.2945,
+ "theoretical_loss": 3.4712016544175217,
+ "tokens_seen": 1916010496
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003390010030090271,
+ "loss": 1.4641,
+ "theoretical_loss": 3.4711917533079526,
+ "tokens_seen": 1916076032
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003389849548645938,
+ "loss": 1.2335,
+ "theoretical_loss": 3.471181852631846,
+ "tokens_seen": 1916141568
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003389689067201605,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4711719523891684,
+ "tokens_seen": 1916207104
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033895285857572723,
+ "loss": 1.2513,
+ "theoretical_loss": 3.471162052579885,
+ "tokens_seen": 1916272640
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033893681043129393,
+ "loss": 1.3361,
+ "theoretical_loss": 3.4711521532039633,
+ "tokens_seen": 1916338176
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033892076228686063,
+ "loss": 1.2282,
+ "theoretical_loss": 3.471142254261369,
+ "tokens_seen": 1916403712
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033890471414242733,
+ "loss": 1.3111,
+ "theoretical_loss": 3.4711323557520677,
+ "tokens_seen": 1916469248
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033888866599799403,
+ "loss": 1.3891,
+ "theoretical_loss": 3.4711224576760262,
+ "tokens_seen": 1916534784
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003388726178535607,
+ "loss": 1.178,
+ "theoretical_loss": 3.4711125600332107,
+ "tokens_seen": 1916600320
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003388565697091274,
+ "loss": 1.3536,
+ "theoretical_loss": 3.4711026628235873,
+ "tokens_seen": 1916665856
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003388405215646941,
+ "loss": 1.349,
+ "theoretical_loss": 3.4710927660471227,
+ "tokens_seen": 1916731392
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003388244734202608,
+ "loss": 1.3321,
+ "theoretical_loss": 3.4710828697037828,
+ "tokens_seen": 1916796928
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003388084252758275,
+ "loss": 1.3324,
+ "theoretical_loss": 3.4710729737935333,
+ "tokens_seen": 1916862464
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 642590,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3644001483917236,
+ "objective/train/theoretical_loss": 3.471063078316341,
+ "objective/train/tokens_used": 1937388000,
+ "theoretical_loss": 3.471063078316341,
+ "tokens_seen": 1916928000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003387923771313942,
+ "loss": 1.2952,
+ "theoretical_loss": 3.471063078316341,
+ "tokens_seen": 1916928000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003387763289869609,
+ "loss": 1.3196,
+ "theoretical_loss": 3.4710531832721725,
+ "tokens_seen": 1916993536
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003387602808425276,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4710432886609937,
+ "tokens_seen": 1917059072
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003387442326980943,
+ "loss": 1.3273,
+ "theoretical_loss": 3.4710333944827703,
+ "tokens_seen": 1917124608
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000338728184553661,
+ "loss": 1.3059,
+ "theoretical_loss": 3.471023500737469,
+ "tokens_seen": 1917190144
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003387121364092277,
+ "loss": 1.3575,
+ "theoretical_loss": 3.471013607425056,
+ "tokens_seen": 1917255680
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386960882647944,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4710037145454984,
+ "tokens_seen": 1917321216
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386800401203611,
+ "loss": 1.1912,
+ "theoretical_loss": 3.470993822098761,
+ "tokens_seen": 1917386752
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386639919759278,
+ "loss": 1.2745,
+ "theoretical_loss": 3.4709839300848113,
+ "tokens_seen": 1917452288
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386479438314945,
+ "loss": 1.1026,
+ "theoretical_loss": 3.4709740385036145,
+ "tokens_seen": 1917517824
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386318956870612,
+ "loss": 1.3516,
+ "theoretical_loss": 3.470964147355138,
+ "tokens_seen": 1917583360
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003386158475426279,
+ "loss": 1.2604,
+ "theoretical_loss": 3.470954256639347,
+ "tokens_seen": 1917648896
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385997993981946,
+ "loss": 1.255,
+ "theoretical_loss": 3.4709443663562087,
+ "tokens_seen": 1917714432
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385837512537613,
+ "loss": 1.3005,
+ "theoretical_loss": 3.470934476505689,
+ "tokens_seen": 1917779968
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000338567703109328,
+ "loss": 1.2588,
+ "theoretical_loss": 3.470924587087754,
+ "tokens_seen": 1917845504
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385516549648947,
+ "loss": 1.197,
+ "theoretical_loss": 3.47091469810237,
+ "tokens_seen": 1917911040
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385356068204614,
+ "loss": 1.2088,
+ "theoretical_loss": 3.470904809549504,
+ "tokens_seen": 1917976576
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385195586760281,
+ "loss": 1.2335,
+ "theoretical_loss": 3.470894921429122,
+ "tokens_seen": 1918042112
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003385035105315948,
+ "loss": 1.3654,
+ "theoretical_loss": 3.4708850337411894,
+ "tokens_seen": 1918107648
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003384874623871615,
+ "loss": 1.2825,
+ "theoretical_loss": 3.4708751464856737,
+ "tokens_seen": 1918173184
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003384714142427282,
+ "loss": 1.3037,
+ "theoretical_loss": 3.4708652596625402,
+ "tokens_seen": 1918238720
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003384553660982949,
+ "loss": 1.2987,
+ "theoretical_loss": 3.470855373271756,
+ "tokens_seen": 1918304256
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003384393179538616,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4708454873132872,
+ "tokens_seen": 1918369792
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003384232698094283,
+ "loss": 1.2161,
+ "theoretical_loss": 3.4708356017871003,
+ "tokens_seen": 1918435328
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000338407221664995,
+ "loss": 1.2209,
+ "theoretical_loss": 3.4708257166931613,
+ "tokens_seen": 1918500864
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383911735205617,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4708158320314366,
+ "tokens_seen": 1918566400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383751253761284,
+ "loss": 1.3434,
+ "theoretical_loss": 3.470805947801893,
+ "tokens_seen": 1918631936
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383590772316951,
+ "loss": 1.289,
+ "theoretical_loss": 3.470796064004496,
+ "tokens_seen": 1918697472
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383430290872618,
+ "loss": 1.3471,
+ "theoretical_loss": 3.4707861806392124,
+ "tokens_seen": 1918763008
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383269809428285,
+ "loss": 1.2051,
+ "theoretical_loss": 3.470776297706009,
+ "tokens_seen": 1918828544
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003383109327983952,
+ "loss": 1.2706,
+ "theoretical_loss": 3.4707664152048516,
+ "tokens_seen": 1918894080
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003382948846539619,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4707565331357064,
+ "tokens_seen": 1918959616
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003382788365095286,
+ "loss": 1.1947,
+ "theoretical_loss": 3.47074665149854,
+ "tokens_seen": 1919025152
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003382627883650953,
+ "loss": 1.3236,
+ "theoretical_loss": 3.4707367702933194,
+ "tokens_seen": 1919090688
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000338246740220662,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4707268895200096,
+ "tokens_seen": 1919156224
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003382306920762287,
+ "loss": 1.2783,
+ "theoretical_loss": 3.470717009178578,
+ "tokens_seen": 1919221760
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003382146439317954,
+ "loss": 1.3064,
+ "theoretical_loss": 3.4707071292689906,
+ "tokens_seen": 1919287296
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381985957873621,
+ "loss": 1.3922,
+ "theoretical_loss": 3.470697249791214,
+ "tokens_seen": 1919352832
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381825476429288,
+ "loss": 1.3519,
+ "theoretical_loss": 3.470687370745215,
+ "tokens_seen": 1919418368
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381664994984955,
+ "loss": 1.3373,
+ "theoretical_loss": 3.4706774921309584,
+ "tokens_seen": 1919483904
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381504513540622,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4706676139484127,
+ "tokens_seen": 1919549440
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381344032096289,
+ "loss": 1.2923,
+ "theoretical_loss": 3.4706577361975426,
+ "tokens_seen": 1919614976
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381183550651956,
+ "loss": 1.2898,
+ "theoretical_loss": 3.470647858878315,
+ "tokens_seen": 1919680512
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003381023069207623,
+ "loss": 1.2055,
+ "theoretical_loss": 3.4706379819906967,
+ "tokens_seen": 1919746048
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000338086258776329,
+ "loss": 1.2727,
+ "theoretical_loss": 3.4706281055346544,
+ "tokens_seen": 1919811584
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003380702106318957,
+ "loss": 1.3049,
+ "theoretical_loss": 3.470618229510153,
+ "tokens_seen": 1919877120
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003380541624874624,
+ "loss": 1.301,
+ "theoretical_loss": 3.470608353917161,
+ "tokens_seen": 1919942656
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003380381143430291,
+ "loss": 1.3751,
+ "theoretical_loss": 3.470598478755643,
+ "tokens_seen": 1920008192
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003380220661985958,
+ "loss": 1.2357,
+ "theoretical_loss": 3.470588604025566,
+ "tokens_seen": 1920073728
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003380060180541625,
+ "loss": 1.3704,
+ "theoretical_loss": 3.4705787297268964,
+ "tokens_seen": 1920139264
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 643782,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1430565118789673,
+ "objective/train/theoretical_loss": 3.4705688558596015,
+ "objective/train/tokens_used": 1940664800,
+ "theoretical_loss": 3.4705688558596015,
+ "tokens_seen": 1920204800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003379899699097292,
+ "loss": 1.1265,
+ "theoretical_loss": 3.4705688558596015,
+ "tokens_seen": 1920204800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003379739217652959,
+ "loss": 1.2697,
+ "theoretical_loss": 3.470558982423647,
+ "tokens_seen": 1920270336
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003379578736208626,
+ "loss": 1.2358,
+ "theoretical_loss": 3.4705491094189984,
+ "tokens_seen": 1920335872
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003379418254764293,
+ "loss": 1.3417,
+ "theoretical_loss": 3.470539236845624,
+ "tokens_seen": 1920401408
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000337925777331996,
+ "loss": 1.3283,
+ "theoretical_loss": 3.4705293647034887,
+ "tokens_seen": 1920466944
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003379097291875627,
+ "loss": 1.2809,
+ "theoretical_loss": 3.47051949299256,
+ "tokens_seen": 1920532480
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378936810431294,
+ "loss": 1.3012,
+ "theoretical_loss": 3.470509621712804,
+ "tokens_seen": 1920598016
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378776328986961,
+ "loss": 1.2338,
+ "theoretical_loss": 3.470499750864187,
+ "tokens_seen": 1920663552
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378615847542628,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4704898804466753,
+ "tokens_seen": 1920729088
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378455366098295,
+ "loss": 1.5689,
+ "theoretical_loss": 3.470480010460236,
+ "tokens_seen": 1920794624
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378294884653962,
+ "loss": 1.3454,
+ "theoretical_loss": 3.470470140904835,
+ "tokens_seen": 1920860160
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003378134403209629,
+ "loss": 1.2351,
+ "theoretical_loss": 3.470460271780439,
+ "tokens_seen": 1920925696
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377973921765296,
+ "loss": 1.3198,
+ "theoretical_loss": 3.470450403087014,
+ "tokens_seen": 1920991232
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377813440320963,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4704405348245277,
+ "tokens_seen": 1921056768
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000337765295887663,
+ "loss": 1.3315,
+ "theoretical_loss": 3.470430666992945,
+ "tokens_seen": 1921122304
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377492477432297,
+ "loss": 1.3322,
+ "theoretical_loss": 3.4704207995922336,
+ "tokens_seen": 1921187840
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377331995987964,
+ "loss": 1.2897,
+ "theoretical_loss": 3.470410932622359,
+ "tokens_seen": 1921253376
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377171514543631,
+ "loss": 1.2736,
+ "theoretical_loss": 3.470401066083289,
+ "tokens_seen": 1921318912
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003377011033099298,
+ "loss": 1.3734,
+ "theoretical_loss": 3.4703911999749892,
+ "tokens_seen": 1921384448
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003376850551654965,
+ "loss": 1.184,
+ "theoretical_loss": 3.470381334297426,
+ "tokens_seen": 1921449984
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003376690070210632,
+ "loss": 1.31,
+ "theoretical_loss": 3.4703714690505665,
+ "tokens_seen": 1921515520
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003376529588766299,
+ "loss": 1.3105,
+ "theoretical_loss": 3.470361604234377,
+ "tokens_seen": 1921581056
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003376369107321966,
+ "loss": 1.2669,
+ "theoretical_loss": 3.470351739848823,
+ "tokens_seen": 1921646592
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003376208625877633,
+ "loss": 1.242,
+ "theoretical_loss": 3.4703418758938724,
+ "tokens_seen": 1921712128
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033760481444333,
+ "loss": 1.3229,
+ "theoretical_loss": 3.470332012369491,
+ "tokens_seen": 1921777664
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003375887662988967,
+ "loss": 1.2635,
+ "theoretical_loss": 3.470322149275646,
+ "tokens_seen": 1921843200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033757271815446343,
+ "loss": 1.2588,
+ "theoretical_loss": 3.4703122866123035,
+ "tokens_seen": 1921908736
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033755667001003013,
+ "loss": 1.3038,
+ "theoretical_loss": 3.4703024243794296,
+ "tokens_seen": 1921974272
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033754062186559683,
+ "loss": 1.2789,
+ "theoretical_loss": 3.470292562576991,
+ "tokens_seen": 1922039808
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033752457372116353,
+ "loss": 1.1687,
+ "theoretical_loss": 3.4702827012049546,
+ "tokens_seen": 1922105344
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033750852557673023,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4702728402632874,
+ "tokens_seen": 1922170880
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033749247743229693,
+ "loss": 1.2238,
+ "theoretical_loss": 3.470262979751955,
+ "tokens_seen": 1922236416
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033747642928786363,
+ "loss": 1.3842,
+ "theoretical_loss": 3.4702531196709243,
+ "tokens_seen": 1922301952
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003374603811434303,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4702432600201614,
+ "tokens_seen": 1922367488
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000337444332998997,
+ "loss": 1.2245,
+ "theoretical_loss": 3.470233400799634,
+ "tokens_seen": 1922433024
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003374282848545637,
+ "loss": 1.2089,
+ "theoretical_loss": 3.4702235420093075,
+ "tokens_seen": 1922498560
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003374122367101304,
+ "loss": 1.354,
+ "theoretical_loss": 3.470213683649149,
+ "tokens_seen": 1922564096
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373961885656971,
+ "loss": 1.2286,
+ "theoretical_loss": 3.470203825719125,
+ "tokens_seen": 1922629632
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373801404212638,
+ "loss": 1.3276,
+ "theoretical_loss": 3.4701939682192022,
+ "tokens_seen": 1922695168
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373640922768305,
+ "loss": 1.2005,
+ "theoretical_loss": 3.470184111149347,
+ "tokens_seen": 1922760704
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373480441323972,
+ "loss": 1.1655,
+ "theoretical_loss": 3.470174254509526,
+ "tokens_seen": 1922826240
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373319959879639,
+ "loss": 1.3572,
+ "theoretical_loss": 3.4701643982997057,
+ "tokens_seen": 1922891776
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003373159478435306,
+ "loss": 1.3709,
+ "theoretical_loss": 3.470154542519853,
+ "tokens_seen": 1922957312
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372998996990973,
+ "loss": 1.2019,
+ "theoretical_loss": 3.4701446871699346,
+ "tokens_seen": 1923022848
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000337283851554664,
+ "loss": 1.3661,
+ "theoretical_loss": 3.470134832249916,
+ "tokens_seen": 1923088384
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372678034102307,
+ "loss": 1.2352,
+ "theoretical_loss": 3.470124977759765,
+ "tokens_seen": 1923153920
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372517552657974,
+ "loss": 1.3082,
+ "theoretical_loss": 3.470115123699448,
+ "tokens_seen": 1923219456
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372357071213641,
+ "loss": 1.2888,
+ "theoretical_loss": 3.470105270068931,
+ "tokens_seen": 1923284992
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372196589769308,
+ "loss": 1.3399,
+ "theoretical_loss": 3.4700954168681815,
+ "tokens_seen": 1923350528
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003372036108324975,
+ "loss": 1.2767,
+ "theoretical_loss": 3.4700855640971646,
+ "tokens_seen": 1923416064
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 644931,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.426339864730835,
+ "objective/train/theoretical_loss": 3.470075711755849,
+ "objective/train/tokens_used": 1943941600,
+ "theoretical_loss": 3.470075711755849,
+ "tokens_seen": 1923481600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003371875626880642,
+ "loss": 1.2773,
+ "theoretical_loss": 3.470075711755849,
+ "tokens_seen": 1923481600
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003371715145436309,
+ "loss": 1.3313,
+ "theoretical_loss": 3.4700658598442,
+ "tokens_seen": 1923547136
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003371554663991976,
+ "loss": 1.3063,
+ "theoretical_loss": 3.4700560083621843,
+ "tokens_seen": 1923612672
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003371394182547643,
+ "loss": 1.4039,
+ "theoretical_loss": 3.4700461573097687,
+ "tokens_seen": 1923678208
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000337123370110331,
+ "loss": 1.3048,
+ "theoretical_loss": 3.4700363066869198,
+ "tokens_seen": 1923743744
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003371073219658977,
+ "loss": 1.3075,
+ "theoretical_loss": 3.470026456493604,
+ "tokens_seen": 1923809280
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370912738214644,
+ "loss": 1.277,
+ "theoretical_loss": 3.4700166067297884,
+ "tokens_seen": 1923874816
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370752256770311,
+ "loss": 1.2305,
+ "theoretical_loss": 3.4700067573954394,
+ "tokens_seen": 1923940352
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370591775325978,
+ "loss": 1.3303,
+ "theoretical_loss": 3.4699969084905242,
+ "tokens_seen": 1924005888
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370431293881645,
+ "loss": 1.263,
+ "theoretical_loss": 3.4699870600150087,
+ "tokens_seen": 1924071424
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370270812437312,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4699772119688594,
+ "tokens_seen": 1924136960
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003370110330992979,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4699673643520437,
+ "tokens_seen": 1924202496
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003369949849548646,
+ "loss": 1.3286,
+ "theoretical_loss": 3.469957517164528,
+ "tokens_seen": 1924268032
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003369789368104313,
+ "loss": 1.2718,
+ "theoretical_loss": 3.4699476704062784,
+ "tokens_seen": 1924333568
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336962888665998,
+ "loss": 1.324,
+ "theoretical_loss": 3.4699378240772623,
+ "tokens_seen": 1924399104
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003369468405215647,
+ "loss": 1.2969,
+ "theoretical_loss": 3.469927978177446,
+ "tokens_seen": 1924464640
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033693079237713145,
+ "loss": 1.4035,
+ "theoretical_loss": 3.4699181327067965,
+ "tokens_seen": 1924530176
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033691474423269815,
+ "loss": 1.2772,
+ "theoretical_loss": 3.4699082876652803,
+ "tokens_seen": 1924595712
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033689869608826485,
+ "loss": 1.329,
+ "theoretical_loss": 3.4698984430528643,
+ "tokens_seen": 1924661248
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033688264794383155,
+ "loss": 1.263,
+ "theoretical_loss": 3.4698885988695145,
+ "tokens_seen": 1924726784
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033686659979939826,
+ "loss": 1.3344,
+ "theoretical_loss": 3.4698787551151984,
+ "tokens_seen": 1924792320
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033685055165496496,
+ "loss": 1.2129,
+ "theoretical_loss": 3.4698689117898818,
+ "tokens_seen": 1924857856
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003368345035105316,
+ "loss": 1.3115,
+ "theoretical_loss": 3.4698590688935322,
+ "tokens_seen": 1924923392
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003368184553660983,
+ "loss": 1.4533,
+ "theoretical_loss": 3.469849226426116,
+ "tokens_seen": 1924988928
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336802407221665,
+ "loss": 1.2906,
+ "theoretical_loss": 3.4698393843876003,
+ "tokens_seen": 1925054464
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367863590772317,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4698295427779513,
+ "tokens_seen": 1925120000
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367703109327984,
+ "loss": 1.3543,
+ "theoretical_loss": 3.4698197015971357,
+ "tokens_seen": 1925185536
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367542627883651,
+ "loss": 1.2315,
+ "theoretical_loss": 3.46980986084512,
+ "tokens_seen": 1925251072
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367382146439318,
+ "loss": 1.2207,
+ "theoretical_loss": 3.469800020521872,
+ "tokens_seen": 1925316608
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367221664994985,
+ "loss": 1.3044,
+ "theoretical_loss": 3.4697901806273572,
+ "tokens_seen": 1925382144
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003367061183550652,
+ "loss": 1.217,
+ "theoretical_loss": 3.4697803411615435,
+ "tokens_seen": 1925447680
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003366900702106319,
+ "loss": 1.2545,
+ "theoretical_loss": 3.4697705021243963,
+ "tokens_seen": 1925513216
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003366740220661986,
+ "loss": 1.2472,
+ "theoretical_loss": 3.4697606635158835,
+ "tokens_seen": 1925578752
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003366579739217653,
+ "loss": 1.2103,
+ "theoretical_loss": 3.469750825335971,
+ "tokens_seen": 1925644288
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336641925777332,
+ "loss": 1.2106,
+ "theoretical_loss": 3.4697409875846263,
+ "tokens_seen": 1925709824
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003366258776328987,
+ "loss": 1.3245,
+ "theoretical_loss": 3.4697311502618153,
+ "tokens_seen": 1925775360
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003366098294884654,
+ "loss": 1.2573,
+ "theoretical_loss": 3.4697213133675056,
+ "tokens_seen": 1925840896
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365937813440321,
+ "loss": 1.2262,
+ "theoretical_loss": 3.4697114769016633,
+ "tokens_seen": 1925906432
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365777331995988,
+ "loss": 1.2229,
+ "theoretical_loss": 3.4697016408642556,
+ "tokens_seen": 1925971968
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365616850551655,
+ "loss": 1.1705,
+ "theoretical_loss": 3.469691805255249,
+ "tokens_seen": 1926037504
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365456369107322,
+ "loss": 1.197,
+ "theoretical_loss": 3.46968197007461,
+ "tokens_seen": 1926103040
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365295887662989,
+ "loss": 1.3286,
+ "theoretical_loss": 3.4696721353223063,
+ "tokens_seen": 1926168576
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003365135406218656,
+ "loss": 1.2913,
+ "theoretical_loss": 3.4696623009983036,
+ "tokens_seen": 1926234112
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364974924774323,
+ "loss": 1.1004,
+ "theoretical_loss": 3.469652467102569,
+ "tokens_seen": 1926299648
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336481444332999,
+ "loss": 1.3217,
+ "theoretical_loss": 3.46964263363507,
+ "tokens_seen": 1926365184
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364653961885657,
+ "loss": 1.1726,
+ "theoretical_loss": 3.4696328005957726,
+ "tokens_seen": 1926430720
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364493480441324,
+ "loss": 1.2257,
+ "theoretical_loss": 3.4696229679846438,
+ "tokens_seen": 1926496256
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364332998996991,
+ "loss": 1.2384,
+ "theoretical_loss": 3.4696131358016506,
+ "tokens_seen": 1926561792
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364172517552658,
+ "loss": 1.2646,
+ "theoretical_loss": 3.4696033040467595,
+ "tokens_seen": 1926627328
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003364012036108325,
+ "loss": 1.1931,
+ "theoretical_loss": 3.469593472719937,
+ "tokens_seen": 1926692864
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 645797,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.161024570465088,
+ "objective/train/theoretical_loss": 3.469583641821151,
+ "objective/train/tokens_used": 1947218400,
+ "theoretical_loss": 3.469583641821151,
+ "tokens_seen": 1926758400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003363851554663992,
+ "loss": 1.2581,
+ "theoretical_loss": 3.469583641821151,
+ "tokens_seen": 1926758400
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003363691073219659,
+ "loss": 1.2966,
+ "theoretical_loss": 3.469573811350367,
+ "tokens_seen": 1926823936
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003363530591775326,
+ "loss": 1.2587,
+ "theoretical_loss": 3.4695639813075525,
+ "tokens_seen": 1926889472
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003363370110330993,
+ "loss": 1.2684,
+ "theoretical_loss": 3.469554151692674,
+ "tokens_seen": 1926955008
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336320962888666,
+ "loss": 1.162,
+ "theoretical_loss": 3.4695443225056994,
+ "tokens_seen": 1927020544
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003363049147442327,
+ "loss": 1.3771,
+ "theoretical_loss": 3.469534493746594,
+ "tokens_seen": 1927086080
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003362888665997994,
+ "loss": 1.2507,
+ "theoretical_loss": 3.4695246654153253,
+ "tokens_seen": 1927151616
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003362728184553661,
+ "loss": 1.262,
+ "theoretical_loss": 3.4695148375118605,
+ "tokens_seen": 1927217152
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003362567703109328,
+ "loss": 1.1684,
+ "theoretical_loss": 3.4695050100361655,
+ "tokens_seen": 1927282688
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003362407221664995,
+ "loss": 1.166,
+ "theoretical_loss": 3.469495182988208,
+ "tokens_seen": 1927348224
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003362246740220662,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4694853563679544,
+ "tokens_seen": 1927413760
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033620862587763293,
+ "loss": 1.1911,
+ "theoretical_loss": 3.469475530175372,
+ "tokens_seen": 1927479296
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033619257773319963,
+ "loss": 1.278,
+ "theoretical_loss": 3.4694657044104273,
+ "tokens_seen": 1927544832
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033617652958876633,
+ "loss": 1.2794,
+ "theoretical_loss": 3.4694558790730867,
+ "tokens_seen": 1927610368
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033616048144433303,
+ "loss": 1.1319,
+ "theoretical_loss": 3.469446054163318,
+ "tokens_seen": 1927675904
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033614443329989973,
+ "loss": 1.2076,
+ "theoretical_loss": 3.4694362296810874,
+ "tokens_seen": 1927741440
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033612838515546643,
+ "loss": 1.3072,
+ "theoretical_loss": 3.469426405626362,
+ "tokens_seen": 1927806976
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033611233701103313,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4694165819991083,
+ "tokens_seen": 1927872512
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033609628886659983,
+ "loss": 1.2288,
+ "theoretical_loss": 3.469406758799294,
+ "tokens_seen": 1927938048
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033608024072216653,
+ "loss": 1.3003,
+ "theoretical_loss": 3.469396936026885,
+ "tokens_seen": 1928003584
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003360641925777332,
+ "loss": 1.272,
+ "theoretical_loss": 3.4693871136818486,
+ "tokens_seen": 1928069120
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003360481444332999,
+ "loss": 1.3479,
+ "theoretical_loss": 3.469377291764152,
+ "tokens_seen": 1928134656
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003360320962888666,
+ "loss": 1.2533,
+ "theoretical_loss": 3.4693674702737614,
+ "tokens_seen": 1928200192
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003360160481444333,
+ "loss": 1.3227,
+ "theoretical_loss": 3.4693576492106444,
+ "tokens_seen": 1928265728
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000336,
+ "loss": 1.4136,
+ "theoretical_loss": 3.4693478285747674,
+ "tokens_seen": 1928331264
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359839518555667,
+ "loss": 1.175,
+ "theoretical_loss": 3.4693380083660976,
+ "tokens_seen": 1928396800
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359679037111334,
+ "loss": 1.26,
+ "theoretical_loss": 3.4693281885846017,
+ "tokens_seen": 1928462336
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359518555667001,
+ "loss": 1.2742,
+ "theoretical_loss": 3.4693183692302467,
+ "tokens_seen": 1928527872
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359358074222668,
+ "loss": 1.2938,
+ "theoretical_loss": 3.4693085503029994,
+ "tokens_seen": 1928593408
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359197592778335,
+ "loss": 1.2441,
+ "theoretical_loss": 3.4692987318028266,
+ "tokens_seen": 1928658944
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003359037111334002,
+ "loss": 1.3074,
+ "theoretical_loss": 3.4692889137296956,
+ "tokens_seen": 1928724480
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003358876629889669,
+ "loss": 1.215,
+ "theoretical_loss": 3.4692790960835733,
+ "tokens_seen": 1928790016
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003358716148445336,
+ "loss": 1.1787,
+ "theoretical_loss": 3.469269278864426,
+ "tokens_seen": 1928855552
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003358555667001003,
+ "loss": 1.3828,
+ "theoretical_loss": 3.4692594620722215,
+ "tokens_seen": 1928921088
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.000335839518555667,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4692496457069257,
+ "tokens_seen": 1928986624
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003358234704112337,
+ "loss": 1.2942,
+ "theoretical_loss": 3.4692398297685063,
+ "tokens_seen": 1929052160
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003358074222668004,
+ "loss": 1.2572,
+ "theoretical_loss": 3.4692300142569303,
+ "tokens_seen": 1929117696
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003357913741223671,
+ "loss": 1.2578,
+ "theoretical_loss": 3.4692201991721645,
+ "tokens_seen": 1929183232
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003357753259779338,
+ "loss": 1.3917,
+ "theoretical_loss": 3.4692103845141755,
+ "tokens_seen": 1929248768
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003357592778335005,
+ "loss": 1.2551,
+ "theoretical_loss": 3.46920057028293,
+ "tokens_seen": 1929314304
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.0003357432296890672,
+ "loss": 1.2612,
+ "theoretical_loss": 3.4691907564783966,
+ "tokens_seen": 1929379840
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033572718154463395,
+ "loss": 1.2458,
+ "theoretical_loss": 3.46918094310054,
+ "tokens_seen": 1929445376
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033571113340020065,
+ "loss": 1.3331,
+ "theoretical_loss": 3.469171130149329,
+ "tokens_seen": 1929510912
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033569508525576735,
+ "loss": 1.1819,
+ "theoretical_loss": 3.469161317624729,
+ "tokens_seen": 1929576448
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033567903711133405,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4691515055267086,
+ "tokens_seen": 1929641984
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033566298896690075,
+ "loss": 1.2165,
+ "theoretical_loss": 3.4691416938552333,
+ "tokens_seen": 1929707520
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033564694082246745,
+ "loss": 1.2814,
+ "theoretical_loss": 3.469131882610271,
+ "tokens_seen": 1929773056
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033563089267803415,
+ "loss": 1.2307,
+ "theoretical_loss": 3.469122071791788,
+ "tokens_seen": 1929838592
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033561484453360085,
+ "loss": 1.196,
+ "theoretical_loss": 3.469112261399752,
+ "tokens_seen": 1929904128
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033559879638916755,
+ "loss": 1.253,
+ "theoretical_loss": 3.4691024514341295,
+ "tokens_seen": 1929969664
+ },
+ {
+ "epoch": 0.58,
+ "objective/train/docs_used": 646757,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2885311841964722,
+ "objective/train/theoretical_loss": 3.4690926418948878,
+ "objective/train/tokens_used": 1950495200,
+ "theoretical_loss": 3.4690926418948878,
+ "tokens_seen": 1930035200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033558274824473425,
+ "loss": 1.2264,
+ "theoretical_loss": 3.4690926418948878,
+ "tokens_seen": 1930035200
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033556670010030095,
+ "loss": 1.2388,
+ "theoretical_loss": 3.469082832781994,
+ "tokens_seen": 1930100736
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033555065195586765,
+ "loss": 1.3518,
+ "theoretical_loss": 3.4690730240954144,
+ "tokens_seen": 1930166272
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033553460381143435,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4690632158351162,
+ "tokens_seen": 1930231808
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033551855566700105,
+ "loss": 1.2367,
+ "theoretical_loss": 3.469053408001067,
+ "tokens_seen": 1930297344
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033550250752256776,
+ "loss": 1.329,
+ "theoretical_loss": 3.469043600593233,
+ "tokens_seen": 1930362880
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033548645937813446,
+ "loss": 1.2201,
+ "theoretical_loss": 3.4690337936115823,
+ "tokens_seen": 1930428416
+ },
+ {
+ "epoch": 0.58,
+ "learning_rate": 0.00033547041123370116,
+ "loss": 1.3149,
+ "theoretical_loss": 3.4690239870560804,
+ "tokens_seen": 1930493952
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033545436308926786,
+ "loss": 1.3232,
+ "theoretical_loss": 3.4690141809266954,
+ "tokens_seen": 1930559488
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033543831494483456,
+ "loss": 1.2863,
+ "theoretical_loss": 3.4690043752233946,
+ "tokens_seen": 1930625024
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003354222668004012,
+ "loss": 1.3294,
+ "theoretical_loss": 3.468994569946144,
+ "tokens_seen": 1930690560
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003354062186559679,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4689847650949117,
+ "tokens_seen": 1930756096
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003353901705115346,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4689749606696636,
+ "tokens_seen": 1930821632
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003353741223671013,
+ "loss": 1.3493,
+ "theoretical_loss": 3.4689651566703676,
+ "tokens_seen": 1930887168
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000335358074222668,
+ "loss": 1.2221,
+ "theoretical_loss": 3.46895535309699,
+ "tokens_seen": 1930952704
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003353420260782347,
+ "loss": 1.2583,
+ "theoretical_loss": 3.468945549949499,
+ "tokens_seen": 1931018240
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003353259779338014,
+ "loss": 1.2836,
+ "theoretical_loss": 3.46893574722786,
+ "tokens_seen": 1931083776
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003353099297893681,
+ "loss": 1.2082,
+ "theoretical_loss": 3.4689259449320415,
+ "tokens_seen": 1931149312
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352938816449348,
+ "loss": 1.245,
+ "theoretical_loss": 3.4689161430620103,
+ "tokens_seen": 1931214848
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352778335005015,
+ "loss": 1.2653,
+ "theoretical_loss": 3.468906341617733,
+ "tokens_seen": 1931280384
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352617853560682,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4688965405991765,
+ "tokens_seen": 1931345920
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352457372116349,
+ "loss": 1.1898,
+ "theoretical_loss": 3.468886740006308,
+ "tokens_seen": 1931411456
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352296890672016,
+ "loss": 1.1997,
+ "theoretical_loss": 3.4688769398390953,
+ "tokens_seen": 1931476992
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003352136409227683,
+ "loss": 1.1747,
+ "theoretical_loss": 3.468867140097505,
+ "tokens_seen": 1931542528
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000335197592778335,
+ "loss": 1.2937,
+ "theoretical_loss": 3.468857340781504,
+ "tokens_seen": 1931608064
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351815446339017,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4688475418910594,
+ "tokens_seen": 1931673600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351654964894684,
+ "loss": 1.3073,
+ "theoretical_loss": 3.468837743426138,
+ "tokens_seen": 1931739136
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351494483450351,
+ "loss": 1.3026,
+ "theoretical_loss": 3.4688279453867077,
+ "tokens_seen": 1931804672
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351334002006018,
+ "loss": 1.2804,
+ "theoretical_loss": 3.468818147772735,
+ "tokens_seen": 1931870208
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351173520561685,
+ "loss": 1.1748,
+ "theoretical_loss": 3.4688083505841876,
+ "tokens_seen": 1931935744
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003351013039117352,
+ "loss": 1.3543,
+ "theoretical_loss": 3.4687985538210317,
+ "tokens_seen": 1932001280
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003350852557673019,
+ "loss": 1.1887,
+ "theoretical_loss": 3.4687887574832352,
+ "tokens_seen": 1932066816
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003350692076228686,
+ "loss": 1.3231,
+ "theoretical_loss": 3.4687789615707647,
+ "tokens_seen": 1932132352
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003350531594784353,
+ "loss": 1.1978,
+ "theoretical_loss": 3.468769166083587,
+ "tokens_seen": 1932197888
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000335037111334002,
+ "loss": 1.3247,
+ "theoretical_loss": 3.4687593710216698,
+ "tokens_seen": 1932263424
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003350210631895687,
+ "loss": 1.4017,
+ "theoretical_loss": 3.4687495763849805,
+ "tokens_seen": 1932328960
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003350050150451354,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4687397821734853,
+ "tokens_seen": 1932394496
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349889669007021,
+ "loss": 1.2721,
+ "theoretical_loss": 3.4687299883871523,
+ "tokens_seen": 1932460032
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349729187562688,
+ "loss": 1.2949,
+ "theoretical_loss": 3.468720195025948,
+ "tokens_seen": 1932525568
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349568706118355,
+ "loss": 1.3218,
+ "theoretical_loss": 3.46871040208984,
+ "tokens_seen": 1932591104
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349408224674022,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4687006095787947,
+ "tokens_seen": 1932656640
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349247743229689,
+ "loss": 1.1875,
+ "theoretical_loss": 3.4686908174927797,
+ "tokens_seen": 1932722176
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003349087261785356,
+ "loss": 1.3231,
+ "theoretical_loss": 3.4686810258317617,
+ "tokens_seen": 1932787712
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003348926780341023,
+ "loss": 1.3218,
+ "theoretical_loss": 3.4686712345957087,
+ "tokens_seen": 1932853248
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000334876629889669,
+ "loss": 1.2315,
+ "theoretical_loss": 3.4686614437845877,
+ "tokens_seen": 1932918784
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003348605817452357,
+ "loss": 1.1819,
+ "theoretical_loss": 3.468651653398365,
+ "tokens_seen": 1932984320
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003348445336008024,
+ "loss": 1.2567,
+ "theoretical_loss": 3.468641863437009,
+ "tokens_seen": 1933049856
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033482848545636913,
+ "loss": 1.3268,
+ "theoretical_loss": 3.4686320739004852,
+ "tokens_seen": 1933115392
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033481243731193583,
+ "loss": 1.409,
+ "theoretical_loss": 3.4686222847887622,
+ "tokens_seen": 1933180928
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033479638916750253,
+ "loss": 1.2153,
+ "theoretical_loss": 3.4686124961018066,
+ "tokens_seen": 1933246464
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 647912,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2195017337799072,
+ "objective/train/theoretical_loss": 3.468602707839586,
+ "objective/train/tokens_used": 1953772000,
+ "theoretical_loss": 3.468602707839586,
+ "tokens_seen": 1933312000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033478034102306923,
+ "loss": 1.2709,
+ "theoretical_loss": 3.468602707839586,
+ "tokens_seen": 1933312000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033476429287863593,
+ "loss": 1.1965,
+ "theoretical_loss": 3.468592920002066,
+ "tokens_seen": 1933377536
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033474824473420263,
+ "loss": 1.2993,
+ "theoretical_loss": 3.4685831325892162,
+ "tokens_seen": 1933443072
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033473219658976933,
+ "loss": 1.3346,
+ "theoretical_loss": 3.468573345601002,
+ "tokens_seen": 1933508608
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033471614844533603,
+ "loss": 1.2599,
+ "theoretical_loss": 3.468563559037392,
+ "tokens_seen": 1933574144
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033470010030090273,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4685537728983515,
+ "tokens_seen": 1933639680
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033468405215646943,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4685439871838497,
+ "tokens_seen": 1933705216
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033466800401203613,
+ "loss": 1.2487,
+ "theoretical_loss": 3.4685342018938523,
+ "tokens_seen": 1933770752
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003346519558676028,
+ "loss": 1.2827,
+ "theoretical_loss": 3.468524417028327,
+ "tokens_seen": 1933836288
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003346359077231695,
+ "loss": 1.305,
+ "theoretical_loss": 3.468514632587241,
+ "tokens_seen": 1933901824
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003346198595787362,
+ "loss": 1.3369,
+ "theoretical_loss": 3.468504848570562,
+ "tokens_seen": 1933967360
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003346038114343029,
+ "loss": 1.3156,
+ "theoretical_loss": 3.4684950649782564,
+ "tokens_seen": 1934032896
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003345877632898696,
+ "loss": 1.202,
+ "theoretical_loss": 3.468485281810292,
+ "tokens_seen": 1934098432
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003345717151454363,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4684754990666358,
+ "tokens_seen": 1934163968
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000334555667001003,
+ "loss": 1.2435,
+ "theoretical_loss": 3.4684657167472546,
+ "tokens_seen": 1934229504
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003345396188565697,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4684559348521162,
+ "tokens_seen": 1934295040
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033452357071213644,
+ "loss": 1.2799,
+ "theoretical_loss": 3.468446153381188,
+ "tokens_seen": 1934360576
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033450752256770314,
+ "loss": 1.3757,
+ "theoretical_loss": 3.4684363723344367,
+ "tokens_seen": 1934426112
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033449147442326984,
+ "loss": 1.2435,
+ "theoretical_loss": 3.4684265917118298,
+ "tokens_seen": 1934491648
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033447542627883654,
+ "loss": 1.32,
+ "theoretical_loss": 3.4684168115133343,
+ "tokens_seen": 1934557184
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033445937813440324,
+ "loss": 1.1397,
+ "theoretical_loss": 3.4684070317389177,
+ "tokens_seen": 1934622720
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033444332998996994,
+ "loss": 1.1978,
+ "theoretical_loss": 3.468397252388547,
+ "tokens_seen": 1934688256
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033442728184553664,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4683874734621902,
+ "tokens_seen": 1934753792
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033441123370110334,
+ "loss": 1.2552,
+ "theoretical_loss": 3.4683776949598135,
+ "tokens_seen": 1934819328
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033439518555667004,
+ "loss": 1.3097,
+ "theoretical_loss": 3.468367916881385,
+ "tokens_seen": 1934884864
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033437913741223675,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4683581392268708,
+ "tokens_seen": 1934950400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033436308926780345,
+ "loss": 1.2063,
+ "theoretical_loss": 3.46834836199624,
+ "tokens_seen": 1935015936
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033434704112337015,
+ "loss": 1.3497,
+ "theoretical_loss": 3.468338585189458,
+ "tokens_seen": 1935081472
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033433099297893685,
+ "loss": 1.3699,
+ "theoretical_loss": 3.468328808806493,
+ "tokens_seen": 1935147008
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033431494483450355,
+ "loss": 1.3364,
+ "theoretical_loss": 3.4683190328473126,
+ "tokens_seen": 1935212544
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033429889669007025,
+ "loss": 1.2013,
+ "theoretical_loss": 3.468309257311883,
+ "tokens_seen": 1935278080
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033428284854563695,
+ "loss": 1.4048,
+ "theoretical_loss": 3.4682994822001727,
+ "tokens_seen": 1935343616
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033426680040120365,
+ "loss": 1.2371,
+ "theoretical_loss": 3.468289707512148,
+ "tokens_seen": 1935409152
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033425075225677035,
+ "loss": 1.2907,
+ "theoretical_loss": 3.4682799332477767,
+ "tokens_seen": 1935474688
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033423470411233705,
+ "loss": 1.4143,
+ "theoretical_loss": 3.4682701594070267,
+ "tokens_seen": 1935540224
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033421865596790375,
+ "loss": 1.2476,
+ "theoretical_loss": 3.468260385989864,
+ "tokens_seen": 1935605760
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033420260782347045,
+ "loss": 1.2119,
+ "theoretical_loss": 3.4682506129962563,
+ "tokens_seen": 1935671296
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033418655967903715,
+ "loss": 1.3302,
+ "theoretical_loss": 3.468240840426171,
+ "tokens_seen": 1935736832
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033417051153460385,
+ "loss": 1.2272,
+ "theoretical_loss": 3.468231068279576,
+ "tokens_seen": 1935802368
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033415446339017055,
+ "loss": 1.2881,
+ "theoretical_loss": 3.4682212965564383,
+ "tokens_seen": 1935867904
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033413841524573726,
+ "loss": 1.4079,
+ "theoretical_loss": 3.4682115252567245,
+ "tokens_seen": 1935933440
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033412236710130396,
+ "loss": 1.282,
+ "theoretical_loss": 3.468201754380403,
+ "tokens_seen": 1935998976
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033410631895687066,
+ "loss": 1.2786,
+ "theoretical_loss": 3.46819198392744,
+ "tokens_seen": 1936064512
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033409027081243736,
+ "loss": 1.3104,
+ "theoretical_loss": 3.468182213897804,
+ "tokens_seen": 1936130048
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033407422266800406,
+ "loss": 1.2637,
+ "theoretical_loss": 3.4681724442914614,
+ "tokens_seen": 1936195584
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033405817452357076,
+ "loss": 1.2341,
+ "theoretical_loss": 3.46816267510838,
+ "tokens_seen": 1936261120
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033404212637913746,
+ "loss": 1.3201,
+ "theoretical_loss": 3.468152906348527,
+ "tokens_seen": 1936326656
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003340260782347041,
+ "loss": 1.2412,
+ "theoretical_loss": 3.46814313801187,
+ "tokens_seen": 1936392192
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003340100300902708,
+ "loss": 1.2872,
+ "theoretical_loss": 3.468133370098375,
+ "tokens_seen": 1936457728
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003339939819458375,
+ "loss": 1.3168,
+ "theoretical_loss": 3.468123602608012,
+ "tokens_seen": 1936523264
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 649105,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.228218913078308,
+ "objective/train/theoretical_loss": 3.4681138355407457,
+ "objective/train/tokens_used": 1957048800,
+ "theoretical_loss": 3.4681138355407457,
+ "tokens_seen": 1936588800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003339779338014042,
+ "loss": 1.2015,
+ "theoretical_loss": 3.4681138355407457,
+ "tokens_seen": 1936588800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003339618856569709,
+ "loss": 1.3447,
+ "theoretical_loss": 3.4681040688965448,
+ "tokens_seen": 1936654336
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003339458375125376,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4680943026753766,
+ "tokens_seen": 1936719872
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003339297893681043,
+ "loss": 1.231,
+ "theoretical_loss": 3.4680845368772077,
+ "tokens_seen": 1936785408
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000333913741223671,
+ "loss": 1.3998,
+ "theoretical_loss": 3.468074771502007,
+ "tokens_seen": 1936850944
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338976930792377,
+ "loss": 1.2819,
+ "theoretical_loss": 3.4680650065497405,
+ "tokens_seen": 1936916480
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338816449348044,
+ "loss": 1.3175,
+ "theoretical_loss": 3.468055242020376,
+ "tokens_seen": 1936982016
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338655967903711,
+ "loss": 1.3159,
+ "theoretical_loss": 3.468045477913881,
+ "tokens_seen": 1937047552
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338495486459378,
+ "loss": 1.3315,
+ "theoretical_loss": 3.468035714230222,
+ "tokens_seen": 1937113088
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338335005015045,
+ "loss": 1.3219,
+ "theoretical_loss": 3.468025950969368,
+ "tokens_seen": 1937178624
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338174523570712,
+ "loss": 1.262,
+ "theoretical_loss": 3.468016188131285,
+ "tokens_seen": 1937244160
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003338014042126379,
+ "loss": 1.1846,
+ "theoretical_loss": 3.468006425715941,
+ "tokens_seen": 1937309696
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003337853560682046,
+ "loss": 1.1698,
+ "theoretical_loss": 3.4679966637233033,
+ "tokens_seen": 1937375232
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003337693079237713,
+ "loss": 1.2867,
+ "theoretical_loss": 3.4679869021533394,
+ "tokens_seen": 1937440768
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000333753259779338,
+ "loss": 1.1951,
+ "theoretical_loss": 3.467977141006017,
+ "tokens_seen": 1937506304
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003337372116349047,
+ "loss": 1.2754,
+ "theoretical_loss": 3.4679673802813022,
+ "tokens_seen": 1937571840
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003337211634904714,
+ "loss": 1.2846,
+ "theoretical_loss": 3.467957619979164,
+ "tokens_seen": 1937637376
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003337051153460381,
+ "loss": 1.3171,
+ "theoretical_loss": 3.467947860099569,
+ "tokens_seen": 1937702912
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336890672016048,
+ "loss": 1.2839,
+ "theoretical_loss": 3.4679381006424848,
+ "tokens_seen": 1937768448
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336730190571715,
+ "loss": 1.3033,
+ "theoretical_loss": 3.467928341607879,
+ "tokens_seen": 1937833984
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336569709127382,
+ "loss": 1.3823,
+ "theoretical_loss": 3.4679185829957184,
+ "tokens_seen": 1937899520
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336409227683049,
+ "loss": 1.1818,
+ "theoretical_loss": 3.4679088248059706,
+ "tokens_seen": 1937965056
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336248746238716,
+ "loss": 1.2439,
+ "theoretical_loss": 3.467899067038604,
+ "tokens_seen": 1938030592
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003336088264794383,
+ "loss": 1.2207,
+ "theoretical_loss": 3.467889309693585,
+ "tokens_seen": 1938096128
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000333592778335005,
+ "loss": 1.2102,
+ "theoretical_loss": 3.467879552770881,
+ "tokens_seen": 1938161664
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003335767301905717,
+ "loss": 1.3421,
+ "theoretical_loss": 3.46786979627046,
+ "tokens_seen": 1938227200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003335606820461384,
+ "loss": 1.1983,
+ "theoretical_loss": 3.4678600401922894,
+ "tokens_seen": 1938292736
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003335446339017051,
+ "loss": 1.2339,
+ "theoretical_loss": 3.467850284536336,
+ "tokens_seen": 1938358272
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003335285857572718,
+ "loss": 1.2118,
+ "theoretical_loss": 3.4678405293025683,
+ "tokens_seen": 1938423808
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003335125376128385,
+ "loss": 1.2339,
+ "theoretical_loss": 3.467830774490953,
+ "tokens_seen": 1938489344
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003334964894684052,
+ "loss": 1.1343,
+ "theoretical_loss": 3.467821020101457,
+ "tokens_seen": 1938554880
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003334804413239719,
+ "loss": 1.2675,
+ "theoretical_loss": 3.4678112661340497,
+ "tokens_seen": 1938620416
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033346439317953863,
+ "loss": 1.1051,
+ "theoretical_loss": 3.4678015125886965,
+ "tokens_seen": 1938685952
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033344834503510533,
+ "loss": 1.2451,
+ "theoretical_loss": 3.467791759465366,
+ "tokens_seen": 1938751488
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033343229689067203,
+ "loss": 1.3313,
+ "theoretical_loss": 3.467782006764026,
+ "tokens_seen": 1938817024
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033341624874623873,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4677722544846428,
+ "tokens_seen": 1938882560
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033340020060180543,
+ "loss": 1.276,
+ "theoretical_loss": 3.4677625026271848,
+ "tokens_seen": 1938948096
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033338415245737213,
+ "loss": 1.232,
+ "theoretical_loss": 3.467752751191619,
+ "tokens_seen": 1939013632
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033336810431293883,
+ "loss": 1.4059,
+ "theoretical_loss": 3.4677430001779124,
+ "tokens_seen": 1939079168
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033335205616850553,
+ "loss": 1.1998,
+ "theoretical_loss": 3.467733249586034,
+ "tokens_seen": 1939144704
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033333600802407223,
+ "loss": 1.251,
+ "theoretical_loss": 3.4677234994159503,
+ "tokens_seen": 1939210240
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033331995987963893,
+ "loss": 1.2957,
+ "theoretical_loss": 3.467713749667629,
+ "tokens_seen": 1939275776
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033330391173520563,
+ "loss": 1.336,
+ "theoretical_loss": 3.467704000341037,
+ "tokens_seen": 1939341312
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033328786359077233,
+ "loss": 1.3312,
+ "theoretical_loss": 3.467694251436143,
+ "tokens_seen": 1939406848
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033327181544633904,
+ "loss": 1.4445,
+ "theoretical_loss": 3.4676845029529133,
+ "tokens_seen": 1939472384
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033325576730190574,
+ "loss": 1.3131,
+ "theoretical_loss": 3.4676747548913163,
+ "tokens_seen": 1939537920
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033323971915747244,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4676650072513193,
+ "tokens_seen": 1939603456
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033322367101303914,
+ "loss": 1.3096,
+ "theoretical_loss": 3.46765526003289,
+ "tokens_seen": 1939668992
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033320762286860584,
+ "loss": 1.181,
+ "theoretical_loss": 3.467645513235995,
+ "tokens_seen": 1939734528
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033319157472417254,
+ "loss": 1.305,
+ "theoretical_loss": 3.4676357668606026,
+ "tokens_seen": 1939800064
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 649964,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0895507335662842,
+ "objective/train/theoretical_loss": 3.4676260209066805,
+ "objective/train/tokens_used": 1960325600,
+ "theoretical_loss": 3.4676260209066805,
+ "tokens_seen": 1939865600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033317552657973924,
+ "loss": 1.1975,
+ "theoretical_loss": 3.4676260209066805,
+ "tokens_seen": 1939865600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033315947843530594,
+ "loss": 1.2098,
+ "theoretical_loss": 3.467616275374196,
+ "tokens_seen": 1939931136
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033314343029087264,
+ "loss": 1.3241,
+ "theoretical_loss": 3.467606530263116,
+ "tokens_seen": 1939996672
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033312738214643934,
+ "loss": 1.2988,
+ "theoretical_loss": 3.4675967855734093,
+ "tokens_seen": 1940062208
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033311133400200604,
+ "loss": 1.2053,
+ "theoretical_loss": 3.467587041305042,
+ "tokens_seen": 1940127744
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033309528585757274,
+ "loss": 1.3138,
+ "theoretical_loss": 3.467577297457983,
+ "tokens_seen": 1940193280
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033307923771313944,
+ "loss": 1.2471,
+ "theoretical_loss": 3.4675675540321995,
+ "tokens_seen": 1940258816
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033306318956870614,
+ "loss": 1.3984,
+ "theoretical_loss": 3.4675578110276586,
+ "tokens_seen": 1940324352
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033304714142427284,
+ "loss": 1.2049,
+ "theoretical_loss": 3.4675480684443274,
+ "tokens_seen": 1940389888
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033303109327983954,
+ "loss": 1.2644,
+ "theoretical_loss": 3.467538326282175,
+ "tokens_seen": 1940455424
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033301504513540625,
+ "loss": 1.2876,
+ "theoretical_loss": 3.467528584541168,
+ "tokens_seen": 1940520960
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033299899699097295,
+ "loss": 1.284,
+ "theoretical_loss": 3.467518843221274,
+ "tokens_seen": 1940586496
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033298294884653965,
+ "loss": 1.0794,
+ "theoretical_loss": 3.4675091023224605,
+ "tokens_seen": 1940652032
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033296690070210635,
+ "loss": 1.1738,
+ "theoretical_loss": 3.4674993618446956,
+ "tokens_seen": 1940717568
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033295085255767305,
+ "loss": 1.2098,
+ "theoretical_loss": 3.467489621787946,
+ "tokens_seen": 1940783104
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033293480441323975,
+ "loss": 1.2535,
+ "theoretical_loss": 3.4674798821521806,
+ "tokens_seen": 1940848640
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033291875626880645,
+ "loss": 1.3445,
+ "theoretical_loss": 3.4674701429373656,
+ "tokens_seen": 1940914176
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033290270812437315,
+ "loss": 1.2581,
+ "theoretical_loss": 3.46746040414347,
+ "tokens_seen": 1940979712
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033288665997993985,
+ "loss": 1.3415,
+ "theoretical_loss": 3.4674506657704596,
+ "tokens_seen": 1941045248
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033287061183550655,
+ "loss": 1.2149,
+ "theoretical_loss": 3.4674409278183034,
+ "tokens_seen": 1941110784
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033285456369107325,
+ "loss": 1.3077,
+ "theoretical_loss": 3.467431190286969,
+ "tokens_seen": 1941176320
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033283851554663995,
+ "loss": 1.2136,
+ "theoretical_loss": 3.4674214531764234,
+ "tokens_seen": 1941241856
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033282246740220665,
+ "loss": 1.1988,
+ "theoretical_loss": 3.467411716486634,
+ "tokens_seen": 1941307392
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033280641925777335,
+ "loss": 1.2851,
+ "theoretical_loss": 3.467401980217569,
+ "tokens_seen": 1941372928
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033279037111334005,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4673922443691962,
+ "tokens_seen": 1941438464
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033277432296890675,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4673825089414825,
+ "tokens_seen": 1941504000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033275827482447346,
+ "loss": 1.1643,
+ "theoretical_loss": 3.4673727739343962,
+ "tokens_seen": 1941569536
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033274222668004016,
+ "loss": 1.2729,
+ "theoretical_loss": 3.4673630393479047,
+ "tokens_seen": 1941635072
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033272617853560686,
+ "loss": 1.2782,
+ "theoretical_loss": 3.4673533051819754,
+ "tokens_seen": 1941700608
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033271013039117356,
+ "loss": 1.1719,
+ "theoretical_loss": 3.467343571436576,
+ "tokens_seen": 1941766144
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033269408224674026,
+ "loss": 1.194,
+ "theoretical_loss": 3.4673338381116743,
+ "tokens_seen": 1941831680
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033267803410230696,
+ "loss": 1.3513,
+ "theoretical_loss": 3.467324105207238,
+ "tokens_seen": 1941897216
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033266198595787366,
+ "loss": 1.2258,
+ "theoretical_loss": 3.467314372723235,
+ "tokens_seen": 1941962752
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033264593781344036,
+ "loss": 1.3266,
+ "theoretical_loss": 3.4673046406596324,
+ "tokens_seen": 1942028288
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000332629889669007,
+ "loss": 1.1875,
+ "theoretical_loss": 3.467294909016398,
+ "tokens_seen": 1942093824
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003326138415245737,
+ "loss": 1.195,
+ "theoretical_loss": 3.4672851777934994,
+ "tokens_seen": 1942159360
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325977933801404,
+ "loss": 1.3459,
+ "theoretical_loss": 3.467275446990904,
+ "tokens_seen": 1942224896
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325817452357071,
+ "loss": 1.2465,
+ "theoretical_loss": 3.4672657166085807,
+ "tokens_seen": 1942290432
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325656970912738,
+ "loss": 1.254,
+ "theoretical_loss": 3.467255986646496,
+ "tokens_seen": 1942355968
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325496489468405,
+ "loss": 1.2533,
+ "theoretical_loss": 3.467246257104617,
+ "tokens_seen": 1942421504
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325336008024072,
+ "loss": 1.3038,
+ "theoretical_loss": 3.467236527982913,
+ "tokens_seen": 1942487040
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325175526579739,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4672267992813515,
+ "tokens_seen": 1942552576
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003325015045135406,
+ "loss": 1.3385,
+ "theoretical_loss": 3.4672170709998986,
+ "tokens_seen": 1942618112
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003324854563691073,
+ "loss": 1.1908,
+ "theoretical_loss": 3.467207343138523,
+ "tokens_seen": 1942683648
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000332469408224674,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4671976156971933,
+ "tokens_seen": 1942749184
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003324533600802407,
+ "loss": 1.289,
+ "theoretical_loss": 3.467187888675876,
+ "tokens_seen": 1942814720
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003324373119358074,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4671781620745383,
+ "tokens_seen": 1942880256
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003324212637913741,
+ "loss": 1.1641,
+ "theoretical_loss": 3.4671684358931496,
+ "tokens_seen": 1942945792
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003324052156469408,
+ "loss": 1.2558,
+ "theoretical_loss": 3.4671587101316765,
+ "tokens_seen": 1943011328
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003323891675025075,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4671489847900867,
+ "tokens_seen": 1943076864
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 651270,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2893718481063843,
+ "objective/train/theoretical_loss": 3.4671392598683477,
+ "objective/train/tokens_used": 1963602400,
+ "theoretical_loss": 3.4671392598683477,
+ "tokens_seen": 1943142400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003323731193580742,
+ "loss": 1.2388,
+ "theoretical_loss": 3.4671392598683477,
+ "tokens_seen": 1943142400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003323570712136409,
+ "loss": 1.2862,
+ "theoretical_loss": 3.4671295353664284,
+ "tokens_seen": 1943207936
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003323410230692076,
+ "loss": 1.2576,
+ "theoretical_loss": 3.467119811284295,
+ "tokens_seen": 1943273472
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003323249749247743,
+ "loss": 1.2646,
+ "theoretical_loss": 3.4671100876219167,
+ "tokens_seen": 1943339008
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000332308926780341,
+ "loss": 1.3252,
+ "theoretical_loss": 3.4671003643792604,
+ "tokens_seen": 1943404544
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322928786359077,
+ "loss": 1.1911,
+ "theoretical_loss": 3.4670906415562937,
+ "tokens_seen": 1943470080
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322768304914744,
+ "loss": 1.2,
+ "theoretical_loss": 3.4670809191529846,
+ "tokens_seen": 1943535616
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322607823470411,
+ "loss": 1.3449,
+ "theoretical_loss": 3.4670711971693007,
+ "tokens_seen": 1943601152
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322447342026078,
+ "loss": 1.4192,
+ "theoretical_loss": 3.46706147560521,
+ "tokens_seen": 1943666688
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322286860581745,
+ "loss": 1.2902,
+ "theoretical_loss": 3.46705175446068,
+ "tokens_seen": 1943732224
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003322126379137412,
+ "loss": 1.2336,
+ "theoretical_loss": 3.4670420337356784,
+ "tokens_seen": 1943797760
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321965897693079,
+ "loss": 1.2207,
+ "theoretical_loss": 3.467032313430173,
+ "tokens_seen": 1943863296
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321805416248746,
+ "loss": 1.2598,
+ "theoretical_loss": 3.467022593544132,
+ "tokens_seen": 1943928832
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321644934804413,
+ "loss": 1.2448,
+ "theoretical_loss": 3.467012874077523,
+ "tokens_seen": 1943994368
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000332148445336008,
+ "loss": 1.2101,
+ "theoretical_loss": 3.4670031550303126,
+ "tokens_seen": 1944059904
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321323971915747,
+ "loss": 1.2665,
+ "theoretical_loss": 3.46699343640247,
+ "tokens_seen": 1944125440
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321163490471415,
+ "loss": 1.3449,
+ "theoretical_loss": 3.466983718193963,
+ "tokens_seen": 1944190976
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003321003009027082,
+ "loss": 1.2873,
+ "theoretical_loss": 3.4669740004047584,
+ "tokens_seen": 1944256512
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003320842527582749,
+ "loss": 1.3207,
+ "theoretical_loss": 3.466964283034824,
+ "tokens_seen": 1944322048
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003320682046138416,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4669545660841283,
+ "tokens_seen": 1944387584
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003320521564694083,
+ "loss": 1.2736,
+ "theoretical_loss": 3.466944849552639,
+ "tokens_seen": 1944453120
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000332036108324975,
+ "loss": 1.24,
+ "theoretical_loss": 3.466935133440324,
+ "tokens_seen": 1944518656
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003320200601805417,
+ "loss": 1.302,
+ "theoretical_loss": 3.4669254177471505,
+ "tokens_seen": 1944584192
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003320040120361084,
+ "loss": 1.2728,
+ "theoretical_loss": 3.466915702473086,
+ "tokens_seen": 1944649728
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033198796389167503,
+ "loss": 1.3289,
+ "theoretical_loss": 3.4669059876181,
+ "tokens_seen": 1944715264
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033197191574724173,
+ "loss": 1.2667,
+ "theoretical_loss": 3.466896273182158,
+ "tokens_seen": 1944780800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033195586760280843,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4668865591652294,
+ "tokens_seen": 1944846336
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033193981945837513,
+ "loss": 1.3277,
+ "theoretical_loss": 3.466876845567282,
+ "tokens_seen": 1944911872
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033192377131394183,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4668671323882823,
+ "tokens_seen": 1944977408
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033190772316950853,
+ "loss": 1.3804,
+ "theoretical_loss": 3.4668574196282,
+ "tokens_seen": 1945042944
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033189167502507524,
+ "loss": 1.353,
+ "theoretical_loss": 3.466847707287001,
+ "tokens_seen": 1945108480
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033187562688064194,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4668379953646546,
+ "tokens_seen": 1945174016
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033185957873620864,
+ "loss": 1.1862,
+ "theoretical_loss": 3.466828283861128,
+ "tokens_seen": 1945239552
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033184353059177534,
+ "loss": 1.2988,
+ "theoretical_loss": 3.466818572776389,
+ "tokens_seen": 1945305088
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033182748244734204,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4668088621104056,
+ "tokens_seen": 1945370624
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033181143430290874,
+ "loss": 1.2951,
+ "theoretical_loss": 3.4667991518631456,
+ "tokens_seen": 1945436160
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033179538615847544,
+ "loss": 1.306,
+ "theoretical_loss": 3.4667894420345764,
+ "tokens_seen": 1945501696
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033177933801404214,
+ "loss": 1.2956,
+ "theoretical_loss": 3.466779732624667,
+ "tokens_seen": 1945567232
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033176328986960884,
+ "loss": 1.3359,
+ "theoretical_loss": 3.466770023633384,
+ "tokens_seen": 1945632768
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033174724172517554,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4667603150606956,
+ "tokens_seen": 1945698304
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033173119358074224,
+ "loss": 1.3038,
+ "theoretical_loss": 3.4667506069065706,
+ "tokens_seen": 1945763840
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033171514543630894,
+ "loss": 1.278,
+ "theoretical_loss": 3.4667408991709756,
+ "tokens_seen": 1945829376
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033169909729187564,
+ "loss": 1.3088,
+ "theoretical_loss": 3.466731191853879,
+ "tokens_seen": 1945894912
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033168304914744234,
+ "loss": 1.288,
+ "theoretical_loss": 3.466721484955248,
+ "tokens_seen": 1945960448
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033166700100300904,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4667117784750516,
+ "tokens_seen": 1946025984
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033165095285857574,
+ "loss": 1.3329,
+ "theoretical_loss": 3.4667020724132573,
+ "tokens_seen": 1946091520
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033163490471414245,
+ "loss": 1.2158,
+ "theoretical_loss": 3.4666923667698324,
+ "tokens_seen": 1946157056
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033161885656970915,
+ "loss": 1.3356,
+ "theoretical_loss": 3.4666826615447452,
+ "tokens_seen": 1946222592
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033160280842527585,
+ "loss": 1.191,
+ "theoretical_loss": 3.4666729567379635,
+ "tokens_seen": 1946288128
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033158676028084255,
+ "loss": 1.2726,
+ "theoretical_loss": 3.4666632523494556,
+ "tokens_seen": 1946353664
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 652242,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2615714073181152,
+ "objective/train/theoretical_loss": 3.466653548379189,
+ "objective/train/tokens_used": 1966879200,
+ "theoretical_loss": 3.466653548379189,
+ "tokens_seen": 1946419200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033157071213640925,
+ "loss": 1.3033,
+ "theoretical_loss": 3.466653548379189,
+ "tokens_seen": 1946419200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033155466399197595,
+ "loss": 1.1922,
+ "theoretical_loss": 3.4666438448271313,
+ "tokens_seen": 1946484736
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033153861584754265,
+ "loss": 1.3145,
+ "theoretical_loss": 3.466634141693251,
+ "tokens_seen": 1946550272
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033152256770310935,
+ "loss": 1.314,
+ "theoretical_loss": 3.4666244389775156,
+ "tokens_seen": 1946615808
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033150651955867605,
+ "loss": 1.1525,
+ "theoretical_loss": 3.4666147366798934,
+ "tokens_seen": 1946681344
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033149047141424275,
+ "loss": 1.2836,
+ "theoretical_loss": 3.466605034800352,
+ "tokens_seen": 1946746880
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033147442326980945,
+ "loss": 1.2922,
+ "theoretical_loss": 3.466595333338859,
+ "tokens_seen": 1946812416
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033145837512537615,
+ "loss": 1.3372,
+ "theoretical_loss": 3.466585632295383,
+ "tokens_seen": 1946877952
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033144232698094285,
+ "loss": 1.2432,
+ "theoretical_loss": 3.466575931669891,
+ "tokens_seen": 1946943488
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033142627883650955,
+ "loss": 1.3169,
+ "theoretical_loss": 3.4665662314623527,
+ "tokens_seen": 1947009024
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033141023069207625,
+ "loss": 1.3301,
+ "theoretical_loss": 3.466556531672734,
+ "tokens_seen": 1947074560
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033139418254764296,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4665468323010034,
+ "tokens_seen": 1947140096
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033137813440320966,
+ "loss": 1.4026,
+ "theoretical_loss": 3.4665371333471295,
+ "tokens_seen": 1947205632
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033136208625877636,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4665274348110797,
+ "tokens_seen": 1947271168
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033134603811434306,
+ "loss": 1.4576,
+ "theoretical_loss": 3.4665177366928224,
+ "tokens_seen": 1947336704
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033132998996990976,
+ "loss": 1.1986,
+ "theoretical_loss": 3.466508038992325,
+ "tokens_seen": 1947402240
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033131394182547646,
+ "loss": 1.3405,
+ "theoretical_loss": 3.4664983417095554,
+ "tokens_seen": 1947467776
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033129789368104316,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4664886448444823,
+ "tokens_seen": 1947533312
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033128184553660986,
+ "loss": 1.222,
+ "theoretical_loss": 3.4664789483970733,
+ "tokens_seen": 1947598848
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033126579739217656,
+ "loss": 1.196,
+ "theoretical_loss": 3.4664692523672955,
+ "tokens_seen": 1947664384
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033124974924774326,
+ "loss": 1.3111,
+ "theoretical_loss": 3.466459556755118,
+ "tokens_seen": 1947729920
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033123370110330996,
+ "loss": 1.2485,
+ "theoretical_loss": 3.466449861560508,
+ "tokens_seen": 1947795456
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003312176529588766,
+ "loss": 1.2839,
+ "theoretical_loss": 3.466440166783434,
+ "tokens_seen": 1947860992
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003312016048144433,
+ "loss": 1.3024,
+ "theoretical_loss": 3.4664304724238644,
+ "tokens_seen": 1947926528
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033118555667001,
+ "loss": 1.1599,
+ "theoretical_loss": 3.4664207784817656,
+ "tokens_seen": 1947992064
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003311695085255767,
+ "loss": 1.3215,
+ "theoretical_loss": 3.466411084957107,
+ "tokens_seen": 1948057600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003311534603811434,
+ "loss": 1.275,
+ "theoretical_loss": 3.466401391849856,
+ "tokens_seen": 1948123136
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003311374122367101,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4663916991599812,
+ "tokens_seen": 1948188672
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003311213640922768,
+ "loss": 1.2924,
+ "theoretical_loss": 3.466382006887449,
+ "tokens_seen": 1948254208
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003311053159478435,
+ "loss": 1.2992,
+ "theoretical_loss": 3.4663723150322294,
+ "tokens_seen": 1948319744
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003310892678034102,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4663626235942893,
+ "tokens_seen": 1948385280
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003310732196589769,
+ "loss": 1.3791,
+ "theoretical_loss": 3.4663529325735967,
+ "tokens_seen": 1948450816
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003310571715145436,
+ "loss": 1.2621,
+ "theoretical_loss": 3.46634324197012,
+ "tokens_seen": 1948516352
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003310411233701103,
+ "loss": 1.3842,
+ "theoretical_loss": 3.466333551783827,
+ "tokens_seen": 1948581888
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000331025075225677,
+ "loss": 1.139,
+ "theoretical_loss": 3.4663238620146855,
+ "tokens_seen": 1948647424
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003310090270812437,
+ "loss": 1.396,
+ "theoretical_loss": 3.466314172662664,
+ "tokens_seen": 1948712960
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003309929789368104,
+ "loss": 1.2506,
+ "theoretical_loss": 3.46630448372773,
+ "tokens_seen": 1948778496
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003309769307923771,
+ "loss": 1.1412,
+ "theoretical_loss": 3.4662947952098517,
+ "tokens_seen": 1948844032
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003309608826479438,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4662851071089973,
+ "tokens_seen": 1948909568
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003309448345035105,
+ "loss": 1.2088,
+ "theoretical_loss": 3.4662754194251346,
+ "tokens_seen": 1948975104
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003309287863590773,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4662657321582317,
+ "tokens_seen": 1949040640
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000330912738214644,
+ "loss": 1.3192,
+ "theoretical_loss": 3.466256045308257,
+ "tokens_seen": 1949106176
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308966900702107,
+ "loss": 1.1845,
+ "theoretical_loss": 3.466246358875178,
+ "tokens_seen": 1949171712
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308806419257774,
+ "loss": 1.2129,
+ "theoretical_loss": 3.4662366728589626,
+ "tokens_seen": 1949237248
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308645937813441,
+ "loss": 1.3485,
+ "theoretical_loss": 3.4662269872595797,
+ "tokens_seen": 1949302784
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308485456369108,
+ "loss": 1.1281,
+ "theoretical_loss": 3.4662173020769966,
+ "tokens_seen": 1949368320
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308324974924775,
+ "loss": 1.171,
+ "theoretical_loss": 3.4662076173111815,
+ "tokens_seen": 1949433856
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308164493480442,
+ "loss": 1.3374,
+ "theoretical_loss": 3.466197932962103,
+ "tokens_seen": 1949499392
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003308004012036109,
+ "loss": 1.244,
+ "theoretical_loss": 3.466188249029728,
+ "tokens_seen": 1949564928
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003307843530591776,
+ "loss": 1.2408,
+ "theoretical_loss": 3.466178565514026,
+ "tokens_seen": 1949630464
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 653381,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3046982288360596,
+ "objective/train/theoretical_loss": 3.466168882414964,
+ "objective/train/tokens_used": 1970156000,
+ "theoretical_loss": 3.466168882414964,
+ "tokens_seen": 1949696000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003307683049147443,
+ "loss": 1.3013,
+ "theoretical_loss": 3.466168882414964,
+ "tokens_seen": 1949696000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000330752256770311,
+ "loss": 1.2409,
+ "theoretical_loss": 3.4661591997325107,
+ "tokens_seen": 1949761536
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003307362086258777,
+ "loss": 1.2333,
+ "theoretical_loss": 3.4661495174666332,
+ "tokens_seen": 1949827072
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003307201604814444,
+ "loss": 1.246,
+ "theoretical_loss": 3.4661398356173008,
+ "tokens_seen": 1949892608
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003307041123370111,
+ "loss": 1.1652,
+ "theoretical_loss": 3.466130154184481,
+ "tokens_seen": 1949958144
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306880641925778,
+ "loss": 1.2934,
+ "theoretical_loss": 3.4661204731681416,
+ "tokens_seen": 1950023680
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306720160481445,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4661107925682515,
+ "tokens_seen": 1950089216
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306559679037112,
+ "loss": 1.269,
+ "theoretical_loss": 3.466101112384778,
+ "tokens_seen": 1950154752
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306399197592779,
+ "loss": 1.2779,
+ "theoretical_loss": 3.4660914326176897,
+ "tokens_seen": 1950220288
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306238716148446,
+ "loss": 1.2395,
+ "theoretical_loss": 3.466081753266954,
+ "tokens_seen": 1950285824
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003306078234704113,
+ "loss": 1.3344,
+ "theoretical_loss": 3.4660720743325397,
+ "tokens_seen": 1950351360
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033059177532597793,
+ "loss": 1.2097,
+ "theoretical_loss": 3.466062395814415,
+ "tokens_seen": 1950416896
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033057572718154463,
+ "loss": 1.1702,
+ "theoretical_loss": 3.4660527177125475,
+ "tokens_seen": 1950482432
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033055967903711133,
+ "loss": 1.3363,
+ "theoretical_loss": 3.4660430400269058,
+ "tokens_seen": 1950547968
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033054363089267803,
+ "loss": 1.218,
+ "theoretical_loss": 3.4660333627574573,
+ "tokens_seen": 1950613504
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033052758274824474,
+ "loss": 1.3456,
+ "theoretical_loss": 3.4660236859041706,
+ "tokens_seen": 1950679040
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033051153460381144,
+ "loss": 1.2671,
+ "theoretical_loss": 3.466014009467014,
+ "tokens_seen": 1950744576
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033049548645937814,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4660043334459556,
+ "tokens_seen": 1950810112
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033047943831494484,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4659946578409633,
+ "tokens_seen": 1950875648
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033046339017051154,
+ "loss": 1.2373,
+ "theoretical_loss": 3.465984982652005,
+ "tokens_seen": 1950941184
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033044734202607824,
+ "loss": 1.3095,
+ "theoretical_loss": 3.4659753078790487,
+ "tokens_seen": 1951006720
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033043129388164494,
+ "loss": 1.2732,
+ "theoretical_loss": 3.465965633522064,
+ "tokens_seen": 1951072256
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033041524573721164,
+ "loss": 1.3532,
+ "theoretical_loss": 3.4659559595810174,
+ "tokens_seen": 1951137792
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033039919759277834,
+ "loss": 1.2832,
+ "theoretical_loss": 3.4659462860558774,
+ "tokens_seen": 1951203328
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033038314944834504,
+ "loss": 1.2847,
+ "theoretical_loss": 3.4659366129466127,
+ "tokens_seen": 1951268864
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033036710130391174,
+ "loss": 1.3175,
+ "theoretical_loss": 3.465926940253191,
+ "tokens_seen": 1951334400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033035105315947844,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4659172679755805,
+ "tokens_seen": 1951399936
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033033500501504514,
+ "loss": 1.353,
+ "theoretical_loss": 3.46590759611375,
+ "tokens_seen": 1951465472
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033031895687061184,
+ "loss": 1.2742,
+ "theoretical_loss": 3.4658979246676664,
+ "tokens_seen": 1951531008
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033030290872617854,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4658882536372992,
+ "tokens_seen": 1951596544
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033028686058174524,
+ "loss": 1.2848,
+ "theoretical_loss": 3.465878583022616,
+ "tokens_seen": 1951662080
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033027081243731195,
+ "loss": 1.4071,
+ "theoretical_loss": 3.4658689128235842,
+ "tokens_seen": 1951727616
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033025476429287865,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4658592430401733,
+ "tokens_seen": 1951793152
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033023871614844535,
+ "loss": 1.1321,
+ "theoretical_loss": 3.465849573672351,
+ "tokens_seen": 1951858688
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033022266800401205,
+ "loss": 1.2115,
+ "theoretical_loss": 3.465839904720085,
+ "tokens_seen": 1951924224
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033020661985957875,
+ "loss": 1.229,
+ "theoretical_loss": 3.465830236183344,
+ "tokens_seen": 1951989760
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033019057171514545,
+ "loss": 1.27,
+ "theoretical_loss": 3.465820568062096,
+ "tokens_seen": 1952055296
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033017452357071215,
+ "loss": 1.2049,
+ "theoretical_loss": 3.4658109003563093,
+ "tokens_seen": 1952120832
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033015847542627885,
+ "loss": 1.3145,
+ "theoretical_loss": 3.4658012330659522,
+ "tokens_seen": 1952186368
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033014242728184555,
+ "loss": 1.2956,
+ "theoretical_loss": 3.465791566190993,
+ "tokens_seen": 1952251904
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033012637913741225,
+ "loss": 1.233,
+ "theoretical_loss": 3.4657818997313994,
+ "tokens_seen": 1952317440
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033011033099297895,
+ "loss": 1.2568,
+ "theoretical_loss": 3.4657722336871397,
+ "tokens_seen": 1952382976
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033009428284854565,
+ "loss": 1.2994,
+ "theoretical_loss": 3.4657625680581825,
+ "tokens_seen": 1952448512
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033007823470411235,
+ "loss": 1.2272,
+ "theoretical_loss": 3.4657529028444958,
+ "tokens_seen": 1952514048
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033006218655967905,
+ "loss": 1.2572,
+ "theoretical_loss": 3.465743238046048,
+ "tokens_seen": 1952579584
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033004613841524575,
+ "loss": 1.2266,
+ "theoretical_loss": 3.465733573662807,
+ "tokens_seen": 1952645120
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033003009027081245,
+ "loss": 1.2673,
+ "theoretical_loss": 3.4657239096947414,
+ "tokens_seen": 1952710656
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00033001404212637916,
+ "loss": 1.2322,
+ "theoretical_loss": 3.465714246141819,
+ "tokens_seen": 1952776192
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032999799398194586,
+ "loss": 1.2502,
+ "theoretical_loss": 3.4657045830040083,
+ "tokens_seen": 1952841728
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032998194583751256,
+ "loss": 1.2254,
+ "theoretical_loss": 3.465694920281277,
+ "tokens_seen": 1952907264
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 654198,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1338186264038086,
+ "objective/train/theoretical_loss": 3.4656852579735946,
+ "objective/train/tokens_used": 1973432800,
+ "theoretical_loss": 3.4656852579735946,
+ "tokens_seen": 1952972800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032996589769307926,
+ "loss": 1.1054,
+ "theoretical_loss": 3.4656852579735946,
+ "tokens_seen": 1952972800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032994984954864596,
+ "loss": 1.3011,
+ "theoretical_loss": 3.4656755960809282,
+ "tokens_seen": 1953038336
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032993380140421266,
+ "loss": 1.2378,
+ "theoretical_loss": 3.465665934603247,
+ "tokens_seen": 1953103872
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032991775325977936,
+ "loss": 1.2829,
+ "theoretical_loss": 3.465656273540518,
+ "tokens_seen": 1953169408
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032990170511534606,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4656466128927104,
+ "tokens_seen": 1953234944
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032988565697091276,
+ "loss": 1.3214,
+ "theoretical_loss": 3.465636952659792,
+ "tokens_seen": 1953300480
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032986960882647946,
+ "loss": 1.2414,
+ "theoretical_loss": 3.4656272928417313,
+ "tokens_seen": 1953366016
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032985356068204616,
+ "loss": 1.2052,
+ "theoretical_loss": 3.4656176334384967,
+ "tokens_seen": 1953431552
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032983751253761286,
+ "loss": 1.2891,
+ "theoretical_loss": 3.465607974450056,
+ "tokens_seen": 1953497088
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003298214643931795,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4655983158763783,
+ "tokens_seen": 1953562624
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003298054162487462,
+ "loss": 1.1997,
+ "theoretical_loss": 3.465588657717431,
+ "tokens_seen": 1953628160
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003297893681043129,
+ "loss": 1.2775,
+ "theoretical_loss": 3.465578999973183,
+ "tokens_seen": 1953693696
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003297733199598796,
+ "loss": 1.2187,
+ "theoretical_loss": 3.465569342643602,
+ "tokens_seen": 1953759232
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003297572718154463,
+ "loss": 1.2553,
+ "theoretical_loss": 3.465559685728657,
+ "tokens_seen": 1953824768
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000329741223671013,
+ "loss": 1.1714,
+ "theoretical_loss": 3.4655500292283152,
+ "tokens_seen": 1953890304
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032972517552657977,
+ "loss": 1.2089,
+ "theoretical_loss": 3.465540373142546,
+ "tokens_seen": 1953955840
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032970912738214647,
+ "loss": 1.2255,
+ "theoretical_loss": 3.4655307174713172,
+ "tokens_seen": 1954021376
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032969307923771317,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4655210622145978,
+ "tokens_seen": 1954086912
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032967703109327987,
+ "loss": 1.2381,
+ "theoretical_loss": 3.4655114073723547,
+ "tokens_seen": 1954152448
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032966098294884657,
+ "loss": 1.1665,
+ "theoretical_loss": 3.465501752944557,
+ "tokens_seen": 1954217984
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032964493480441327,
+ "loss": 1.267,
+ "theoretical_loss": 3.4654920989311737,
+ "tokens_seen": 1954283520
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032962888665997997,
+ "loss": 1.2521,
+ "theoretical_loss": 3.4654824453321718,
+ "tokens_seen": 1954349056
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032961283851554667,
+ "loss": 1.2265,
+ "theoretical_loss": 3.46547279214752,
+ "tokens_seen": 1954414592
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032959679037111337,
+ "loss": 1.1826,
+ "theoretical_loss": 3.4654631393771878,
+ "tokens_seen": 1954480128
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032958074222668007,
+ "loss": 1.342,
+ "theoretical_loss": 3.4654534870211418,
+ "tokens_seen": 1954545664
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003295646940822468,
+ "loss": 1.1846,
+ "theoretical_loss": 3.465443835079351,
+ "tokens_seen": 1954611200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003295486459378135,
+ "loss": 1.1963,
+ "theoretical_loss": 3.4654341835517846,
+ "tokens_seen": 1954676736
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003295325977933802,
+ "loss": 1.2572,
+ "theoretical_loss": 3.46542453243841,
+ "tokens_seen": 1954742272
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003295165496489469,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4654148817391954,
+ "tokens_seen": 1954807808
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003295005015045136,
+ "loss": 1.2157,
+ "theoretical_loss": 3.4654052314541097,
+ "tokens_seen": 1954873344
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003294844533600803,
+ "loss": 1.2038,
+ "theoretical_loss": 3.4653955815831208,
+ "tokens_seen": 1954938880
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000329468405215647,
+ "loss": 1.4662,
+ "theoretical_loss": 3.4653859321261975,
+ "tokens_seen": 1955004416
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003294523570712137,
+ "loss": 1.25,
+ "theoretical_loss": 3.465376283083308,
+ "tokens_seen": 1955069952
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003294363089267804,
+ "loss": 1.2503,
+ "theoretical_loss": 3.46536663445442,
+ "tokens_seen": 1955135488
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003294202607823471,
+ "loss": 1.2677,
+ "theoretical_loss": 3.4653569862395033,
+ "tokens_seen": 1955201024
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003294042126379138,
+ "loss": 1.3817,
+ "theoretical_loss": 3.4653473384385247,
+ "tokens_seen": 1955266560
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003293881644934805,
+ "loss": 1.3955,
+ "theoretical_loss": 3.4653376910514533,
+ "tokens_seen": 1955332096
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003293721163490472,
+ "loss": 1.3591,
+ "theoretical_loss": 3.465328044078258,
+ "tokens_seen": 1955397632
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003293560682046139,
+ "loss": 1.1738,
+ "theoretical_loss": 3.465318397518906,
+ "tokens_seen": 1955463168
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003293400200601806,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4653087513733665,
+ "tokens_seen": 1955528704
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003293239719157473,
+ "loss": 1.2499,
+ "theoretical_loss": 3.465299105641608,
+ "tokens_seen": 1955594240
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000329307923771314,
+ "loss": 1.3249,
+ "theoretical_loss": 3.465289460323598,
+ "tokens_seen": 1955659776
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292918756268807,
+ "loss": 1.3,
+ "theoretical_loss": 3.465279815419306,
+ "tokens_seen": 1955725312
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292758274824474,
+ "loss": 1.2575,
+ "theoretical_loss": 3.465270170928699,
+ "tokens_seen": 1955790848
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292597793380141,
+ "loss": 1.2391,
+ "theoretical_loss": 3.465260526851747,
+ "tokens_seen": 1955856384
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292437311935808,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4652508831884177,
+ "tokens_seen": 1955921920
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292276830491475,
+ "loss": 1.3291,
+ "theoretical_loss": 3.4652412399386794,
+ "tokens_seen": 1955987456
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003292116349047142,
+ "loss": 1.3256,
+ "theoretical_loss": 3.4652315971025,
+ "tokens_seen": 1956052992
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003291955867602809,
+ "loss": 1.365,
+ "theoretical_loss": 3.4652219546798486,
+ "tokens_seen": 1956118528
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032917953861584753,
+ "loss": 1.2115,
+ "theoretical_loss": 3.4652123126706935,
+ "tokens_seen": 1956184064
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 655460,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.333320140838623,
+ "objective/train/theoretical_loss": 3.4652026710750032,
+ "objective/train/tokens_used": 1976709600,
+ "theoretical_loss": 3.4652026710750032,
+ "tokens_seen": 1956249600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032916349047141423,
+ "loss": 1.4231,
+ "theoretical_loss": 3.4652026710750032,
+ "tokens_seen": 1956249600
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032914744232698094,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4651930298927462,
+ "tokens_seen": 1956315136
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032913139418254764,
+ "loss": 1.1809,
+ "theoretical_loss": 3.46518338912389,
+ "tokens_seen": 1956380672
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032911534603811434,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4651737487684047,
+ "tokens_seen": 1956446208
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032909929789368104,
+ "loss": 1.3129,
+ "theoretical_loss": 3.465164108826257,
+ "tokens_seen": 1956511744
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032908324974924774,
+ "loss": 1.1499,
+ "theoretical_loss": 3.4651544692974166,
+ "tokens_seen": 1956577280
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032906720160481444,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4651448301818513,
+ "tokens_seen": 1956642816
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032905115346038114,
+ "loss": 1.3475,
+ "theoretical_loss": 3.4651351914795296,
+ "tokens_seen": 1956708352
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032903510531594784,
+ "loss": 1.3061,
+ "theoretical_loss": 3.46512555319042,
+ "tokens_seen": 1956773888
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032901905717151454,
+ "loss": 1.2135,
+ "theoretical_loss": 3.465115915314491,
+ "tokens_seen": 1956839424
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032900300902708124,
+ "loss": 1.1992,
+ "theoretical_loss": 3.465106277851711,
+ "tokens_seen": 1956904960
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032898696088264794,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4650966408020487,
+ "tokens_seen": 1956970496
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032897091273821464,
+ "loss": 1.3494,
+ "theoretical_loss": 3.465087004165472,
+ "tokens_seen": 1957036032
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032895486459378134,
+ "loss": 1.3465,
+ "theoretical_loss": 3.46507736794195,
+ "tokens_seen": 1957101568
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032893881644934804,
+ "loss": 1.2072,
+ "theoretical_loss": 3.465067732131451,
+ "tokens_seen": 1957167104
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032892276830491474,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4650580967339426,
+ "tokens_seen": 1957232640
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032890672016048145,
+ "loss": 1.3029,
+ "theoretical_loss": 3.4650484617493946,
+ "tokens_seen": 1957298176
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032889067201604815,
+ "loss": 1.2609,
+ "theoretical_loss": 3.465038827177775,
+ "tokens_seen": 1957363712
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032887462387161485,
+ "loss": 1.3106,
+ "theoretical_loss": 3.4650291930190518,
+ "tokens_seen": 1957429248
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032885857572718155,
+ "loss": 1.2851,
+ "theoretical_loss": 3.465019559273194,
+ "tokens_seen": 1957494784
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032884252758274825,
+ "loss": 1.2054,
+ "theoretical_loss": 3.46500992594017,
+ "tokens_seen": 1957560320
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032882647943831495,
+ "loss": 1.3576,
+ "theoretical_loss": 3.4650002930199477,
+ "tokens_seen": 1957625856
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032881043129388165,
+ "loss": 1.2345,
+ "theoretical_loss": 3.4649906605124965,
+ "tokens_seen": 1957691392
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032879438314944835,
+ "loss": 1.2881,
+ "theoretical_loss": 3.4649810284177844,
+ "tokens_seen": 1957756928
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032877833500501505,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4649713967357805,
+ "tokens_seen": 1957822464
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032876228686058175,
+ "loss": 1.3024,
+ "theoretical_loss": 3.4649617654664526,
+ "tokens_seen": 1957888000
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032874623871614845,
+ "loss": 1.2561,
+ "theoretical_loss": 3.464952134609769,
+ "tokens_seen": 1957953536
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032873019057171515,
+ "loss": 1.242,
+ "theoretical_loss": 3.4649425041656987,
+ "tokens_seen": 1958019072
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032871414242728185,
+ "loss": 1.3327,
+ "theoretical_loss": 3.4649328741342105,
+ "tokens_seen": 1958084608
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032869809428284855,
+ "loss": 1.1571,
+ "theoretical_loss": 3.464923244515272,
+ "tokens_seen": 1958150144
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032868204613841525,
+ "loss": 1.1995,
+ "theoretical_loss": 3.464913615308853,
+ "tokens_seen": 1958215680
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032866599799398195,
+ "loss": 1.2167,
+ "theoretical_loss": 3.4649039865149205,
+ "tokens_seen": 1958281216
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032864994984954866,
+ "loss": 1.3154,
+ "theoretical_loss": 3.464894358133444,
+ "tokens_seen": 1958346752
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032863390170511536,
+ "loss": 1.3238,
+ "theoretical_loss": 3.4648847301643926,
+ "tokens_seen": 1958412288
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032861785356068206,
+ "loss": 1.2233,
+ "theoretical_loss": 3.464875102607733,
+ "tokens_seen": 1958477824
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032860180541624876,
+ "loss": 1.2041,
+ "theoretical_loss": 3.4648654754634354,
+ "tokens_seen": 1958543360
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032858575727181546,
+ "loss": 1.3761,
+ "theoretical_loss": 3.464855848731468,
+ "tokens_seen": 1958608896
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032856970912738216,
+ "loss": 1.2536,
+ "theoretical_loss": 3.464846222411799,
+ "tokens_seen": 1958674432
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032855366098294886,
+ "loss": 1.3121,
+ "theoretical_loss": 3.464836596504396,
+ "tokens_seen": 1958739968
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032853761283851556,
+ "loss": 1.3333,
+ "theoretical_loss": 3.46482697100923,
+ "tokens_seen": 1958805504
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032852156469408226,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4648173459262672,
+ "tokens_seen": 1958871040
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032850551654964896,
+ "loss": 1.3231,
+ "theoretical_loss": 3.464807721255478,
+ "tokens_seen": 1958936576
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032848946840521566,
+ "loss": 1.3195,
+ "theoretical_loss": 3.4647980969968293,
+ "tokens_seen": 1959002112
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032847342026078236,
+ "loss": 1.2719,
+ "theoretical_loss": 3.464788473150291,
+ "tokens_seen": 1959067648
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032845737211634906,
+ "loss": 1.1816,
+ "theoretical_loss": 3.4647788497158305,
+ "tokens_seen": 1959133184
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032844132397191576,
+ "loss": 1.1692,
+ "theoretical_loss": 3.4647692266934174,
+ "tokens_seen": 1959198720
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032842527582748246,
+ "loss": 1.2307,
+ "theoretical_loss": 3.46475960408302,
+ "tokens_seen": 1959264256
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032840922768304916,
+ "loss": 1.2162,
+ "theoretical_loss": 3.464749981884606,
+ "tokens_seen": 1959329792
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032839317953861587,
+ "loss": 1.2723,
+ "theoretical_loss": 3.4647403600981455,
+ "tokens_seen": 1959395328
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032837713139418257,
+ "loss": 1.2184,
+ "theoretical_loss": 3.464730738723606,
+ "tokens_seen": 1959460864
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 656810,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5059330463409424,
+ "objective/train/theoretical_loss": 3.4647211177609565,
+ "objective/train/tokens_used": 1979986400,
+ "theoretical_loss": 3.4647211177609565,
+ "tokens_seen": 1959526400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032836108324974927,
+ "loss": 1.4133,
+ "theoretical_loss": 3.4647211177609565,
+ "tokens_seen": 1959526400
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032834503510531597,
+ "loss": 1.3079,
+ "theoretical_loss": 3.464711497210165,
+ "tokens_seen": 1959591936
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032832898696088267,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4647018770712013,
+ "tokens_seen": 1959657472
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032831293881644937,
+ "loss": 1.291,
+ "theoretical_loss": 3.4646922573440326,
+ "tokens_seen": 1959723008
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032829689067201607,
+ "loss": 1.2494,
+ "theoretical_loss": 3.464682638028629,
+ "tokens_seen": 1959788544
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032828084252758277,
+ "loss": 1.2652,
+ "theoretical_loss": 3.4646730191249575,
+ "tokens_seen": 1959854080
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032826479438314947,
+ "loss": 1.1878,
+ "theoretical_loss": 3.464663400632988,
+ "tokens_seen": 1959919616
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032824874623871617,
+ "loss": 1.3116,
+ "theoretical_loss": 3.4646537825526877,
+ "tokens_seen": 1959985152
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032823269809428287,
+ "loss": 1.1721,
+ "theoretical_loss": 3.464644164884027,
+ "tokens_seen": 1960050688
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032821664994984957,
+ "loss": 1.2462,
+ "theoretical_loss": 3.4646345476269733,
+ "tokens_seen": 1960116224
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003282006018054163,
+ "loss": 1.2995,
+ "theoretical_loss": 3.464624930781496,
+ "tokens_seen": 1960181760
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000328184553660983,
+ "loss": 1.3637,
+ "theoretical_loss": 3.4646153143475633,
+ "tokens_seen": 1960247296
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003281685055165497,
+ "loss": 1.2276,
+ "theoretical_loss": 3.464605698325143,
+ "tokens_seen": 1960312832
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003281524573721164,
+ "loss": 1.2547,
+ "theoretical_loss": 3.4645960827142055,
+ "tokens_seen": 1960378368
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003281364092276831,
+ "loss": 1.3351,
+ "theoretical_loss": 3.464586467514718,
+ "tokens_seen": 1960443904
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003281203610832498,
+ "loss": 1.3077,
+ "theoretical_loss": 3.46457685272665,
+ "tokens_seen": 1960509440
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003281043129388165,
+ "loss": 1.156,
+ "theoretical_loss": 3.4645672383499693,
+ "tokens_seen": 1960574976
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003280882647943832,
+ "loss": 1.2096,
+ "theoretical_loss": 3.4645576243846454,
+ "tokens_seen": 1960640512
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003280722166499499,
+ "loss": 1.351,
+ "theoretical_loss": 3.4645480108306463,
+ "tokens_seen": 1960706048
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003280561685055166,
+ "loss": 1.3297,
+ "theoretical_loss": 3.4645383976879414,
+ "tokens_seen": 1960771584
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003280401203610833,
+ "loss": 1.2935,
+ "theoretical_loss": 3.4645287849564985,
+ "tokens_seen": 1960837120
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032802407221665,
+ "loss": 1.3245,
+ "theoretical_loss": 3.4645191726362867,
+ "tokens_seen": 1960902656
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003280080240722167,
+ "loss": 1.3396,
+ "theoretical_loss": 3.4645095607272745,
+ "tokens_seen": 1960968192
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279919759277834,
+ "loss": 1.2586,
+ "theoretical_loss": 3.464499949229431,
+ "tokens_seen": 1961033728
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279759277833501,
+ "loss": 1.261,
+ "theoretical_loss": 3.464490338142724,
+ "tokens_seen": 1961099264
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279598796389168,
+ "loss": 1.3892,
+ "theoretical_loss": 3.464480727467123,
+ "tokens_seen": 1961164800
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279438314944835,
+ "loss": 1.2508,
+ "theoretical_loss": 3.4644711172025966,
+ "tokens_seen": 1961230336
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279277833500502,
+ "loss": 1.3271,
+ "theoretical_loss": 3.464461507349113,
+ "tokens_seen": 1961295872
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003279117352056169,
+ "loss": 1.211,
+ "theoretical_loss": 3.464451897906642,
+ "tokens_seen": 1961361408
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003278956870611836,
+ "loss": 1.3761,
+ "theoretical_loss": 3.4644422888751505,
+ "tokens_seen": 1961426944
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003278796389167503,
+ "loss": 1.2552,
+ "theoretical_loss": 3.4644326802546086,
+ "tokens_seen": 1961492480
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.000327863590772317,
+ "loss": 1.3391,
+ "theoretical_loss": 3.4644230720449842,
+ "tokens_seen": 1961558016
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003278475426278837,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4644134642462467,
+ "tokens_seen": 1961623552
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003278314944834504,
+ "loss": 1.2387,
+ "theoretical_loss": 3.4644038568583646,
+ "tokens_seen": 1961689088
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003278154463390171,
+ "loss": 1.2228,
+ "theoretical_loss": 3.464394249881306,
+ "tokens_seen": 1961754624
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.0003277993981945838,
+ "loss": 1.2147,
+ "theoretical_loss": 3.4643846433150403,
+ "tokens_seen": 1961820160
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032778335005015044,
+ "loss": 1.2534,
+ "theoretical_loss": 3.464375037159536,
+ "tokens_seen": 1961885696
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032776730190571714,
+ "loss": 1.2739,
+ "theoretical_loss": 3.4643654314147616,
+ "tokens_seen": 1961951232
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032775125376128384,
+ "loss": 1.3218,
+ "theoretical_loss": 3.464355826080686,
+ "tokens_seen": 1962016768
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032773520561685054,
+ "loss": 1.2753,
+ "theoretical_loss": 3.464346221157278,
+ "tokens_seen": 1962082304
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032771915747241724,
+ "loss": 1.2796,
+ "theoretical_loss": 3.4643366166445064,
+ "tokens_seen": 1962147840
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032770310932798394,
+ "loss": 1.2492,
+ "theoretical_loss": 3.46432701254234,
+ "tokens_seen": 1962213376
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032768706118355064,
+ "loss": 1.1715,
+ "theoretical_loss": 3.4643174088507473,
+ "tokens_seen": 1962278912
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032767101303911734,
+ "loss": 1.379,
+ "theoretical_loss": 3.464307805569697,
+ "tokens_seen": 1962344448
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032765496489468404,
+ "loss": 1.1766,
+ "theoretical_loss": 3.4642982026991573,
+ "tokens_seen": 1962409984
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032763891675025074,
+ "loss": 1.2037,
+ "theoretical_loss": 3.4642886002390982,
+ "tokens_seen": 1962475520
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032762286860581744,
+ "loss": 1.2102,
+ "theoretical_loss": 3.4642789981894877,
+ "tokens_seen": 1962541056
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032760682046138414,
+ "loss": 1.3144,
+ "theoretical_loss": 3.4642693965502946,
+ "tokens_seen": 1962606592
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032759077231695084,
+ "loss": 1.3125,
+ "theoretical_loss": 3.464259795321487,
+ "tokens_seen": 1962672128
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032757472417251754,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4642501945030357,
+ "tokens_seen": 1962737664
+ },
+ {
+ "epoch": 0.59,
+ "objective/train/docs_used": 657689,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2876102924346924,
+ "objective/train/theoretical_loss": 3.4642405940949073,
+ "objective/train/tokens_used": 1983263200,
+ "theoretical_loss": 3.4642405940949073,
+ "tokens_seen": 1962803200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032755867602808424,
+ "loss": 1.2158,
+ "theoretical_loss": 3.4642405940949073,
+ "tokens_seen": 1962803200
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032754262788365094,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4642309940970715,
+ "tokens_seen": 1962868736
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032752657973921765,
+ "loss": 1.2317,
+ "theoretical_loss": 3.464221394509497,
+ "tokens_seen": 1962934272
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032751053159478435,
+ "loss": 1.1704,
+ "theoretical_loss": 3.4642117953321523,
+ "tokens_seen": 1962999808
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032749448345035105,
+ "loss": 1.3569,
+ "theoretical_loss": 3.4642021965650067,
+ "tokens_seen": 1963065344
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032747843530591775,
+ "loss": 1.2822,
+ "theoretical_loss": 3.4641925982080286,
+ "tokens_seen": 1963130880
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032746238716148445,
+ "loss": 1.2775,
+ "theoretical_loss": 3.464183000261187,
+ "tokens_seen": 1963196416
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032744633901705115,
+ "loss": 1.1282,
+ "theoretical_loss": 3.46417340272445,
+ "tokens_seen": 1963261952
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032743029087261785,
+ "loss": 1.2584,
+ "theoretical_loss": 3.4641638055977872,
+ "tokens_seen": 1963327488
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032741424272818455,
+ "loss": 1.2974,
+ "theoretical_loss": 3.464154208881167,
+ "tokens_seen": 1963393024
+ },
+ {
+ "epoch": 0.59,
+ "learning_rate": 0.00032739819458375125,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4641446125745587,
+ "tokens_seen": 1963458560
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032738214643931795,
+ "loss": 1.1956,
+ "theoretical_loss": 3.464135016677931,
+ "tokens_seen": 1963524096
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032736609829488465,
+ "loss": 1.2721,
+ "theoretical_loss": 3.4641254211912518,
+ "tokens_seen": 1963589632
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032735005015045135,
+ "loss": 1.1859,
+ "theoretical_loss": 3.46411582611449,
+ "tokens_seen": 1963655168
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032733400200601805,
+ "loss": 1.3437,
+ "theoretical_loss": 3.464106231447616,
+ "tokens_seen": 1963720704
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003273179538615848,
+ "loss": 1.2472,
+ "theoretical_loss": 3.464096637190597,
+ "tokens_seen": 1963786240
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003273019057171515,
+ "loss": 1.3096,
+ "theoretical_loss": 3.4640870433434023,
+ "tokens_seen": 1963851776
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003272858575727182,
+ "loss": 1.1336,
+ "theoretical_loss": 3.464077449906001,
+ "tokens_seen": 1963917312
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003272698094282849,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4640678568783616,
+ "tokens_seen": 1963982848
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003272537612838516,
+ "loss": 1.2316,
+ "theoretical_loss": 3.464058264260453,
+ "tokens_seen": 1964048384
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003272377131394183,
+ "loss": 1.0924,
+ "theoretical_loss": 3.464048672052244,
+ "tokens_seen": 1964113920
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000327221664994985,
+ "loss": 1.302,
+ "theoretical_loss": 3.4640390802537038,
+ "tokens_seen": 1964179456
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003272056168505517,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4640294888648,
+ "tokens_seen": 1964244992
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003271895687061184,
+ "loss": 1.3096,
+ "theoretical_loss": 3.4640198978855032,
+ "tokens_seen": 1964310528
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003271735205616851,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4640103073157817,
+ "tokens_seen": 1964376064
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032715747241725176,
+ "loss": 1.1945,
+ "theoretical_loss": 3.4640007171556033,
+ "tokens_seen": 1964441600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032714142427281846,
+ "loss": 1.3418,
+ "theoretical_loss": 3.4639911274049378,
+ "tokens_seen": 1964507136
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032712537612838516,
+ "loss": 1.1271,
+ "theoretical_loss": 3.463981538063754,
+ "tokens_seen": 1964572672
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032710932798395186,
+ "loss": 1.264,
+ "theoretical_loss": 3.4639719491320204,
+ "tokens_seen": 1964638208
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032709327983951856,
+ "loss": 1.3211,
+ "theoretical_loss": 3.463962360609706,
+ "tokens_seen": 1964703744
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032707723169508526,
+ "loss": 1.2012,
+ "theoretical_loss": 3.4639527724967802,
+ "tokens_seen": 1964769280
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032706118355065196,
+ "loss": 1.3207,
+ "theoretical_loss": 3.463943184793211,
+ "tokens_seen": 1964834816
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032704513540621866,
+ "loss": 1.2717,
+ "theoretical_loss": 3.4639335974989676,
+ "tokens_seen": 1964900352
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032702908726178537,
+ "loss": 1.3023,
+ "theoretical_loss": 3.463924010614019,
+ "tokens_seen": 1964965888
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032701303911735207,
+ "loss": 1.1863,
+ "theoretical_loss": 3.463914424138334,
+ "tokens_seen": 1965031424
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032699699097291877,
+ "loss": 1.2445,
+ "theoretical_loss": 3.4639048380718815,
+ "tokens_seen": 1965096960
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032698094282848547,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4638952524146305,
+ "tokens_seen": 1965162496
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032696489468405217,
+ "loss": 1.2847,
+ "theoretical_loss": 3.46388566716655,
+ "tokens_seen": 1965228032
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032694884653961887,
+ "loss": 1.2845,
+ "theoretical_loss": 3.4638760823276082,
+ "tokens_seen": 1965293568
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032693279839518557,
+ "loss": 1.2086,
+ "theoretical_loss": 3.4638664978977745,
+ "tokens_seen": 1965359104
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032691675025075227,
+ "loss": 1.1956,
+ "theoretical_loss": 3.463856913877018,
+ "tokens_seen": 1965424640
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032690070210631897,
+ "loss": 1.2191,
+ "theoretical_loss": 3.463847330265307,
+ "tokens_seen": 1965490176
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032688465396188567,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4638377470626107,
+ "tokens_seen": 1965555712
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032686860581745237,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4638281642688984,
+ "tokens_seen": 1965621248
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032685255767301907,
+ "loss": 1.1754,
+ "theoretical_loss": 3.463818581884139,
+ "tokens_seen": 1965686784
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032683650952858577,
+ "loss": 1.3173,
+ "theoretical_loss": 3.4638089999083004,
+ "tokens_seen": 1965752320
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003268204613841525,
+ "loss": 1.1933,
+ "theoretical_loss": 3.4637994183413525,
+ "tokens_seen": 1965817856
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003268044132397192,
+ "loss": 1.1741,
+ "theoretical_loss": 3.4637898371832634,
+ "tokens_seen": 1965883392
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003267883650952859,
+ "loss": 1.2382,
+ "theoretical_loss": 3.463780256434003,
+ "tokens_seen": 1965948928
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003267723169508526,
+ "loss": 1.309,
+ "theoretical_loss": 3.4637706760935396,
+ "tokens_seen": 1966014464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.7325822315813355,
+ "debugging/entropy-1-grams": 5.057375499583508,
+ "debugging/length": 560.6,
+ "debugging/num_segments": 10,
+ "debugging/score": 0.065810269192784,
+ "debugging/score_std": 0.01045031538172114,
+ "epoch": 0.6,
+ "objective/train/docs_used": 658485,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.171479344367981,
+ "objective/train/theoretical_loss": 3.463761096161843,
+ "objective/train/tokens_used": 1986540000,
+ "theoretical_loss": 3.463761096161843,
+ "tokens_seen": 1966080000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003267562688064193,
+ "loss": 1.2288,
+ "theoretical_loss": 3.463761096161843,
+ "tokens_seen": 1966080000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000326740220661986,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4637515166388804,
+ "tokens_seen": 1966145536
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003267241725175527,
+ "loss": 1.1963,
+ "theoretical_loss": 3.463741937524622,
+ "tokens_seen": 1966211072
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003267081243731194,
+ "loss": 1.3209,
+ "theoretical_loss": 3.463732358819037,
+ "tokens_seen": 1966276608
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266920762286861,
+ "loss": 1.3658,
+ "theoretical_loss": 3.463722780522094,
+ "tokens_seen": 1966342144
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266760280842528,
+ "loss": 1.2307,
+ "theoretical_loss": 3.4637132026337607,
+ "tokens_seen": 1966407680
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266599799398195,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4637036251540083,
+ "tokens_seen": 1966473216
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266439317953862,
+ "loss": 1.2292,
+ "theoretical_loss": 3.463694048082804,
+ "tokens_seen": 1966538752
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266278836509529,
+ "loss": 1.2924,
+ "theoretical_loss": 3.463684471420118,
+ "tokens_seen": 1966604288
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003266118355065196,
+ "loss": 1.2146,
+ "theoretical_loss": 3.4636748951659175,
+ "tokens_seen": 1966669824
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003265957873620863,
+ "loss": 1.3442,
+ "theoretical_loss": 3.4636653193201736,
+ "tokens_seen": 1966735360
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000326579739217653,
+ "loss": 1.2257,
+ "theoretical_loss": 3.463655743882854,
+ "tokens_seen": 1966800896
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003265636910732197,
+ "loss": 1.2411,
+ "theoretical_loss": 3.463646168853928,
+ "tokens_seen": 1966866432
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003265476429287864,
+ "loss": 1.3025,
+ "theoretical_loss": 3.463636594233364,
+ "tokens_seen": 1966931968
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003265315947843531,
+ "loss": 1.197,
+ "theoretical_loss": 3.463627020021132,
+ "tokens_seen": 1966997504
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003265155466399198,
+ "loss": 1.2567,
+ "theoretical_loss": 3.4636174462172002,
+ "tokens_seen": 1967063040
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264994984954865,
+ "loss": 1.2023,
+ "theoretical_loss": 3.463607872821538,
+ "tokens_seen": 1967128576
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264834503510532,
+ "loss": 1.3127,
+ "theoretical_loss": 3.463598299834114,
+ "tokens_seen": 1967194112
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264674022066199,
+ "loss": 1.2774,
+ "theoretical_loss": 3.4635887272548977,
+ "tokens_seen": 1967259648
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264513540621866,
+ "loss": 1.3666,
+ "theoretical_loss": 3.463579155083858,
+ "tokens_seen": 1967325184
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264353059177533,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4635695833209637,
+ "tokens_seen": 1967390720
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032641925777332,
+ "loss": 1.3354,
+ "theoretical_loss": 3.4635600119661833,
+ "tokens_seen": 1967456256
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003264032096288867,
+ "loss": 1.3263,
+ "theoretical_loss": 3.463550441019487,
+ "tokens_seen": 1967521792
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032638716148445334,
+ "loss": 1.2991,
+ "theoretical_loss": 3.463540870480843,
+ "tokens_seen": 1967587328
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032637111334002004,
+ "loss": 1.206,
+ "theoretical_loss": 3.46353130035022,
+ "tokens_seen": 1967652864
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032635506519558674,
+ "loss": 1.2202,
+ "theoretical_loss": 3.463521730627588,
+ "tokens_seen": 1967718400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032633901705115344,
+ "loss": 1.2919,
+ "theoretical_loss": 3.463512161312915,
+ "tokens_seen": 1967783936
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032632296890672014,
+ "loss": 1.1321,
+ "theoretical_loss": 3.463502592406171,
+ "tokens_seen": 1967849472
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032630692076228684,
+ "loss": 1.229,
+ "theoretical_loss": 3.4634930239073243,
+ "tokens_seen": 1967915008
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032629087261785354,
+ "loss": 1.266,
+ "theoretical_loss": 3.4634834558163443,
+ "tokens_seen": 1967980544
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032627482447342024,
+ "loss": 1.2568,
+ "theoretical_loss": 3.4634738881331995,
+ "tokens_seen": 1968046080
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032625877632898694,
+ "loss": 1.1917,
+ "theoretical_loss": 3.46346432085786,
+ "tokens_seen": 1968111616
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032624272818455364,
+ "loss": 1.2718,
+ "theoretical_loss": 3.463454753990294,
+ "tokens_seen": 1968177152
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032622668004012034,
+ "loss": 1.3484,
+ "theoretical_loss": 3.4634451875304704,
+ "tokens_seen": 1968242688
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032621063189568704,
+ "loss": 1.4432,
+ "theoretical_loss": 3.4634356214783586,
+ "tokens_seen": 1968308224
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032619458375125374,
+ "loss": 1.252,
+ "theoretical_loss": 3.463426055833928,
+ "tokens_seen": 1968373760
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032617853560682044,
+ "loss": 1.3883,
+ "theoretical_loss": 3.463416490597147,
+ "tokens_seen": 1968439296
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032616248746238715,
+ "loss": 1.2708,
+ "theoretical_loss": 3.463406925767985,
+ "tokens_seen": 1968504832
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032614643931795385,
+ "loss": 1.214,
+ "theoretical_loss": 3.463397361346411,
+ "tokens_seen": 1968570368
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032613039117352055,
+ "loss": 1.2125,
+ "theoretical_loss": 3.463387797332394,
+ "tokens_seen": 1968635904
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003261143430290873,
+ "loss": 1.2935,
+ "theoretical_loss": 3.463378233725903,
+ "tokens_seen": 1968701440
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000326098294884654,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4633686705269073,
+ "tokens_seen": 1968766976
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260822467402207,
+ "loss": 1.2776,
+ "theoretical_loss": 3.463359107735376,
+ "tokens_seen": 1968832512
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260661985957874,
+ "loss": 1.3121,
+ "theoretical_loss": 3.463349545351278,
+ "tokens_seen": 1968898048
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260501504513541,
+ "loss": 1.3264,
+ "theoretical_loss": 3.463339983374582,
+ "tokens_seen": 1968963584
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260341023069208,
+ "loss": 1.2132,
+ "theoretical_loss": 3.4633304218052583,
+ "tokens_seen": 1969029120
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260180541624875,
+ "loss": 1.2347,
+ "theoretical_loss": 3.4633208606432744,
+ "tokens_seen": 1969094656
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003260020060180542,
+ "loss": 1.2675,
+ "theoretical_loss": 3.463311299888601,
+ "tokens_seen": 1969160192
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003259859578736209,
+ "loss": 1.248,
+ "theoretical_loss": 3.4633017395412056,
+ "tokens_seen": 1969225728
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003259699097291876,
+ "loss": 1.2427,
+ "theoretical_loss": 3.4632921796010585,
+ "tokens_seen": 1969291264
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 659633,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3816819190979004,
+ "objective/train/theoretical_loss": 3.463282620068128,
+ "objective/train/tokens_used": 1989816800,
+ "theoretical_loss": 3.463282620068128,
+ "tokens_seen": 1969356800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003259538615847543,
+ "loss": 1.3355,
+ "theoretical_loss": 3.463282620068128,
+ "tokens_seen": 1969356800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000325937813440321,
+ "loss": 1.24,
+ "theoretical_loss": 3.463273060942384,
+ "tokens_seen": 1969422336
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003259217652958877,
+ "loss": 1.2217,
+ "theoretical_loss": 3.4632635022237954,
+ "tokens_seen": 1969487872
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003259057171514544,
+ "loss": 1.231,
+ "theoretical_loss": 3.4632539439123304,
+ "tokens_seen": 1969553408
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258896690070211,
+ "loss": 1.3296,
+ "theoretical_loss": 3.4632443860079594,
+ "tokens_seen": 1969618944
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258736208625878,
+ "loss": 1.2672,
+ "theoretical_loss": 3.4632348285106502,
+ "tokens_seen": 1969684480
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258575727181545,
+ "loss": 1.2484,
+ "theoretical_loss": 3.4632252714203733,
+ "tokens_seen": 1969750016
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258415245737212,
+ "loss": 1.3167,
+ "theoretical_loss": 3.463215714737097,
+ "tokens_seen": 1969815552
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258254764292879,
+ "loss": 1.2029,
+ "theoretical_loss": 3.4632061584607907,
+ "tokens_seen": 1969881088
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003258094282848546,
+ "loss": 1.2591,
+ "theoretical_loss": 3.463196602591423,
+ "tokens_seen": 1969946624
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003257933801404213,
+ "loss": 1.3009,
+ "theoretical_loss": 3.4631870471289634,
+ "tokens_seen": 1970012160
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000325777331995988,
+ "loss": 1.3173,
+ "theoretical_loss": 3.4631774920733815,
+ "tokens_seen": 1970077696
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003257612838515547,
+ "loss": 1.1998,
+ "theoretical_loss": 3.463167937424646,
+ "tokens_seen": 1970143232
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032574523570712136,
+ "loss": 1.2499,
+ "theoretical_loss": 3.4631583831827264,
+ "tokens_seen": 1970208768
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032572918756268806,
+ "loss": 1.2767,
+ "theoretical_loss": 3.463148829347591,
+ "tokens_seen": 1970274304
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032571313941825476,
+ "loss": 1.2591,
+ "theoretical_loss": 3.4631392759192092,
+ "tokens_seen": 1970339840
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032569709127382146,
+ "loss": 1.4643,
+ "theoretical_loss": 3.463129722897551,
+ "tokens_seen": 1970405376
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032568104312938816,
+ "loss": 1.1429,
+ "theoretical_loss": 3.4631201702825853,
+ "tokens_seen": 1970470912
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032566499498495486,
+ "loss": 1.3224,
+ "theoretical_loss": 3.46311061807428,
+ "tokens_seen": 1970536448
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032564894684052157,
+ "loss": 1.2521,
+ "theoretical_loss": 3.463101066272606,
+ "tokens_seen": 1970601984
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032563289869608827,
+ "loss": 1.3094,
+ "theoretical_loss": 3.4630915148775316,
+ "tokens_seen": 1970667520
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032561685055165497,
+ "loss": 1.2385,
+ "theoretical_loss": 3.463081963889026,
+ "tokens_seen": 1970733056
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032560080240722167,
+ "loss": 1.3842,
+ "theoretical_loss": 3.4630724133070583,
+ "tokens_seen": 1970798592
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032558475426278837,
+ "loss": 1.2771,
+ "theoretical_loss": 3.4630628631315976,
+ "tokens_seen": 1970864128
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032556870611835507,
+ "loss": 1.3529,
+ "theoretical_loss": 3.4630533133626136,
+ "tokens_seen": 1970929664
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032555265797392177,
+ "loss": 1.2603,
+ "theoretical_loss": 3.463043764000075,
+ "tokens_seen": 1970995200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032553660982948847,
+ "loss": 1.173,
+ "theoretical_loss": 3.4630342150439515,
+ "tokens_seen": 1971060736
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032552056168505517,
+ "loss": 1.363,
+ "theoretical_loss": 3.4630246664942117,
+ "tokens_seen": 1971126272
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032550451354062187,
+ "loss": 1.2181,
+ "theoretical_loss": 3.4630151183508247,
+ "tokens_seen": 1971191808
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032548846539618857,
+ "loss": 1.2296,
+ "theoretical_loss": 3.4630055706137606,
+ "tokens_seen": 1971257344
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032547241725175527,
+ "loss": 1.1683,
+ "theoretical_loss": 3.462996023282988,
+ "tokens_seen": 1971322880
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000325456369107322,
+ "loss": 1.2918,
+ "theoretical_loss": 3.462986476358476,
+ "tokens_seen": 1971388416
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003254403209628887,
+ "loss": 1.2018,
+ "theoretical_loss": 3.4629769298401936,
+ "tokens_seen": 1971453952
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003254242728184554,
+ "loss": 1.2581,
+ "theoretical_loss": 3.462967383728111,
+ "tokens_seen": 1971519488
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003254082246740221,
+ "loss": 1.3155,
+ "theoretical_loss": 3.4629578380221964,
+ "tokens_seen": 1971585024
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253921765295888,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4629482927224196,
+ "tokens_seen": 1971650560
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253761283851555,
+ "loss": 1.3117,
+ "theoretical_loss": 3.4629387478287494,
+ "tokens_seen": 1971716096
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253600802407222,
+ "loss": 1.2997,
+ "theoretical_loss": 3.4629292033411554,
+ "tokens_seen": 1971781632
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253440320962889,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4629196592596068,
+ "tokens_seen": 1971847168
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253279839518556,
+ "loss": 1.2099,
+ "theoretical_loss": 3.4629101155840725,
+ "tokens_seen": 1971912704
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003253119358074223,
+ "loss": 1.2358,
+ "theoretical_loss": 3.462900572314522,
+ "tokens_seen": 1971978240
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000325295887662989,
+ "loss": 1.2764,
+ "theoretical_loss": 3.4628910294509243,
+ "tokens_seen": 1972043776
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003252798395185557,
+ "loss": 1.3344,
+ "theoretical_loss": 3.462881486993249,
+ "tokens_seen": 1972109312
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003252637913741224,
+ "loss": 1.2796,
+ "theoretical_loss": 3.4628719449414653,
+ "tokens_seen": 1972174848
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003252477432296891,
+ "loss": 1.2834,
+ "theoretical_loss": 3.462862403295542,
+ "tokens_seen": 1972240384
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003252316950852558,
+ "loss": 1.245,
+ "theoretical_loss": 3.462852862055449,
+ "tokens_seen": 1972305920
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003252156469408225,
+ "loss": 1.2679,
+ "theoretical_loss": 3.462843321221155,
+ "tokens_seen": 1972371456
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251995987963892,
+ "loss": 1.1321,
+ "theoretical_loss": 3.4628337807926295,
+ "tokens_seen": 1972436992
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251835506519559,
+ "loss": 1.2254,
+ "theoretical_loss": 3.4628242407698417,
+ "tokens_seen": 1972502528
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251675025075226,
+ "loss": 1.3533,
+ "theoretical_loss": 3.4628147011527615,
+ "tokens_seen": 1972568064
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 660847,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.228856086730957,
+ "objective/train/theoretical_loss": 3.4628051619413567,
+ "objective/train/tokens_used": 1993093600,
+ "theoretical_loss": 3.4628051619413567,
+ "tokens_seen": 1972633600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251514543630893,
+ "loss": 1.2568,
+ "theoretical_loss": 3.4628051619413567,
+ "tokens_seen": 1972633600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000325135406218656,
+ "loss": 1.3084,
+ "theoretical_loss": 3.462795623135598,
+ "tokens_seen": 1972699136
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251193580742227,
+ "loss": 1.2835,
+ "theoretical_loss": 3.462786084735454,
+ "tokens_seen": 1972764672
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003251033099297894,
+ "loss": 1.223,
+ "theoretical_loss": 3.462776546740894,
+ "tokens_seen": 1972830208
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250872617853561,
+ "loss": 1.3382,
+ "theoretical_loss": 3.4627670091518876,
+ "tokens_seen": 1972895744
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250712136409228,
+ "loss": 1.2468,
+ "theoretical_loss": 3.4627574719684038,
+ "tokens_seen": 1972961280
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250551654964895,
+ "loss": 1.2697,
+ "theoretical_loss": 3.4627479351904116,
+ "tokens_seen": 1973026816
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250391173520562,
+ "loss": 1.2133,
+ "theoretical_loss": 3.462738398817881,
+ "tokens_seen": 1973092352
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250230692076229,
+ "loss": 1.2424,
+ "theoretical_loss": 3.4627288628507813,
+ "tokens_seen": 1973157888
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003250070210631896,
+ "loss": 1.2329,
+ "theoretical_loss": 3.4627193272890806,
+ "tokens_seen": 1973223424
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003249909729187563,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4627097921327494,
+ "tokens_seen": 1973288960
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032497492477432294,
+ "loss": 1.221,
+ "theoretical_loss": 3.4627002573817567,
+ "tokens_seen": 1973354496
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032495887662988964,
+ "loss": 1.3335,
+ "theoretical_loss": 3.462690723036072,
+ "tokens_seen": 1973420032
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032494282848545634,
+ "loss": 1.2219,
+ "theoretical_loss": 3.4626811890956644,
+ "tokens_seen": 1973485568
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032492678034102304,
+ "loss": 1.2665,
+ "theoretical_loss": 3.4626716555605026,
+ "tokens_seen": 1973551104
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003249107321965898,
+ "loss": 1.203,
+ "theoretical_loss": 3.4626621224305567,
+ "tokens_seen": 1973616640
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003248946840521565,
+ "loss": 1.3343,
+ "theoretical_loss": 3.462652589705796,
+ "tokens_seen": 1973682176
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003248786359077232,
+ "loss": 1.3397,
+ "theoretical_loss": 3.4626430573861895,
+ "tokens_seen": 1973747712
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003248625877632899,
+ "loss": 1.3358,
+ "theoretical_loss": 3.462633525471707,
+ "tokens_seen": 1973813248
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003248465396188566,
+ "loss": 1.2848,
+ "theoretical_loss": 3.462623993962317,
+ "tokens_seen": 1973878784
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003248304914744233,
+ "loss": 1.3387,
+ "theoretical_loss": 3.4626144628579896,
+ "tokens_seen": 1973944320
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032481444332999,
+ "loss": 1.3364,
+ "theoretical_loss": 3.462604932158694,
+ "tokens_seen": 1974009856
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247983951855567,
+ "loss": 1.3473,
+ "theoretical_loss": 3.4625954018643994,
+ "tokens_seen": 1974075392
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247823470411234,
+ "loss": 1.2092,
+ "theoretical_loss": 3.462585871975075,
+ "tokens_seen": 1974140928
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247662988966901,
+ "loss": 1.323,
+ "theoretical_loss": 3.4625763424906904,
+ "tokens_seen": 1974206464
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247502507522568,
+ "loss": 1.2732,
+ "theoretical_loss": 3.4625668134112146,
+ "tokens_seen": 1974272000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247342026078235,
+ "loss": 1.1487,
+ "theoretical_loss": 3.4625572847366177,
+ "tokens_seen": 1974337536
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247181544633902,
+ "loss": 1.2068,
+ "theoretical_loss": 3.4625477564668685,
+ "tokens_seen": 1974403072
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003247021063189569,
+ "loss": 1.2197,
+ "theoretical_loss": 3.462538228601937,
+ "tokens_seen": 1974468608
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003246860581745236,
+ "loss": 1.2228,
+ "theoretical_loss": 3.4625287011417907,
+ "tokens_seen": 1974534144
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003246700100300903,
+ "loss": 1.3324,
+ "theoretical_loss": 3.462519174086401,
+ "tokens_seen": 1974599680
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000324653961885657,
+ "loss": 1.299,
+ "theoretical_loss": 3.462509647435737,
+ "tokens_seen": 1974665216
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003246379137412237,
+ "loss": 1.2588,
+ "theoretical_loss": 3.462500121189767,
+ "tokens_seen": 1974730752
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003246218655967904,
+ "loss": 1.2224,
+ "theoretical_loss": 3.462490595348461,
+ "tokens_seen": 1974796288
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003246058174523571,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4624810699117887,
+ "tokens_seen": 1974861824
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245897693079238,
+ "loss": 1.3196,
+ "theoretical_loss": 3.4624715448797185,
+ "tokens_seen": 1974927360
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245737211634905,
+ "loss": 1.2767,
+ "theoretical_loss": 3.462462020252221,
+ "tokens_seen": 1974992896
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245576730190572,
+ "loss": 1.2753,
+ "theoretical_loss": 3.4624524960292655,
+ "tokens_seen": 1975058432
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245416248746239,
+ "loss": 1.424,
+ "theoretical_loss": 3.46244297221082,
+ "tokens_seen": 1975123968
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245255767301906,
+ "loss": 1.2469,
+ "theoretical_loss": 3.4624334487968556,
+ "tokens_seen": 1975189504
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003245095285857573,
+ "loss": 1.2875,
+ "theoretical_loss": 3.4624239257873404,
+ "tokens_seen": 1975255040
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000324493480441324,
+ "loss": 1.291,
+ "theoretical_loss": 3.462414403182245,
+ "tokens_seen": 1975320576
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003244774322968907,
+ "loss": 1.3391,
+ "theoretical_loss": 3.4624048809815378,
+ "tokens_seen": 1975386112
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003244613841524574,
+ "loss": 1.2151,
+ "theoretical_loss": 3.462395359185188,
+ "tokens_seen": 1975451648
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003244453360080241,
+ "loss": 1.3123,
+ "theoretical_loss": 3.4623858377931667,
+ "tokens_seen": 1975517184
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003244292878635908,
+ "loss": 1.2665,
+ "theoretical_loss": 3.4623763168054413,
+ "tokens_seen": 1975582720
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003244132397191575,
+ "loss": 1.2887,
+ "theoretical_loss": 3.462366796221983,
+ "tokens_seen": 1975648256
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003243971915747242,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4623572760427592,
+ "tokens_seen": 1975713792
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003243811434302909,
+ "loss": 1.1689,
+ "theoretical_loss": 3.462347756267741,
+ "tokens_seen": 1975779328
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003243650952858576,
+ "loss": 1.193,
+ "theoretical_loss": 3.4623382368968976,
+ "tokens_seen": 1975844864
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 661744,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3551071882247925,
+ "objective/train/theoretical_loss": 3.4623287179301983,
+ "objective/train/tokens_used": 1996370400,
+ "theoretical_loss": 3.4623287179301983,
+ "tokens_seen": 1975910400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032434904714142426,
+ "loss": 1.3324,
+ "theoretical_loss": 3.4623287179301983,
+ "tokens_seen": 1975910400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032433299899699096,
+ "loss": 1.2214,
+ "theoretical_loss": 3.462319199367612,
+ "tokens_seen": 1975975936
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032431695085255766,
+ "loss": 1.227,
+ "theoretical_loss": 3.462309681209109,
+ "tokens_seen": 1976041472
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032430090270812436,
+ "loss": 1.2567,
+ "theoretical_loss": 3.4623001634546577,
+ "tokens_seen": 1976107008
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032428485456369107,
+ "loss": 1.2492,
+ "theoretical_loss": 3.462290646104228,
+ "tokens_seen": 1976172544
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032426880641925777,
+ "loss": 1.3693,
+ "theoretical_loss": 3.46228112915779,
+ "tokens_seen": 1976238080
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032425275827482447,
+ "loss": 1.268,
+ "theoretical_loss": 3.4622716126153126,
+ "tokens_seen": 1976303616
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032423671013039117,
+ "loss": 1.2839,
+ "theoretical_loss": 3.462262096476765,
+ "tokens_seen": 1976369152
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032422066198595787,
+ "loss": 1.1754,
+ "theoretical_loss": 3.462252580742117,
+ "tokens_seen": 1976434688
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032420461384152457,
+ "loss": 1.2756,
+ "theoretical_loss": 3.462243065411338,
+ "tokens_seen": 1976500224
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032418856569709127,
+ "loss": 1.3355,
+ "theoretical_loss": 3.462233550484398,
+ "tokens_seen": 1976565760
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032417251755265797,
+ "loss": 1.3453,
+ "theoretical_loss": 3.462224035961265,
+ "tokens_seen": 1976631296
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032415646940822467,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4622145218419105,
+ "tokens_seen": 1976696832
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032414042126379137,
+ "loss": 1.3588,
+ "theoretical_loss": 3.4622050081263023,
+ "tokens_seen": 1976762368
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032412437311935807,
+ "loss": 1.3484,
+ "theoretical_loss": 3.4621954948144107,
+ "tokens_seen": 1976827904
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032410832497492477,
+ "loss": 1.1935,
+ "theoretical_loss": 3.462185981906205,
+ "tokens_seen": 1976893440
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032409227683049147,
+ "loss": 1.2972,
+ "theoretical_loss": 3.4621764694016544,
+ "tokens_seen": 1976958976
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003240762286860582,
+ "loss": 1.2619,
+ "theoretical_loss": 3.4621669573007288,
+ "tokens_seen": 1977024512
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003240601805416249,
+ "loss": 1.2888,
+ "theoretical_loss": 3.462157445603397,
+ "tokens_seen": 1977090048
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003240441323971916,
+ "loss": 1.31,
+ "theoretical_loss": 3.4621479343096304,
+ "tokens_seen": 1977155584
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003240280842527583,
+ "loss": 1.3589,
+ "theoretical_loss": 3.462138423419396,
+ "tokens_seen": 1977221120
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000324012036108325,
+ "loss": 1.2796,
+ "theoretical_loss": 3.462128912932665,
+ "tokens_seen": 1977286656
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239959879638917,
+ "loss": 1.252,
+ "theoretical_loss": 3.4621194028494062,
+ "tokens_seen": 1977352192
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239799398194584,
+ "loss": 1.3325,
+ "theoretical_loss": 3.462109893169589,
+ "tokens_seen": 1977417728
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239638916750251,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4621003838931834,
+ "tokens_seen": 1977483264
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239478435305918,
+ "loss": 1.1203,
+ "theoretical_loss": 3.462090875020159,
+ "tokens_seen": 1977548800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239317953861585,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4620813665504846,
+ "tokens_seen": 1977614336
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003239157472417252,
+ "loss": 1.2962,
+ "theoretical_loss": 3.462071858484131,
+ "tokens_seen": 1977679872
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238996990972919,
+ "loss": 1.2436,
+ "theoretical_loss": 3.462062350821066,
+ "tokens_seen": 1977745408
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238836509528586,
+ "loss": 1.2434,
+ "theoretical_loss": 3.46205284356126,
+ "tokens_seen": 1977810944
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238676028084253,
+ "loss": 1.1758,
+ "theoretical_loss": 3.4620433367046832,
+ "tokens_seen": 1977876480
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000323851554663992,
+ "loss": 1.349,
+ "theoretical_loss": 3.462033830251304,
+ "tokens_seen": 1977942016
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238355065195587,
+ "loss": 1.2886,
+ "theoretical_loss": 3.462024324201092,
+ "tokens_seen": 1978007552
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238194583751254,
+ "loss": 1.2048,
+ "theoretical_loss": 3.4620148185540183,
+ "tokens_seen": 1978073088
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003238034102306921,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4620053133100503,
+ "tokens_seen": 1978138624
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237873620862588,
+ "loss": 1.3814,
+ "theoretical_loss": 3.461995808469159,
+ "tokens_seen": 1978204160
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237713139418255,
+ "loss": 1.3033,
+ "theoretical_loss": 3.461986304031314,
+ "tokens_seen": 1978269696
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237552657973922,
+ "loss": 1.2435,
+ "theoretical_loss": 3.461976799996484,
+ "tokens_seen": 1978335232
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237392176529589,
+ "loss": 1.2223,
+ "theoretical_loss": 3.461967296364639,
+ "tokens_seen": 1978400768
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237231695085256,
+ "loss": 1.3353,
+ "theoretical_loss": 3.461957793135748,
+ "tokens_seen": 1978466304
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003237071213640923,
+ "loss": 1.2502,
+ "theoretical_loss": 3.4619482903097816,
+ "tokens_seen": 1978531840
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000323691073219659,
+ "loss": 1.2918,
+ "theoretical_loss": 3.4619387878867087,
+ "tokens_seen": 1978597376
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003236750250752257,
+ "loss": 1.3095,
+ "theoretical_loss": 3.461929285866499,
+ "tokens_seen": 1978662912
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003236589769307924,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4619197842491225,
+ "tokens_seen": 1978728448
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003236429287863591,
+ "loss": 1.2511,
+ "theoretical_loss": 3.461910283034548,
+ "tokens_seen": 1978793984
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003236268806419258,
+ "loss": 1.2603,
+ "theoretical_loss": 3.461900782222745,
+ "tokens_seen": 1978859520
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003236108324974925,
+ "loss": 1.2968,
+ "theoretical_loss": 3.461891281813685,
+ "tokens_seen": 1978925056
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003235947843530592,
+ "loss": 1.3442,
+ "theoretical_loss": 3.4618817818073344,
+ "tokens_seen": 1978990592
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003235787362086259,
+ "loss": 1.3922,
+ "theoretical_loss": 3.461872282203666,
+ "tokens_seen": 1979056128
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003235626880641926,
+ "loss": 1.1998,
+ "theoretical_loss": 3.461862783002647,
+ "tokens_seen": 1979121664
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 662851,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.253308892250061,
+ "objective/train/theoretical_loss": 3.461853284204248,
+ "objective/train/tokens_used": 1999647200,
+ "theoretical_loss": 3.461853284204248,
+ "tokens_seen": 1979187200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003235466399197593,
+ "loss": 1.2605,
+ "theoretical_loss": 3.461853284204248,
+ "tokens_seen": 1979187200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000323530591775326,
+ "loss": 1.3037,
+ "theoretical_loss": 3.461843785808439,
+ "tokens_seen": 1979252736
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003235145436308927,
+ "loss": 1.3261,
+ "theoretical_loss": 3.461834287815189,
+ "tokens_seen": 1979318272
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234984954864594,
+ "loss": 1.3114,
+ "theoretical_loss": 3.461824790224468,
+ "tokens_seen": 1979383808
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234824473420261,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4618152930362447,
+ "tokens_seen": 1979449344
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234663991975928,
+ "loss": 1.2895,
+ "theoretical_loss": 3.46180579625049,
+ "tokens_seen": 1979514880
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234503510531595,
+ "loss": 1.3307,
+ "theoretical_loss": 3.4617962998671725,
+ "tokens_seen": 1979580416
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234343029087262,
+ "loss": 1.3148,
+ "theoretical_loss": 3.461786803886263,
+ "tokens_seen": 1979645952
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234182547642929,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4617773083077297,
+ "tokens_seen": 1979711488
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003234022066198596,
+ "loss": 1.1496,
+ "theoretical_loss": 3.461767813131543,
+ "tokens_seen": 1979777024
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003233861584754263,
+ "loss": 1.3028,
+ "theoretical_loss": 3.4617583183576723,
+ "tokens_seen": 1979842560
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000323370110330993,
+ "loss": 1.2057,
+ "theoretical_loss": 3.4617488239860874,
+ "tokens_seen": 1979908096
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003233540621865597,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4617393300167585,
+ "tokens_seen": 1979973632
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003233380140421264,
+ "loss": 1.1871,
+ "theoretical_loss": 3.4617298364496536,
+ "tokens_seen": 1980039168
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003233219658976931,
+ "loss": 1.1707,
+ "theoretical_loss": 3.4617203432847443,
+ "tokens_seen": 1980104704
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003233059177532598,
+ "loss": 1.2787,
+ "theoretical_loss": 3.461710850521999,
+ "tokens_seen": 1980170240
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003232898696088265,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4617013581613882,
+ "tokens_seen": 1980235776
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003232738214643932,
+ "loss": 1.2491,
+ "theoretical_loss": 3.46169186620288,
+ "tokens_seen": 1980301312
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003232577733199599,
+ "loss": 1.2275,
+ "theoretical_loss": 3.461682374646446,
+ "tokens_seen": 1980366848
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003232417251755266,
+ "loss": 1.3797,
+ "theoretical_loss": 3.4616728834920547,
+ "tokens_seen": 1980432384
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003232256770310933,
+ "loss": 1.2025,
+ "theoretical_loss": 3.461663392739676,
+ "tokens_seen": 1980497920
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032320962888666,
+ "loss": 1.2285,
+ "theoretical_loss": 3.46165390238928,
+ "tokens_seen": 1980563456
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231935807422267,
+ "loss": 1.2729,
+ "theoretical_loss": 3.461644412440836,
+ "tokens_seen": 1980628992
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231775325977934,
+ "loss": 1.2762,
+ "theoretical_loss": 3.461634922894313,
+ "tokens_seen": 1980694528
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231614844533601,
+ "loss": 1.1996,
+ "theoretical_loss": 3.4616254337496812,
+ "tokens_seen": 1980760064
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231454363089268,
+ "loss": 1.1993,
+ "theoretical_loss": 3.461615945006911,
+ "tokens_seen": 1980825600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231293881644935,
+ "loss": 1.2744,
+ "theoretical_loss": 3.4616064566659714,
+ "tokens_seen": 1980891136
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003231133400200602,
+ "loss": 1.2419,
+ "theoretical_loss": 3.4615969687268326,
+ "tokens_seen": 1980956672
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230972918756269,
+ "loss": 1.194,
+ "theoretical_loss": 3.461587481189464,
+ "tokens_seen": 1981022208
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230812437311936,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4615779940538345,
+ "tokens_seen": 1981087744
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230651955867603,
+ "loss": 1.2812,
+ "theoretical_loss": 3.4615685073199147,
+ "tokens_seen": 1981153280
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000323049147442327,
+ "loss": 1.2192,
+ "theoretical_loss": 3.461559020987674,
+ "tokens_seen": 1981218816
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230330992978937,
+ "loss": 1.346,
+ "theoretical_loss": 3.461549535057083,
+ "tokens_seen": 1981284352
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230170511534604,
+ "loss": 1.297,
+ "theoretical_loss": 3.46154004952811,
+ "tokens_seen": 1981349888
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003230010030090271,
+ "loss": 1.282,
+ "theoretical_loss": 3.461530564400725,
+ "tokens_seen": 1981415424
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003229849548645938,
+ "loss": 1.3516,
+ "theoretical_loss": 3.4615210796748985,
+ "tokens_seen": 1981480960
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003229689067201605,
+ "loss": 1.2885,
+ "theoretical_loss": 3.4615115953505997,
+ "tokens_seen": 1981546496
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032295285857572716,
+ "loss": 1.3675,
+ "theoretical_loss": 3.4615021114277984,
+ "tokens_seen": 1981612032
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032293681043129386,
+ "loss": 1.203,
+ "theoretical_loss": 3.4614926279064644,
+ "tokens_seen": 1981677568
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032292076228686056,
+ "loss": 1.4016,
+ "theoretical_loss": 3.461483144786567,
+ "tokens_seen": 1981743104
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032290471414242727,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4614736620680766,
+ "tokens_seen": 1981808640
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032288866599799397,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4614641797509624,
+ "tokens_seen": 1981874176
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032287261785356067,
+ "loss": 1.2891,
+ "theoretical_loss": 3.4614546978351948,
+ "tokens_seen": 1981939712
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032285656970912737,
+ "loss": 1.2287,
+ "theoretical_loss": 3.4614452163207425,
+ "tokens_seen": 1982005248
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032284052156469407,
+ "loss": 1.3051,
+ "theoretical_loss": 3.4614357352075764,
+ "tokens_seen": 1982070784
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032282447342026077,
+ "loss": 1.2015,
+ "theoretical_loss": 3.4614262544956653,
+ "tokens_seen": 1982136320
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032280842527582747,
+ "loss": 1.2178,
+ "theoretical_loss": 3.461416774184979,
+ "tokens_seen": 1982201856
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032279237713139417,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4614072942754883,
+ "tokens_seen": 1982267392
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032277632898696087,
+ "loss": 1.2974,
+ "theoretical_loss": 3.461397814767162,
+ "tokens_seen": 1982332928
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032276028084252757,
+ "loss": 1.3706,
+ "theoretical_loss": 3.4613883356599704,
+ "tokens_seen": 1982398464
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 663739,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3018696308135986,
+ "objective/train/theoretical_loss": 3.461378856953883,
+ "objective/train/tokens_used": 2002924000,
+ "theoretical_loss": 3.461378856953883,
+ "tokens_seen": 1982464000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032274423269809427,
+ "loss": 1.3068,
+ "theoretical_loss": 3.461378856953883,
+ "tokens_seen": 1982464000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032272818455366097,
+ "loss": 1.1333,
+ "theoretical_loss": 3.461369378648869,
+ "tokens_seen": 1982529536
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003227121364092277,
+ "loss": 1.2512,
+ "theoretical_loss": 3.461359900744899,
+ "tokens_seen": 1982595072
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226960882647944,
+ "loss": 1.3132,
+ "theoretical_loss": 3.4613504232419423,
+ "tokens_seen": 1982660608
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226800401203611,
+ "loss": 1.2537,
+ "theoretical_loss": 3.461340946139969,
+ "tokens_seen": 1982726144
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226639919759278,
+ "loss": 1.3256,
+ "theoretical_loss": 3.461331469438949,
+ "tokens_seen": 1982791680
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226479438314945,
+ "loss": 1.298,
+ "theoretical_loss": 3.461321993138852,
+ "tokens_seen": 1982857216
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226318956870612,
+ "loss": 1.2358,
+ "theoretical_loss": 3.461312517239647,
+ "tokens_seen": 1982922752
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003226158475426279,
+ "loss": 1.273,
+ "theoretical_loss": 3.4613030417413047,
+ "tokens_seen": 1982988288
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003225997993981946,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4612935666437945,
+ "tokens_seen": 1983053824
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003225837512537613,
+ "loss": 1.259,
+ "theoretical_loss": 3.4612840919470864,
+ "tokens_seen": 1983119360
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000322567703109328,
+ "loss": 1.2519,
+ "theoretical_loss": 3.46127461765115,
+ "tokens_seen": 1983184896
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003225516549648947,
+ "loss": 1.2249,
+ "theoretical_loss": 3.461265143755956,
+ "tokens_seen": 1983250432
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003225356068204614,
+ "loss": 1.2658,
+ "theoretical_loss": 3.4612556702614725,
+ "tokens_seen": 1983315968
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032251955867602813,
+ "loss": 1.2707,
+ "theoretical_loss": 3.4612461971676707,
+ "tokens_seen": 1983381504
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032250351053159484,
+ "loss": 1.3275,
+ "theoretical_loss": 3.46123672447452,
+ "tokens_seen": 1983447040
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032248746238716154,
+ "loss": 1.3517,
+ "theoretical_loss": 3.46122725218199,
+ "tokens_seen": 1983512576
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032247141424272824,
+ "loss": 1.201,
+ "theoretical_loss": 3.461217780290051,
+ "tokens_seen": 1983578112
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032245536609829494,
+ "loss": 1.3523,
+ "theoretical_loss": 3.461208308798672,
+ "tokens_seen": 1983643648
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032243931795386164,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4611988377078236,
+ "tokens_seen": 1983709184
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032242326980942834,
+ "loss": 1.2372,
+ "theoretical_loss": 3.4611893670174756,
+ "tokens_seen": 1983774720
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032240722166499504,
+ "loss": 1.1743,
+ "theoretical_loss": 3.4611798967275975,
+ "tokens_seen": 1983840256
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032239117352056174,
+ "loss": 1.1264,
+ "theoretical_loss": 3.4611704268381587,
+ "tokens_seen": 1983905792
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032237512537612844,
+ "loss": 1.2904,
+ "theoretical_loss": 3.46116095734913,
+ "tokens_seen": 1983971328
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032235907723169514,
+ "loss": 1.3452,
+ "theoretical_loss": 3.461151488260481,
+ "tokens_seen": 1984036864
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032234302908726184,
+ "loss": 1.2573,
+ "theoretical_loss": 3.461142019572182,
+ "tokens_seen": 1984102400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032232698094282854,
+ "loss": 1.2881,
+ "theoretical_loss": 3.461132551284201,
+ "tokens_seen": 1984167936
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003223109327983952,
+ "loss": 1.2507,
+ "theoretical_loss": 3.46112308339651,
+ "tokens_seen": 1984233472
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003222948846539619,
+ "loss": 1.3358,
+ "theoretical_loss": 3.461113615909077,
+ "tokens_seen": 1984299008
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003222788365095286,
+ "loss": 1.2354,
+ "theoretical_loss": 3.4611041488218737,
+ "tokens_seen": 1984364544
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003222627883650953,
+ "loss": 1.3219,
+ "theoretical_loss": 3.4610946821348687,
+ "tokens_seen": 1984430080
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000322246740220662,
+ "loss": 1.2491,
+ "theoretical_loss": 3.461085215848032,
+ "tokens_seen": 1984495616
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003222306920762287,
+ "loss": 1.3243,
+ "theoretical_loss": 3.461075749961334,
+ "tokens_seen": 1984561152
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003222146439317954,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4610662844747444,
+ "tokens_seen": 1984626688
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221985957873621,
+ "loss": 1.3372,
+ "theoretical_loss": 3.461056819388233,
+ "tokens_seen": 1984692224
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221825476429288,
+ "loss": 1.2272,
+ "theoretical_loss": 3.461047354701769,
+ "tokens_seen": 1984757760
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221664994984955,
+ "loss": 1.2722,
+ "theoretical_loss": 3.461037890415324,
+ "tokens_seen": 1984823296
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221504513540622,
+ "loss": 1.2487,
+ "theoretical_loss": 3.4610284265288653,
+ "tokens_seen": 1984888832
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221344032096289,
+ "loss": 1.2598,
+ "theoretical_loss": 3.4610189630423656,
+ "tokens_seen": 1984954368
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221183550651956,
+ "loss": 1.3557,
+ "theoretical_loss": 3.461009499955793,
+ "tokens_seen": 1985019904
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003221023069207623,
+ "loss": 1.32,
+ "theoretical_loss": 3.461000037269118,
+ "tokens_seen": 1985085440
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000322086258776329,
+ "loss": 1.3916,
+ "theoretical_loss": 3.4609905749823104,
+ "tokens_seen": 1985150976
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003220702106318957,
+ "loss": 1.3763,
+ "theoretical_loss": 3.4609811130953396,
+ "tokens_seen": 1985216512
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003220541624874624,
+ "loss": 1.1434,
+ "theoretical_loss": 3.4609716516081765,
+ "tokens_seen": 1985282048
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003220381143430291,
+ "loss": 1.2956,
+ "theoretical_loss": 3.4609621905207906,
+ "tokens_seen": 1985347584
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003220220661985958,
+ "loss": 1.1931,
+ "theoretical_loss": 3.460952729833151,
+ "tokens_seen": 1985413120
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003220060180541625,
+ "loss": 1.3251,
+ "theoretical_loss": 3.460943269545229,
+ "tokens_seen": 1985478656
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003219899699097292,
+ "loss": 1.3677,
+ "theoretical_loss": 3.4609338096569937,
+ "tokens_seen": 1985544192
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003219739217652959,
+ "loss": 1.3312,
+ "theoretical_loss": 3.4609243501684155,
+ "tokens_seen": 1985609728
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003219578736208626,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4609148910794634,
+ "tokens_seen": 1985675264
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 664892,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1819332838058472,
+ "objective/train/theoretical_loss": 3.460905432390108,
+ "objective/train/tokens_used": 2006200800,
+ "theoretical_loss": 3.460905432390108,
+ "tokens_seen": 1985740800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003219418254764293,
+ "loss": 1.2115,
+ "theoretical_loss": 3.460905432390108,
+ "tokens_seen": 1985740800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000321925777331996,
+ "loss": 1.2688,
+ "theoretical_loss": 3.4608959741003193,
+ "tokens_seen": 1985806336
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003219097291875627,
+ "loss": 1.282,
+ "theoretical_loss": 3.460886516210067,
+ "tokens_seen": 1985871872
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218936810431294,
+ "loss": 1.171,
+ "theoretical_loss": 3.460877058719322,
+ "tokens_seen": 1985937408
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218776328986961,
+ "loss": 1.3415,
+ "theoretical_loss": 3.460867601628052,
+ "tokens_seen": 1986002944
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218615847542628,
+ "loss": 1.333,
+ "theoretical_loss": 3.4608581449362292,
+ "tokens_seen": 1986068480
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218455366098295,
+ "loss": 1.2052,
+ "theoretical_loss": 3.4608486886438223,
+ "tokens_seen": 1986134016
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218294884653962,
+ "loss": 1.1502,
+ "theoretical_loss": 3.4608392327508017,
+ "tokens_seen": 1986199552
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003218134403209629,
+ "loss": 1.1916,
+ "theoretical_loss": 3.4608297772571373,
+ "tokens_seen": 1986265088
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217973921765296,
+ "loss": 1.1894,
+ "theoretical_loss": 3.4608203221627987,
+ "tokens_seen": 1986330624
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217813440320963,
+ "loss": 1.2802,
+ "theoretical_loss": 3.4608108674677567,
+ "tokens_seen": 1986396160
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000321765295887663,
+ "loss": 1.3686,
+ "theoretical_loss": 3.4608014131719806,
+ "tokens_seen": 1986461696
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217492477432297,
+ "loss": 1.4064,
+ "theoretical_loss": 3.460791959275441,
+ "tokens_seen": 1986527232
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217331995987964,
+ "loss": 1.3098,
+ "theoretical_loss": 3.4607825057781065,
+ "tokens_seen": 1986592768
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217171514543631,
+ "loss": 1.2966,
+ "theoretical_loss": 3.460773052679948,
+ "tokens_seen": 1986658304
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003217011033099298,
+ "loss": 1.1314,
+ "theoretical_loss": 3.460763599980936,
+ "tokens_seen": 1986723840
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003216850551654965,
+ "loss": 1.1984,
+ "theoretical_loss": 3.4607541476810395,
+ "tokens_seen": 1986789376
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003216690070210632,
+ "loss": 1.2855,
+ "theoretical_loss": 3.460744695780229,
+ "tokens_seen": 1986854912
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003216529588766299,
+ "loss": 1.2458,
+ "theoretical_loss": 3.460735244278475,
+ "tokens_seen": 1986920448
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003216369107321966,
+ "loss": 1.3415,
+ "theoretical_loss": 3.4607257931757465,
+ "tokens_seen": 1986985984
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003216208625877633,
+ "loss": 1.1949,
+ "theoretical_loss": 3.4607163424720135,
+ "tokens_seen": 1987051520
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032160481444333,
+ "loss": 1.2702,
+ "theoretical_loss": 3.460706892167247,
+ "tokens_seen": 1987117056
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003215887662988967,
+ "loss": 1.2309,
+ "theoretical_loss": 3.4606974422614156,
+ "tokens_seen": 1987182592
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003215727181544634,
+ "loss": 1.3673,
+ "theoretical_loss": 3.4606879927544907,
+ "tokens_seen": 1987248128
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003215566700100301,
+ "loss": 1.3188,
+ "theoretical_loss": 3.4606785436464413,
+ "tokens_seen": 1987313664
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032154062186559677,
+ "loss": 1.2936,
+ "theoretical_loss": 3.460669094937238,
+ "tokens_seen": 1987379200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032152457372116347,
+ "loss": 1.29,
+ "theoretical_loss": 3.460659646626851,
+ "tokens_seen": 1987444736
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032150852557673017,
+ "loss": 1.2297,
+ "theoretical_loss": 3.4606501987152494,
+ "tokens_seen": 1987510272
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032149247743229687,
+ "loss": 1.2765,
+ "theoretical_loss": 3.460640751202404,
+ "tokens_seen": 1987575808
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032147642928786357,
+ "loss": 1.2303,
+ "theoretical_loss": 3.4606313040882846,
+ "tokens_seen": 1987641344
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032146038114343027,
+ "loss": 1.2595,
+ "theoretical_loss": 3.460621857372861,
+ "tokens_seen": 1987706880
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032144433299899697,
+ "loss": 1.3184,
+ "theoretical_loss": 3.460612411056103,
+ "tokens_seen": 1987772416
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032142828485456367,
+ "loss": 1.2134,
+ "theoretical_loss": 3.4606029651379817,
+ "tokens_seen": 1987837952
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032141223671013037,
+ "loss": 1.3586,
+ "theoretical_loss": 3.4605935196184667,
+ "tokens_seen": 1987903488
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032139618856569707,
+ "loss": 1.217,
+ "theoretical_loss": 3.460584074497527,
+ "tokens_seen": 1987969024
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032138014042126377,
+ "loss": 1.1311,
+ "theoretical_loss": 3.4605746297751345,
+ "tokens_seen": 1988034560
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032136409227683047,
+ "loss": 1.1807,
+ "theoretical_loss": 3.460565185451258,
+ "tokens_seen": 1988100096
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032134804413239717,
+ "loss": 1.1941,
+ "theoretical_loss": 3.4605557415258668,
+ "tokens_seen": 1988165632
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003213319959879639,
+ "loss": 1.2327,
+ "theoretical_loss": 3.460546297998933,
+ "tokens_seen": 1988231168
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032131594784353063,
+ "loss": 1.2142,
+ "theoretical_loss": 3.460536854870425,
+ "tokens_seen": 1988296704
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032129989969909733,
+ "loss": 1.367,
+ "theoretical_loss": 3.460527412140314,
+ "tokens_seen": 1988362240
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032128385155466403,
+ "loss": 1.2278,
+ "theoretical_loss": 3.460517969808569,
+ "tokens_seen": 1988427776
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032126780341023073,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4605085278751604,
+ "tokens_seen": 1988493312
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032125175526579743,
+ "loss": 1.2688,
+ "theoretical_loss": 3.4604990863400586,
+ "tokens_seen": 1988558848
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032123570712136413,
+ "loss": 1.3267,
+ "theoretical_loss": 3.460489645203234,
+ "tokens_seen": 1988624384
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032121965897693083,
+ "loss": 1.2963,
+ "theoretical_loss": 3.4604802044646554,
+ "tokens_seen": 1988689920
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032120361083249753,
+ "loss": 1.2214,
+ "theoretical_loss": 3.460470764124294,
+ "tokens_seen": 1988755456
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032118756268806423,
+ "loss": 1.2053,
+ "theoretical_loss": 3.46046132418212,
+ "tokens_seen": 1988820992
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032117151454363093,
+ "loss": 1.1426,
+ "theoretical_loss": 3.4604518846381023,
+ "tokens_seen": 1988886528
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032115546639919763,
+ "loss": 1.2487,
+ "theoretical_loss": 3.460442445492212,
+ "tokens_seen": 1988952064
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 665696,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3255927562713623,
+ "objective/train/theoretical_loss": 3.460433006744419,
+ "objective/train/tokens_used": 2009477600,
+ "theoretical_loss": 3.460433006744419,
+ "tokens_seen": 1989017600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032113941825476434,
+ "loss": 1.2721,
+ "theoretical_loss": 3.460433006744419,
+ "tokens_seen": 1989017600
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032112337011033104,
+ "loss": 1.2777,
+ "theoretical_loss": 3.460423568394693,
+ "tokens_seen": 1989083136
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032110732196589774,
+ "loss": 1.3445,
+ "theoretical_loss": 3.4604141304430045,
+ "tokens_seen": 1989148672
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032109127382146444,
+ "loss": 1.3093,
+ "theoretical_loss": 3.4604046928893237,
+ "tokens_seen": 1989214208
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032107522567703114,
+ "loss": 1.2275,
+ "theoretical_loss": 3.46039525573362,
+ "tokens_seen": 1989279744
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032105917753259784,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4603858189758645,
+ "tokens_seen": 1989345280
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032104312938816454,
+ "loss": 1.1227,
+ "theoretical_loss": 3.460376382616027,
+ "tokens_seen": 1989410816
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032102708124373124,
+ "loss": 1.2227,
+ "theoretical_loss": 3.4603669466540774,
+ "tokens_seen": 1989476352
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032101103309929794,
+ "loss": 1.3095,
+ "theoretical_loss": 3.4603575110899856,
+ "tokens_seen": 1989541888
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032099498495486464,
+ "loss": 1.269,
+ "theoretical_loss": 3.4603480759237217,
+ "tokens_seen": 1989607424
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032097893681043134,
+ "loss": 1.1718,
+ "theoretical_loss": 3.460338641155256,
+ "tokens_seen": 1989672960
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032096288866599804,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4603292067845595,
+ "tokens_seen": 1989738496
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032094684052156474,
+ "loss": 1.2095,
+ "theoretical_loss": 3.460319772811601,
+ "tokens_seen": 1989804032
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032093079237713144,
+ "loss": 1.2088,
+ "theoretical_loss": 3.4603103392363512,
+ "tokens_seen": 1989869568
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003209147442326981,
+ "loss": 1.2959,
+ "theoretical_loss": 3.46030090605878,
+ "tokens_seen": 1989935104
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208986960882648,
+ "loss": 1.1243,
+ "theoretical_loss": 3.4602914732788586,
+ "tokens_seen": 1990000640
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208826479438315,
+ "loss": 1.2151,
+ "theoretical_loss": 3.4602820408965558,
+ "tokens_seen": 1990066176
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208665997993982,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4602726089118416,
+ "tokens_seen": 1990131712
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208505516549649,
+ "loss": 1.2606,
+ "theoretical_loss": 3.4602631773246877,
+ "tokens_seen": 1990197248
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208345035105316,
+ "loss": 1.2455,
+ "theoretical_loss": 3.460253746135063,
+ "tokens_seen": 1990262784
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003208184553660983,
+ "loss": 1.2167,
+ "theoretical_loss": 3.460244315342938,
+ "tokens_seen": 1990328320
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000320802407221665,
+ "loss": 1.178,
+ "theoretical_loss": 3.4602348849482825,
+ "tokens_seen": 1990393856
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207863590772317,
+ "loss": 1.3953,
+ "theoretical_loss": 3.4602254549510674,
+ "tokens_seen": 1990459392
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207703109327984,
+ "loss": 1.2184,
+ "theoretical_loss": 3.460216025351263,
+ "tokens_seen": 1990524928
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207542627883651,
+ "loss": 1.1341,
+ "theoretical_loss": 3.460206596148838,
+ "tokens_seen": 1990590464
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207382146439318,
+ "loss": 1.1911,
+ "theoretical_loss": 3.4601971673437637,
+ "tokens_seen": 1990656000
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207221664994985,
+ "loss": 1.3415,
+ "theoretical_loss": 3.4601877389360105,
+ "tokens_seen": 1990721536
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003207061183550652,
+ "loss": 1.3122,
+ "theoretical_loss": 3.4601783109255484,
+ "tokens_seen": 1990787072
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003206900702106319,
+ "loss": 1.3456,
+ "theoretical_loss": 3.4601688833123467,
+ "tokens_seen": 1990852608
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003206740220661986,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4601594560963767,
+ "tokens_seen": 1990918144
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003206579739217653,
+ "loss": 1.1944,
+ "theoretical_loss": 3.460150029277608,
+ "tokens_seen": 1990983680
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000320641925777332,
+ "loss": 1.3037,
+ "theoretical_loss": 3.4601406028560104,
+ "tokens_seen": 1991049216
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003206258776328987,
+ "loss": 1.2505,
+ "theoretical_loss": 3.4601311768315552,
+ "tokens_seen": 1991114752
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003206098294884654,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4601217512042117,
+ "tokens_seen": 1991180288
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205937813440321,
+ "loss": 1.2272,
+ "theoretical_loss": 3.460112325973951,
+ "tokens_seen": 1991245824
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205777331995988,
+ "loss": 1.4319,
+ "theoretical_loss": 3.4601029011407425,
+ "tokens_seen": 1991311360
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205616850551655,
+ "loss": 1.3005,
+ "theoretical_loss": 3.4600934767045564,
+ "tokens_seen": 1991376896
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205456369107322,
+ "loss": 1.2495,
+ "theoretical_loss": 3.460084052665363,
+ "tokens_seen": 1991442432
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205295887662989,
+ "loss": 1.3024,
+ "theoretical_loss": 3.460074629023133,
+ "tokens_seen": 1991507968
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003205135406218656,
+ "loss": 1.2622,
+ "theoretical_loss": 3.460065205777836,
+ "tokens_seen": 1991573504
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204974924774323,
+ "loss": 1.2789,
+ "theoretical_loss": 3.4600557829294427,
+ "tokens_seen": 1991639040
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000320481444332999,
+ "loss": 1.1847,
+ "theoretical_loss": 3.460046360477923,
+ "tokens_seen": 1991704576
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204653961885657,
+ "loss": 1.1588,
+ "theoretical_loss": 3.460036938423247,
+ "tokens_seen": 1991770112
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204493480441324,
+ "loss": 1.3105,
+ "theoretical_loss": 3.4600275167653853,
+ "tokens_seen": 1991835648
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204332998996991,
+ "loss": 1.3075,
+ "theoretical_loss": 3.4600180955043083,
+ "tokens_seen": 1991901184
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204172517552658,
+ "loss": 1.3275,
+ "theoretical_loss": 3.4600086746399854,
+ "tokens_seen": 1991966720
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003204012036108325,
+ "loss": 1.1806,
+ "theoretical_loss": 3.459999254172388,
+ "tokens_seen": 1992032256
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003203851554663992,
+ "loss": 1.204,
+ "theoretical_loss": 3.459989834101485,
+ "tokens_seen": 1992097792
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003203691073219659,
+ "loss": 1.2993,
+ "theoretical_loss": 3.459980414427248,
+ "tokens_seen": 1992163328
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003203530591775326,
+ "loss": 1.2179,
+ "theoretical_loss": 3.4599709951496465,
+ "tokens_seen": 1992228864
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 666903,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2762541770935059,
+ "objective/train/theoretical_loss": 3.4599615762686504,
+ "objective/train/tokens_used": 2012754400,
+ "theoretical_loss": 3.4599615762686504,
+ "tokens_seen": 1992294400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003203370110330993,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4599615762686504,
+ "tokens_seen": 1992294400
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000320320962888666,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4599521577842305,
+ "tokens_seen": 1992359936
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003203049147442327,
+ "loss": 1.2236,
+ "theoretical_loss": 3.4599427396963573,
+ "tokens_seen": 1992425472
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202888665997994,
+ "loss": 1.3269,
+ "theoretical_loss": 3.459933322005001,
+ "tokens_seen": 1992491008
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202728184553661,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4599239047101307,
+ "tokens_seen": 1992556544
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202567703109328,
+ "loss": 1.2264,
+ "theoretical_loss": 3.4599144878117185,
+ "tokens_seen": 1992622080
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202407221664995,
+ "loss": 1.2591,
+ "theoretical_loss": 3.4599050713097332,
+ "tokens_seen": 1992687616
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202246740220662,
+ "loss": 1.2588,
+ "theoretical_loss": 3.4598956552041455,
+ "tokens_seen": 1992753152
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003202086258776329,
+ "loss": 1.2807,
+ "theoretical_loss": 3.459886239494926,
+ "tokens_seen": 1992818688
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003201925777331996,
+ "loss": 1.1525,
+ "theoretical_loss": 3.4598768241820452,
+ "tokens_seen": 1992884224
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003201765295887663,
+ "loss": 1.2788,
+ "theoretical_loss": 3.459867409265472,
+ "tokens_seen": 1992949760
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000320160481444333,
+ "loss": 1.2724,
+ "theoretical_loss": 3.4598579947451786,
+ "tokens_seen": 1993015296
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032014443329989967,
+ "loss": 1.3001,
+ "theoretical_loss": 3.459848580621134,
+ "tokens_seen": 1993080832
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032012838515546637,
+ "loss": 1.1552,
+ "theoretical_loss": 3.4598391668933086,
+ "tokens_seen": 1993146368
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003201123370110331,
+ "loss": 1.2294,
+ "theoretical_loss": 3.4598297535616735,
+ "tokens_seen": 1993211904
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200962888665998,
+ "loss": 1.3142,
+ "theoretical_loss": 3.459820340626198,
+ "tokens_seen": 1993277440
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200802407221665,
+ "loss": 1.266,
+ "theoretical_loss": 3.4598109280868528,
+ "tokens_seen": 1993342976
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200641925777332,
+ "loss": 1.2689,
+ "theoretical_loss": 3.459801515943608,
+ "tokens_seen": 1993408512
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200481444332999,
+ "loss": 1.1802,
+ "theoretical_loss": 3.4597921041964343,
+ "tokens_seen": 1993474048
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200320962888666,
+ "loss": 1.2232,
+ "theoretical_loss": 3.4597826928453017,
+ "tokens_seen": 1993539584
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003200160481444333,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4597732818901816,
+ "tokens_seen": 1993605120
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00032,
+ "loss": 1.2034,
+ "theoretical_loss": 3.4597638713310426,
+ "tokens_seen": 1993670656
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003199839518555667,
+ "loss": 1.213,
+ "theoretical_loss": 3.459754461167856,
+ "tokens_seen": 1993736192
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031996790371113343,
+ "loss": 1.2727,
+ "theoretical_loss": 3.4597450514005916,
+ "tokens_seen": 1993801728
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031995185556670013,
+ "loss": 1.3125,
+ "theoretical_loss": 3.4597356420292202,
+ "tokens_seen": 1993867264
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031993580742226683,
+ "loss": 1.2843,
+ "theoretical_loss": 3.459726233053712,
+ "tokens_seen": 1993932800
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031991975927783353,
+ "loss": 1.1092,
+ "theoretical_loss": 3.4597168244740377,
+ "tokens_seen": 1993998336
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031990371113340023,
+ "loss": 1.2958,
+ "theoretical_loss": 3.459707416290167,
+ "tokens_seen": 1994063872
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031988766298896693,
+ "loss": 1.3187,
+ "theoretical_loss": 3.4596980085020705,
+ "tokens_seen": 1994129408
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031987161484453363,
+ "loss": 1.3156,
+ "theoretical_loss": 3.459688601109719,
+ "tokens_seen": 1994194944
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031985556670010033,
+ "loss": 1.2592,
+ "theoretical_loss": 3.459679194113082,
+ "tokens_seen": 1994260480
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031983951855566703,
+ "loss": 1.2869,
+ "theoretical_loss": 3.45966978751213,
+ "tokens_seen": 1994326016
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031982347041123373,
+ "loss": 1.1774,
+ "theoretical_loss": 3.4596603813068345,
+ "tokens_seen": 1994391552
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031980742226680043,
+ "loss": 1.2784,
+ "theoretical_loss": 3.459650975497164,
+ "tokens_seen": 1994457088
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031979137412236713,
+ "loss": 1.1861,
+ "theoretical_loss": 3.4596415700830905,
+ "tokens_seen": 1994522624
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031977532597793384,
+ "loss": 1.2795,
+ "theoretical_loss": 3.4596321650645834,
+ "tokens_seen": 1994588160
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031975927783350054,
+ "loss": 1.2154,
+ "theoretical_loss": 3.459622760441613,
+ "tokens_seen": 1994653696
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031974322968906724,
+ "loss": 1.2445,
+ "theoretical_loss": 3.4596133562141507,
+ "tokens_seen": 1994719232
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031972718154463394,
+ "loss": 1.2929,
+ "theoretical_loss": 3.459603952382166,
+ "tokens_seen": 1994784768
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031971113340020064,
+ "loss": 1.2318,
+ "theoretical_loss": 3.4595945489456295,
+ "tokens_seen": 1994850304
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031969508525576734,
+ "loss": 1.3232,
+ "theoretical_loss": 3.4595851459045113,
+ "tokens_seen": 1994915840
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031967903711133404,
+ "loss": 1.2139,
+ "theoretical_loss": 3.459575743258782,
+ "tokens_seen": 1994981376
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031966298896690074,
+ "loss": 1.1856,
+ "theoretical_loss": 3.4595663410084123,
+ "tokens_seen": 1995046912
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031964694082246744,
+ "loss": 1.2733,
+ "theoretical_loss": 3.4595569391533725,
+ "tokens_seen": 1995112448
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031963089267803414,
+ "loss": 1.2606,
+ "theoretical_loss": 3.4595475376936324,
+ "tokens_seen": 1995177984
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031961484453360084,
+ "loss": 1.2216,
+ "theoretical_loss": 3.4595381366291633,
+ "tokens_seen": 1995243520
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031959879638916754,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4595287359599345,
+ "tokens_seen": 1995309056
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031958274824473424,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4595193356859175,
+ "tokens_seen": 1995374592
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031956670010030094,
+ "loss": 1.362,
+ "theoretical_loss": 3.459509935807082,
+ "tokens_seen": 1995440128
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031955065195586764,
+ "loss": 1.1668,
+ "theoretical_loss": 3.4595005363233984,
+ "tokens_seen": 1995505664
+ },
+ {
+ "epoch": 0.6,
+ "objective/train/docs_used": 667918,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2358568906784058,
+ "objective/train/theoretical_loss": 3.4594911372348376,
+ "objective/train/tokens_used": 2016031200,
+ "theoretical_loss": 3.4594911372348376,
+ "tokens_seen": 1995571200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031953460381143434,
+ "loss": 1.2884,
+ "theoretical_loss": 3.4594911372348376,
+ "tokens_seen": 1995571200
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.00031951855566700105,
+ "loss": 1.211,
+ "theoretical_loss": 3.45948173854137,
+ "tokens_seen": 1995636736
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003195025075225677,
+ "loss": 1.248,
+ "theoretical_loss": 3.4594723402429657,
+ "tokens_seen": 1995702272
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194864593781344,
+ "loss": 1.3623,
+ "theoretical_loss": 3.459462942339595,
+ "tokens_seen": 1995767808
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194704112337011,
+ "loss": 1.1604,
+ "theoretical_loss": 3.459453544831229,
+ "tokens_seen": 1995833344
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194543630892678,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4594441477178366,
+ "tokens_seen": 1995898880
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194383149448345,
+ "loss": 1.167,
+ "theoretical_loss": 3.45943475099939,
+ "tokens_seen": 1995964416
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194222668004012,
+ "loss": 1.3237,
+ "theoretical_loss": 3.459425354675859,
+ "tokens_seen": 1996029952
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003194062186559679,
+ "loss": 1.2519,
+ "theoretical_loss": 3.459415958747214,
+ "tokens_seen": 1996095488
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003193901705115346,
+ "loss": 1.3209,
+ "theoretical_loss": 3.4594065632134248,
+ "tokens_seen": 1996161024
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003193741223671013,
+ "loss": 1.3826,
+ "theoretical_loss": 3.459397168074463,
+ "tokens_seen": 1996226560
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.000319358074222668,
+ "loss": 1.2225,
+ "theoretical_loss": 3.459387773330298,
+ "tokens_seen": 1996292096
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003193420260782347,
+ "loss": 1.41,
+ "theoretical_loss": 3.459378378980901,
+ "tokens_seen": 1996357632
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003193259779338014,
+ "loss": 1.3175,
+ "theoretical_loss": 3.4593689850262423,
+ "tokens_seen": 1996423168
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 0.0003193099297893681,
+ "loss": 1.2849,
+ "theoretical_loss": 3.459359591466292,
+ "tokens_seen": 1996488704
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192938816449348,
+ "loss": 1.2669,
+ "theoretical_loss": 3.459350198301021,
+ "tokens_seen": 1996554240
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192778335005015,
+ "loss": 1.2577,
+ "theoretical_loss": 3.4593408055304,
+ "tokens_seen": 1996619776
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192617853560682,
+ "loss": 1.3572,
+ "theoretical_loss": 3.4593314131543984,
+ "tokens_seen": 1996685312
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192457372116349,
+ "loss": 1.2545,
+ "theoretical_loss": 3.459322021172987,
+ "tokens_seen": 1996750848
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192296890672016,
+ "loss": 1.3256,
+ "theoretical_loss": 3.459312629586137,
+ "tokens_seen": 1996816384
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003192136409227683,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4593032383938187,
+ "tokens_seen": 1996881920
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000319197592778335,
+ "loss": 1.2946,
+ "theoretical_loss": 3.459293847596002,
+ "tokens_seen": 1996947456
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191815446339017,
+ "loss": 1.2394,
+ "theoretical_loss": 3.4592844571926578,
+ "tokens_seen": 1997012992
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191654964894684,
+ "loss": 1.259,
+ "theoretical_loss": 3.459275067183756,
+ "tokens_seen": 1997078528
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191494483450351,
+ "loss": 1.3783,
+ "theoretical_loss": 3.4592656775692685,
+ "tokens_seen": 1997144064
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191334002006018,
+ "loss": 1.3465,
+ "theoretical_loss": 3.4592562883491644,
+ "tokens_seen": 1997209600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191173520561685,
+ "loss": 1.2976,
+ "theoretical_loss": 3.4592468995234147,
+ "tokens_seen": 1997275136
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003191013039117352,
+ "loss": 1.3208,
+ "theoretical_loss": 3.45923751109199,
+ "tokens_seen": 1997340672
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003190852557673019,
+ "loss": 1.1888,
+ "theoretical_loss": 3.4592281230548605,
+ "tokens_seen": 1997406208
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003190692076228686,
+ "loss": 1.2235,
+ "theoretical_loss": 3.459218735411997,
+ "tokens_seen": 1997471744
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003190531594784353,
+ "loss": 1.3613,
+ "theoretical_loss": 3.4592093481633697,
+ "tokens_seen": 1997537280
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000319037111334002,
+ "loss": 1.232,
+ "theoretical_loss": 3.4591999613089492,
+ "tokens_seen": 1997602816
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003190210631895687,
+ "loss": 1.358,
+ "theoretical_loss": 3.459190574848706,
+ "tokens_seen": 1997668352
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003190050150451354,
+ "loss": 1.2946,
+ "theoretical_loss": 3.4591811887826114,
+ "tokens_seen": 1997733888
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003189889669007021,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4591718031106344,
+ "tokens_seen": 1997799424
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003189729187562688,
+ "loss": 1.2532,
+ "theoretical_loss": 3.459162417832747,
+ "tokens_seen": 1997864960
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003189568706118355,
+ "loss": 1.186,
+ "theoretical_loss": 3.459153032948919,
+ "tokens_seen": 1997930496
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003189408224674022,
+ "loss": 1.3609,
+ "theoretical_loss": 3.4591436484591207,
+ "tokens_seen": 1997996032
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003189247743229689,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4591342643633234,
+ "tokens_seen": 1998061568
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031890872617853567,
+ "loss": 1.2729,
+ "theoretical_loss": 3.4591248806614967,
+ "tokens_seen": 1998127104
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031889267803410237,
+ "loss": 1.2624,
+ "theoretical_loss": 3.459115497353612,
+ "tokens_seen": 1998192640
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000318876629889669,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4591061144396393,
+ "tokens_seen": 1998258176
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003188605817452357,
+ "loss": 1.3345,
+ "theoretical_loss": 3.4590967319195496,
+ "tokens_seen": 1998323712
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003188445336008024,
+ "loss": 1.1979,
+ "theoretical_loss": 3.459087349793313,
+ "tokens_seen": 1998389248
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003188284854563691,
+ "loss": 1.243,
+ "theoretical_loss": 3.4590779680608996,
+ "tokens_seen": 1998454784
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003188124373119358,
+ "loss": 1.2696,
+ "theoretical_loss": 3.4590685867222812,
+ "tokens_seen": 1998520320
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187963891675025,
+ "loss": 1.3569,
+ "theoretical_loss": 3.4590592057774274,
+ "tokens_seen": 1998585856
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187803410230692,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4590498252263098,
+ "tokens_seen": 1998651392
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187642928786359,
+ "loss": 1.2312,
+ "theoretical_loss": 3.459040445068897,
+ "tokens_seen": 1998716928
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187482447342026,
+ "loss": 1.1778,
+ "theoretical_loss": 3.4590310653051617,
+ "tokens_seen": 1998782464
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 669104,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2669732570648193,
+ "objective/train/theoretical_loss": 3.459021685935074,
+ "objective/train/tokens_used": 2019308000,
+ "theoretical_loss": 3.459021685935074,
+ "tokens_seen": 1998848000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187321965897693,
+ "loss": 1.2328,
+ "theoretical_loss": 3.459021685935074,
+ "tokens_seen": 1998848000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000318716148445336,
+ "loss": 1.155,
+ "theoretical_loss": 3.459012306958603,
+ "tokens_seen": 1998913536
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003187001003009027,
+ "loss": 1.243,
+ "theoretical_loss": 3.459002928375721,
+ "tokens_seen": 1998979072
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003186840521564694,
+ "loss": 1.3237,
+ "theoretical_loss": 3.4589935501863978,
+ "tokens_seen": 1999044608
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003186680040120361,
+ "loss": 1.2906,
+ "theoretical_loss": 3.458984172390604,
+ "tokens_seen": 1999110144
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003186519558676028,
+ "loss": 1.3192,
+ "theoretical_loss": 3.4589747949883103,
+ "tokens_seen": 1999175680
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003186359077231695,
+ "loss": 1.254,
+ "theoretical_loss": 3.4589654179794875,
+ "tokens_seen": 1999241216
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003186198595787362,
+ "loss": 1.2594,
+ "theoretical_loss": 3.4589560413641056,
+ "tokens_seen": 1999306752
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031860381143430293,
+ "loss": 1.1965,
+ "theoretical_loss": 3.458946665142136,
+ "tokens_seen": 1999372288
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031858776328986963,
+ "loss": 1.265,
+ "theoretical_loss": 3.458937289313549,
+ "tokens_seen": 1999437824
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031857171514543633,
+ "loss": 1.3567,
+ "theoretical_loss": 3.4589279138783144,
+ "tokens_seen": 1999503360
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031855566700100303,
+ "loss": 1.3493,
+ "theoretical_loss": 3.4589185388364037,
+ "tokens_seen": 1999568896
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031853961885656973,
+ "loss": 1.2755,
+ "theoretical_loss": 3.4589091641877876,
+ "tokens_seen": 1999634432
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031852357071213643,
+ "loss": 1.2332,
+ "theoretical_loss": 3.458899789932436,
+ "tokens_seen": 1999699968
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031850752256770313,
+ "loss": 1.1523,
+ "theoretical_loss": 3.4588904160703207,
+ "tokens_seen": 1999765504
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031849147442326983,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4588810426014103,
+ "tokens_seen": 1999831040
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031847542627883653,
+ "loss": 1.3163,
+ "theoretical_loss": 3.458871669525678,
+ "tokens_seen": 1999896576
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031845937813440323,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4588622968430918,
+ "tokens_seen": 1999962112
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031844332998996993,
+ "loss": 1.2808,
+ "theoretical_loss": 3.458852924553624,
+ "tokens_seen": 2000027648
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031842728184553663,
+ "loss": 1.3126,
+ "theoretical_loss": 3.458843552657245,
+ "tokens_seen": 2000093184
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031841123370110333,
+ "loss": 1.19,
+ "theoretical_loss": 3.4588341811539256,
+ "tokens_seen": 2000158720
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031839518555667004,
+ "loss": 1.1766,
+ "theoretical_loss": 3.4588248100436356,
+ "tokens_seen": 2000224256
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031837913741223674,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4588154393263464,
+ "tokens_seen": 2000289792
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031836308926780344,
+ "loss": 1.1677,
+ "theoretical_loss": 3.458806069002028,
+ "tokens_seen": 2000355328
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031834704112337014,
+ "loss": 1.3727,
+ "theoretical_loss": 3.458796699070652,
+ "tokens_seen": 2000420864
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031833099297893684,
+ "loss": 1.249,
+ "theoretical_loss": 3.4587873295321883,
+ "tokens_seen": 2000486400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031831494483450354,
+ "loss": 1.3561,
+ "theoretical_loss": 3.458777960386608,
+ "tokens_seen": 2000551936
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031829889669007024,
+ "loss": 1.2476,
+ "theoretical_loss": 3.4587685916338806,
+ "tokens_seen": 2000617472
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031828284854563694,
+ "loss": 1.2328,
+ "theoretical_loss": 3.4587592232739786,
+ "tokens_seen": 2000683008
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031826680040120364,
+ "loss": 1.2158,
+ "theoretical_loss": 3.458749855306871,
+ "tokens_seen": 2000748544
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031825075225677034,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4587404877325296,
+ "tokens_seen": 2000814080
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031823470411233704,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4587311205509246,
+ "tokens_seen": 2000879616
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031821865596790374,
+ "loss": 1.3109,
+ "theoretical_loss": 3.4587217537620267,
+ "tokens_seen": 2000945152
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031820260782347044,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4587123873658063,
+ "tokens_seen": 2001010688
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031818655967903714,
+ "loss": 1.222,
+ "theoretical_loss": 3.458703021362235,
+ "tokens_seen": 2001076224
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031817051153460384,
+ "loss": 1.1502,
+ "theoretical_loss": 3.458693655751283,
+ "tokens_seen": 2001141760
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031815446339017054,
+ "loss": 1.2481,
+ "theoretical_loss": 3.45868429053292,
+ "tokens_seen": 2001207296
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031813841524573725,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4586749257071174,
+ "tokens_seen": 2001272832
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031812236710130395,
+ "loss": 1.2768,
+ "theoretical_loss": 3.458665561273847,
+ "tokens_seen": 2001338368
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003181063189568706,
+ "loss": 1.3149,
+ "theoretical_loss": 3.458656197233078,
+ "tokens_seen": 2001403904
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003180902708124373,
+ "loss": 1.2879,
+ "theoretical_loss": 3.4586468335847815,
+ "tokens_seen": 2001469440
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000318074222668004,
+ "loss": 1.3222,
+ "theoretical_loss": 3.458637470328928,
+ "tokens_seen": 2001534976
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003180581745235707,
+ "loss": 1.3136,
+ "theoretical_loss": 3.4586281074654894,
+ "tokens_seen": 2001600512
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003180421263791374,
+ "loss": 1.2326,
+ "theoretical_loss": 3.458618744994435,
+ "tokens_seen": 2001666048
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003180260782347041,
+ "loss": 1.2437,
+ "theoretical_loss": 3.458609382915736,
+ "tokens_seen": 2001731584
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003180100300902708,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4586000212293633,
+ "tokens_seen": 2001797120
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003179939819458375,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4585906599352874,
+ "tokens_seen": 2001862656
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003179779338014042,
+ "loss": 1.2408,
+ "theoretical_loss": 3.458581299033479,
+ "tokens_seen": 2001928192
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003179618856569709,
+ "loss": 1.2478,
+ "theoretical_loss": 3.458571938523909,
+ "tokens_seen": 2001993728
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003179458375125376,
+ "loss": 1.1506,
+ "theoretical_loss": 3.458562578406548,
+ "tokens_seen": 2002059264
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 670405,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1057586669921875,
+ "objective/train/theoretical_loss": 3.4585532186813666,
+ "objective/train/tokens_used": 2022584800,
+ "theoretical_loss": 3.4585532186813666,
+ "tokens_seen": 2002124800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003179297893681043,
+ "loss": 1.1716,
+ "theoretical_loss": 3.4585532186813666,
+ "tokens_seen": 2002124800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000317913741223671,
+ "loss": 1.2949,
+ "theoretical_loss": 3.458543859348336,
+ "tokens_seen": 2002190336
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178976930792377,
+ "loss": 1.3765,
+ "theoretical_loss": 3.4585345004074264,
+ "tokens_seen": 2002255872
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178816449348044,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4585251418586087,
+ "tokens_seen": 2002321408
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178655967903711,
+ "loss": 1.3964,
+ "theoretical_loss": 3.458515783701854,
+ "tokens_seen": 2002386944
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178495486459378,
+ "loss": 1.1777,
+ "theoretical_loss": 3.4585064259371325,
+ "tokens_seen": 2002452480
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178335005015045,
+ "loss": 1.2345,
+ "theoretical_loss": 3.458497068564415,
+ "tokens_seen": 2002518016
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178174523570712,
+ "loss": 1.2328,
+ "theoretical_loss": 3.4584877115836727,
+ "tokens_seen": 2002583552
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003178014042126379,
+ "loss": 1.2715,
+ "theoretical_loss": 3.458478354994876,
+ "tokens_seen": 2002649088
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003177853560682046,
+ "loss": 1.2451,
+ "theoretical_loss": 3.4584689987979957,
+ "tokens_seen": 2002714624
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003177693079237713,
+ "loss": 1.311,
+ "theoretical_loss": 3.458459642993003,
+ "tokens_seen": 2002780160
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000317753259779338,
+ "loss": 1.3298,
+ "theoretical_loss": 3.4584502875798675,
+ "tokens_seen": 2002845696
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003177372116349047,
+ "loss": 1.2584,
+ "theoretical_loss": 3.4584409325585614,
+ "tokens_seen": 2002911232
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003177211634904714,
+ "loss": 1.27,
+ "theoretical_loss": 3.458431577929055,
+ "tokens_seen": 2002976768
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031770511534603816,
+ "loss": 1.1746,
+ "theoretical_loss": 3.4584222236913185,
+ "tokens_seen": 2003042304
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031768906720160486,
+ "loss": 1.3136,
+ "theoretical_loss": 3.458412869845323,
+ "tokens_seen": 2003107840
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031767301905717156,
+ "loss": 1.2681,
+ "theoretical_loss": 3.458403516391039,
+ "tokens_seen": 2003173376
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031765697091273826,
+ "loss": 1.2695,
+ "theoretical_loss": 3.458394163328438,
+ "tokens_seen": 2003238912
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031764092276830497,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4583848106574906,
+ "tokens_seen": 2003304448
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031762487462387167,
+ "loss": 1.2239,
+ "theoretical_loss": 3.458375458378167,
+ "tokens_seen": 2003369984
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031760882647943837,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4583661064904385,
+ "tokens_seen": 2003435520
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031759277833500507,
+ "loss": 1.2416,
+ "theoretical_loss": 3.458356754994276,
+ "tokens_seen": 2003501056
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031757673019057177,
+ "loss": 1.3232,
+ "theoretical_loss": 3.4583474038896496,
+ "tokens_seen": 2003566592
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031756068204613847,
+ "loss": 1.3648,
+ "theoretical_loss": 3.458338053176531,
+ "tokens_seen": 2003632128
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031754463390170517,
+ "loss": 1.3,
+ "theoretical_loss": 3.4583287028548906,
+ "tokens_seen": 2003697664
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031752858575727187,
+ "loss": 1.2187,
+ "theoretical_loss": 3.458319352924699,
+ "tokens_seen": 2003763200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031751253761283857,
+ "loss": 1.2122,
+ "theoretical_loss": 3.458310003385927,
+ "tokens_seen": 2003828736
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031749648946840527,
+ "loss": 1.3151,
+ "theoretical_loss": 3.4583006542385464,
+ "tokens_seen": 2003894272
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003174804413239719,
+ "loss": 1.2365,
+ "theoretical_loss": 3.4582913054825264,
+ "tokens_seen": 2003959808
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003174643931795386,
+ "loss": 1.2237,
+ "theoretical_loss": 3.4582819571178387,
+ "tokens_seen": 2004025344
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003174483450351053,
+ "loss": 1.178,
+ "theoretical_loss": 3.4582726091444544,
+ "tokens_seen": 2004090880
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000317432296890672,
+ "loss": 1.1719,
+ "theoretical_loss": 3.458263261562344,
+ "tokens_seen": 2004156416
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003174162487462387,
+ "loss": 1.3065,
+ "theoretical_loss": 3.458253914371478,
+ "tokens_seen": 2004221952
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003174002006018054,
+ "loss": 1.3441,
+ "theoretical_loss": 3.458244567571828,
+ "tokens_seen": 2004287488
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173841524573721,
+ "loss": 1.268,
+ "theoretical_loss": 3.4582352211633642,
+ "tokens_seen": 2004353024
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173681043129388,
+ "loss": 1.3368,
+ "theoretical_loss": 3.4582258751460575,
+ "tokens_seen": 2004418560
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173520561685055,
+ "loss": 1.1921,
+ "theoretical_loss": 3.458216529519879,
+ "tokens_seen": 2004484096
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173360080240722,
+ "loss": 1.2697,
+ "theoretical_loss": 3.458207184284799,
+ "tokens_seen": 2004549632
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173199598796389,
+ "loss": 1.185,
+ "theoretical_loss": 3.458197839440789,
+ "tokens_seen": 2004615168
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003173039117352056,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4581884949878194,
+ "tokens_seen": 2004680704
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003172878635907723,
+ "loss": 1.1381,
+ "theoretical_loss": 3.4581791509258615,
+ "tokens_seen": 2004746240
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000317271815446339,
+ "loss": 1.4164,
+ "theoretical_loss": 3.458169807254886,
+ "tokens_seen": 2004811776
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003172557673019057,
+ "loss": 1.1845,
+ "theoretical_loss": 3.4581604639748638,
+ "tokens_seen": 2004877312
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003172397191574724,
+ "loss": 1.2115,
+ "theoretical_loss": 3.4581511210857654,
+ "tokens_seen": 2004942848
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031722367101303913,
+ "loss": 1.3386,
+ "theoretical_loss": 3.458141778587562,
+ "tokens_seen": 2005008384
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031720762286860583,
+ "loss": 1.2416,
+ "theoretical_loss": 3.4581324364802244,
+ "tokens_seen": 2005073920
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031719157472417253,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4581230947637236,
+ "tokens_seen": 2005139456
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031717552657973923,
+ "loss": 1.2578,
+ "theoretical_loss": 3.45811375343803,
+ "tokens_seen": 2005204992
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031715947843530593,
+ "loss": 1.2442,
+ "theoretical_loss": 3.458104412503115,
+ "tokens_seen": 2005270528
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031714343029087263,
+ "loss": 1.3346,
+ "theoretical_loss": 3.458095071958949,
+ "tokens_seen": 2005336064
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 671345,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2735745906829834,
+ "objective/train/theoretical_loss": 3.4580857318055034,
+ "objective/train/tokens_used": 2025861600,
+ "theoretical_loss": 3.4580857318055034,
+ "tokens_seen": 2005401600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031712738214643933,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4580857318055034,
+ "tokens_seen": 2005401600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031711133400200603,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4580763920427486,
+ "tokens_seen": 2005467136
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031709528585757273,
+ "loss": 1.2701,
+ "theoretical_loss": 3.458067052670656,
+ "tokens_seen": 2005532672
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031707923771313943,
+ "loss": 1.1594,
+ "theoretical_loss": 3.458057713689196,
+ "tokens_seen": 2005598208
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031706318956870613,
+ "loss": 1.259,
+ "theoretical_loss": 3.45804837509834,
+ "tokens_seen": 2005663744
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031704714142427283,
+ "loss": 1.1806,
+ "theoretical_loss": 3.458039036898059,
+ "tokens_seen": 2005729280
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031703109327983954,
+ "loss": 1.2136,
+ "theoretical_loss": 3.458029699088323,
+ "tokens_seen": 2005794816
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031701504513540624,
+ "loss": 1.2813,
+ "theoretical_loss": 3.4580203616691034,
+ "tokens_seen": 2005860352
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031699899699097294,
+ "loss": 1.2302,
+ "theoretical_loss": 3.4580110246403715,
+ "tokens_seen": 2005925888
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031698294884653964,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4580016880020974,
+ "tokens_seen": 2005991424
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031696690070210634,
+ "loss": 1.2526,
+ "theoretical_loss": 3.4579923517542532,
+ "tokens_seen": 2006056960
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031695085255767304,
+ "loss": 1.1677,
+ "theoretical_loss": 3.457983015896809,
+ "tokens_seen": 2006122496
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031693480441323974,
+ "loss": 1.2033,
+ "theoretical_loss": 3.457973680429735,
+ "tokens_seen": 2006188032
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031691875626880644,
+ "loss": 1.1112,
+ "theoretical_loss": 3.4579643453530036,
+ "tokens_seen": 2006253568
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031690270812437314,
+ "loss": 1.1891,
+ "theoretical_loss": 3.457955010666585,
+ "tokens_seen": 2006319104
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031688665997993984,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4579456763704504,
+ "tokens_seen": 2006384640
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031687061183550654,
+ "loss": 1.3585,
+ "theoretical_loss": 3.4579363424645706,
+ "tokens_seen": 2006450176
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031685456369107324,
+ "loss": 1.2335,
+ "theoretical_loss": 3.457927008948916,
+ "tokens_seen": 2006515712
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031683851554663994,
+ "loss": 1.2356,
+ "theoretical_loss": 3.457917675823458,
+ "tokens_seen": 2006581248
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031682246740220664,
+ "loss": 1.135,
+ "theoretical_loss": 3.4579083430881683,
+ "tokens_seen": 2006646784
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031680641925777334,
+ "loss": 1.2078,
+ "theoretical_loss": 3.4578990107430165,
+ "tokens_seen": 2006712320
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031679037111334004,
+ "loss": 1.3739,
+ "theoretical_loss": 3.457889678787975,
+ "tokens_seen": 2006777856
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031677432296890675,
+ "loss": 1.1849,
+ "theoretical_loss": 3.457880347223013,
+ "tokens_seen": 2006843392
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031675827482447345,
+ "loss": 1.267,
+ "theoretical_loss": 3.4578710160481023,
+ "tokens_seen": 2006908928
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031674222668004015,
+ "loss": 1.1858,
+ "theoretical_loss": 3.4578616852632145,
+ "tokens_seen": 2006974464
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031672617853560685,
+ "loss": 1.2572,
+ "theoretical_loss": 3.45785235486832,
+ "tokens_seen": 2007040000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003167101303911735,
+ "loss": 1.1521,
+ "theoretical_loss": 3.4578430248633896,
+ "tokens_seen": 2007105536
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003166940822467402,
+ "loss": 1.1478,
+ "theoretical_loss": 3.457833695248395,
+ "tokens_seen": 2007171072
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003166780341023069,
+ "loss": 1.2805,
+ "theoretical_loss": 3.4578243660233055,
+ "tokens_seen": 2007236608
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003166619859578736,
+ "loss": 1.3188,
+ "theoretical_loss": 3.4578150371880936,
+ "tokens_seen": 2007302144
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003166459378134403,
+ "loss": 1.361,
+ "theoretical_loss": 3.4578057087427303,
+ "tokens_seen": 2007367680
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000316629889669007,
+ "loss": 1.2297,
+ "theoretical_loss": 3.457796380687186,
+ "tokens_seen": 2007433216
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003166138415245737,
+ "loss": 1.2545,
+ "theoretical_loss": 3.4577870530214314,
+ "tokens_seen": 2007498752
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165977933801404,
+ "loss": 1.3016,
+ "theoretical_loss": 3.4577777257454385,
+ "tokens_seen": 2007564288
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165817452357071,
+ "loss": 1.3507,
+ "theoretical_loss": 3.4577683988591774,
+ "tokens_seen": 2007629824
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165656970912738,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4577590723626193,
+ "tokens_seen": 2007695360
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165496489468405,
+ "loss": 1.2689,
+ "theoretical_loss": 3.4577497462557356,
+ "tokens_seen": 2007760896
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165336008024072,
+ "loss": 1.3171,
+ "theoretical_loss": 3.4577404205384967,
+ "tokens_seen": 2007826432
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003165175526579739,
+ "loss": 1.1521,
+ "theoretical_loss": 3.4577310952108746,
+ "tokens_seen": 2007891968
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031650150451354066,
+ "loss": 1.178,
+ "theoretical_loss": 3.4577217702728387,
+ "tokens_seen": 2007957504
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031648545636910736,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4577124457243613,
+ "tokens_seen": 2008023040
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031646940822467406,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4577031215654133,
+ "tokens_seen": 2008088576
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031645336008024076,
+ "loss": 1.1754,
+ "theoretical_loss": 3.457693797795965,
+ "tokens_seen": 2008154112
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031643731193580746,
+ "loss": 1.298,
+ "theoretical_loss": 3.4576844744159883,
+ "tokens_seen": 2008219648
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031642126379137416,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4576751514254536,
+ "tokens_seen": 2008285184
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031640521564694086,
+ "loss": 1.272,
+ "theoretical_loss": 3.4576658288243323,
+ "tokens_seen": 2008350720
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031638916750250756,
+ "loss": 1.3267,
+ "theoretical_loss": 3.4576565066125946,
+ "tokens_seen": 2008416256
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031637311935807426,
+ "loss": 1.2737,
+ "theoretical_loss": 3.457647184790213,
+ "tokens_seen": 2008481792
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031635707121364096,
+ "loss": 1.2197,
+ "theoretical_loss": 3.4576378633571574,
+ "tokens_seen": 2008547328
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031634102306920766,
+ "loss": 1.2409,
+ "theoretical_loss": 3.4576285423133992,
+ "tokens_seen": 2008612864
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 672663,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.369638442993164,
+ "objective/train/theoretical_loss": 3.4576192216589092,
+ "objective/train/tokens_used": 2029138400,
+ "theoretical_loss": 3.4576192216589092,
+ "tokens_seen": 2008678400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031632497492477436,
+ "loss": 1.3517,
+ "theoretical_loss": 3.4576192216589092,
+ "tokens_seen": 2008678400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031630892678034106,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4576099013936585,
+ "tokens_seen": 2008743936
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031629287863590776,
+ "loss": 1.1586,
+ "theoretical_loss": 3.4576005815176183,
+ "tokens_seen": 2008809472
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031627683049147446,
+ "loss": 1.3099,
+ "theoretical_loss": 3.45759126203076,
+ "tokens_seen": 2008875008
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031626078234704117,
+ "loss": 1.3467,
+ "theoretical_loss": 3.457581942933054,
+ "tokens_seen": 2008940544
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031624473420260787,
+ "loss": 1.237,
+ "theoretical_loss": 3.457572624224472,
+ "tokens_seen": 2009006080
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031622868605817457,
+ "loss": 1.2176,
+ "theoretical_loss": 3.4575633059049844,
+ "tokens_seen": 2009071616
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031621263791374127,
+ "loss": 1.2523,
+ "theoretical_loss": 3.4575539879745625,
+ "tokens_seen": 2009137152
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031619658976930797,
+ "loss": 1.2515,
+ "theoretical_loss": 3.4575446704331774,
+ "tokens_seen": 2009202688
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031618054162487467,
+ "loss": 1.265,
+ "theoretical_loss": 3.4575353532808,
+ "tokens_seen": 2009268224
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031616449348044137,
+ "loss": 1.2193,
+ "theoretical_loss": 3.457526036517402,
+ "tokens_seen": 2009333760
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031614844533600807,
+ "loss": 1.2702,
+ "theoretical_loss": 3.457516720142954,
+ "tokens_seen": 2009399296
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031613239719157477,
+ "loss": 1.3199,
+ "theoretical_loss": 3.457507404157427,
+ "tokens_seen": 2009464832
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031611634904714147,
+ "loss": 1.3002,
+ "theoretical_loss": 3.4574980885607927,
+ "tokens_seen": 2009530368
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031610030090270817,
+ "loss": 1.3355,
+ "theoretical_loss": 3.4574887733530213,
+ "tokens_seen": 2009595904
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031608425275827487,
+ "loss": 1.3138,
+ "theoretical_loss": 3.457479458534084,
+ "tokens_seen": 2009661440
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003160682046138415,
+ "loss": 1.3121,
+ "theoretical_loss": 3.4574701441039526,
+ "tokens_seen": 2009726976
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003160521564694082,
+ "loss": 1.2193,
+ "theoretical_loss": 3.457460830062597,
+ "tokens_seen": 2009792512
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003160361083249749,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4574515164099897,
+ "tokens_seen": 2009858048
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003160200601805416,
+ "loss": 1.2837,
+ "theoretical_loss": 3.457442203146101,
+ "tokens_seen": 2009923584
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003160040120361083,
+ "loss": 1.33,
+ "theoretical_loss": 3.457432890270902,
+ "tokens_seen": 2009989120
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000315987963891675,
+ "loss": 1.2559,
+ "theoretical_loss": 3.4574235777843643,
+ "tokens_seen": 2010054656
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003159719157472417,
+ "loss": 1.2591,
+ "theoretical_loss": 3.4574142656864586,
+ "tokens_seen": 2010120192
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003159558676028084,
+ "loss": 1.3054,
+ "theoretical_loss": 3.457404953977156,
+ "tokens_seen": 2010185728
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003159398194583751,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4573956426564276,
+ "tokens_seen": 2010251264
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003159237713139418,
+ "loss": 1.258,
+ "theoretical_loss": 3.4573863317242446,
+ "tokens_seen": 2010316800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003159077231695085,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4573770211805783,
+ "tokens_seen": 2010382336
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003158916750250752,
+ "loss": 1.2494,
+ "theoretical_loss": 3.457367711025399,
+ "tokens_seen": 2010447872
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003158756268806419,
+ "loss": 1.2882,
+ "theoretical_loss": 3.4573584012586793,
+ "tokens_seen": 2010513408
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031585957873620863,
+ "loss": 1.2828,
+ "theoretical_loss": 3.4573490918803893,
+ "tokens_seen": 2010578944
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031584353059177533,
+ "loss": 1.2842,
+ "theoretical_loss": 3.4573397828905,
+ "tokens_seen": 2010644480
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031582748244734203,
+ "loss": 1.23,
+ "theoretical_loss": 3.4573304742889834,
+ "tokens_seen": 2010710016
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031581143430290873,
+ "loss": 1.3398,
+ "theoretical_loss": 3.45732116607581,
+ "tokens_seen": 2010775552
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031579538615847543,
+ "loss": 1.2349,
+ "theoretical_loss": 3.457311858250951,
+ "tokens_seen": 2010841088
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031577933801404213,
+ "loss": 1.1745,
+ "theoretical_loss": 3.4573025508143775,
+ "tokens_seen": 2010906624
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031576328986960883,
+ "loss": 1.3773,
+ "theoretical_loss": 3.4572932437660606,
+ "tokens_seen": 2010972160
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031574724172517553,
+ "loss": 1.2861,
+ "theoretical_loss": 3.457283937105972,
+ "tokens_seen": 2011037696
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031573119358074223,
+ "loss": 1.2441,
+ "theoretical_loss": 3.457274630834082,
+ "tokens_seen": 2011103232
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031571514543630893,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4572653249503627,
+ "tokens_seen": 2011168768
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031569909729187563,
+ "loss": 1.2249,
+ "theoretical_loss": 3.4572560194547846,
+ "tokens_seen": 2011234304
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031568304914744233,
+ "loss": 1.4085,
+ "theoretical_loss": 3.4572467143473187,
+ "tokens_seen": 2011299840
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031566700100300903,
+ "loss": 1.2425,
+ "theoretical_loss": 3.457237409627937,
+ "tokens_seen": 2011365376
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031565095285857574,
+ "loss": 1.2921,
+ "theoretical_loss": 3.4572281052966103,
+ "tokens_seen": 2011430912
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031563490471414244,
+ "loss": 1.2687,
+ "theoretical_loss": 3.457218801353309,
+ "tokens_seen": 2011496448
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031561885656970914,
+ "loss": 1.3747,
+ "theoretical_loss": 3.4572094977980052,
+ "tokens_seen": 2011561984
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031560280842527584,
+ "loss": 1.2738,
+ "theoretical_loss": 3.45720019463067,
+ "tokens_seen": 2011627520
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031558676028084254,
+ "loss": 1.332,
+ "theoretical_loss": 3.4571908918512744,
+ "tokens_seen": 2011693056
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031557071213640924,
+ "loss": 1.328,
+ "theoretical_loss": 3.4571815894597897,
+ "tokens_seen": 2011758592
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031555466399197594,
+ "loss": 1.1391,
+ "theoretical_loss": 3.4571722874561868,
+ "tokens_seen": 2011824128
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031553861584754264,
+ "loss": 1.1817,
+ "theoretical_loss": 3.457162985840437,
+ "tokens_seen": 2011889664
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 673428,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3442015647888184,
+ "objective/train/theoretical_loss": 3.4571536846125115,
+ "objective/train/tokens_used": 2032415200,
+ "theoretical_loss": 3.4571536846125115,
+ "tokens_seen": 2011955200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031552256770310934,
+ "loss": 1.3111,
+ "theoretical_loss": 3.4571536846125115,
+ "tokens_seen": 2011955200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031550651955867604,
+ "loss": 1.361,
+ "theoretical_loss": 3.4571443837723814,
+ "tokens_seen": 2012020736
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031549047141424274,
+ "loss": 1.3822,
+ "theoretical_loss": 3.4571350833200185,
+ "tokens_seen": 2012086272
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031547442326980944,
+ "loss": 1.2538,
+ "theoretical_loss": 3.4571257832553934,
+ "tokens_seen": 2012151808
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031545837512537614,
+ "loss": 1.2017,
+ "theoretical_loss": 3.4571164835784773,
+ "tokens_seen": 2012217344
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031544232698094284,
+ "loss": 1.2188,
+ "theoretical_loss": 3.457107184289242,
+ "tokens_seen": 2012282880
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031542627883650954,
+ "loss": 1.2986,
+ "theoretical_loss": 3.457097885387658,
+ "tokens_seen": 2012348416
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031541023069207625,
+ "loss": 1.2914,
+ "theoretical_loss": 3.457088586873697,
+ "tokens_seen": 2012413952
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031539418254764295,
+ "loss": 1.2898,
+ "theoretical_loss": 3.4570792887473294,
+ "tokens_seen": 2012479488
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031537813440320965,
+ "loss": 1.2779,
+ "theoretical_loss": 3.4570699910085274,
+ "tokens_seen": 2012545024
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031536208625877635,
+ "loss": 1.2538,
+ "theoretical_loss": 3.457060693657262,
+ "tokens_seen": 2012610560
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031534603811434305,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4570513966935046,
+ "tokens_seen": 2012676096
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031532998996990975,
+ "loss": 1.2307,
+ "theoretical_loss": 3.457042100117226,
+ "tokens_seen": 2012741632
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031531394182547645,
+ "loss": 1.1729,
+ "theoretical_loss": 3.457032803928397,
+ "tokens_seen": 2012807168
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031529789368104315,
+ "loss": 1.2087,
+ "theoretical_loss": 3.4570235081269898,
+ "tokens_seen": 2012872704
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031528184553660985,
+ "loss": 1.1964,
+ "theoretical_loss": 3.4570142127129753,
+ "tokens_seen": 2012938240
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031526579739217655,
+ "loss": 1.2167,
+ "theoretical_loss": 3.4570049176863247,
+ "tokens_seen": 2013003776
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031524974924774325,
+ "loss": 1.2488,
+ "theoretical_loss": 3.4569956230470087,
+ "tokens_seen": 2013069312
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031523370110330995,
+ "loss": 1.2628,
+ "theoretical_loss": 3.456986328795,
+ "tokens_seen": 2013134848
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031521765295887665,
+ "loss": 1.3149,
+ "theoretical_loss": 3.4569770349302686,
+ "tokens_seen": 2013200384
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031520160481444335,
+ "loss": 1.3399,
+ "theoretical_loss": 3.456967741452786,
+ "tokens_seen": 2013265920
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031518555667001005,
+ "loss": 1.2906,
+ "theoretical_loss": 3.4569584483625233,
+ "tokens_seen": 2013331456
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031516950852557675,
+ "loss": 1.2277,
+ "theoretical_loss": 3.4569491556594523,
+ "tokens_seen": 2013396992
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031515346038114346,
+ "loss": 1.2712,
+ "theoretical_loss": 3.4569398633435444,
+ "tokens_seen": 2013462528
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031513741223671016,
+ "loss": 1.2211,
+ "theoretical_loss": 3.4569305714147696,
+ "tokens_seen": 2013528064
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031512136409227686,
+ "loss": 1.1884,
+ "theoretical_loss": 3.4569212798731006,
+ "tokens_seen": 2013593600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031510531594784356,
+ "loss": 1.2723,
+ "theoretical_loss": 3.456911988718508,
+ "tokens_seen": 2013659136
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031508926780341026,
+ "loss": 1.2175,
+ "theoretical_loss": 3.456902697950963,
+ "tokens_seen": 2013724672
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031507321965897696,
+ "loss": 1.3602,
+ "theoretical_loss": 3.456893407570438,
+ "tokens_seen": 2013790208
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031505717151454366,
+ "loss": 1.1215,
+ "theoretical_loss": 3.456884117576902,
+ "tokens_seen": 2013855744
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031504112337011036,
+ "loss": 1.2069,
+ "theoretical_loss": 3.4568748279703283,
+ "tokens_seen": 2013921280
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031502507522567706,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4568655387506873,
+ "tokens_seen": 2013986816
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031500902708124376,
+ "loss": 1.2578,
+ "theoretical_loss": 3.4568562499179505,
+ "tokens_seen": 2014052352
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031499297893681046,
+ "loss": 1.3153,
+ "theoretical_loss": 3.4568469614720896,
+ "tokens_seen": 2014117888
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031497693079237716,
+ "loss": 1.2058,
+ "theoretical_loss": 3.4568376734130752,
+ "tokens_seen": 2014183424
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031496088264794386,
+ "loss": 1.2386,
+ "theoretical_loss": 3.456828385740879,
+ "tokens_seen": 2014248960
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031494483450351056,
+ "loss": 1.1387,
+ "theoretical_loss": 3.4568190984554725,
+ "tokens_seen": 2014314496
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031492878635907726,
+ "loss": 1.2466,
+ "theoretical_loss": 3.4568098115568264,
+ "tokens_seen": 2014380032
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031491273821464396,
+ "loss": 1.3329,
+ "theoretical_loss": 3.4568005250449123,
+ "tokens_seen": 2014445568
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031489669007021067,
+ "loss": 1.2391,
+ "theoretical_loss": 3.4567912389197017,
+ "tokens_seen": 2014511104
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031488064192577737,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4567819531811654,
+ "tokens_seen": 2014576640
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031486459378134407,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4567726678292754,
+ "tokens_seen": 2014642176
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031484854563691077,
+ "loss": 1.2283,
+ "theoretical_loss": 3.456763382864003,
+ "tokens_seen": 2014707712
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031483249749247747,
+ "loss": 1.3018,
+ "theoretical_loss": 3.456754098285319,
+ "tokens_seen": 2014773248
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031481644934804417,
+ "loss": 1.2549,
+ "theoretical_loss": 3.456744814093195,
+ "tokens_seen": 2014838784
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031480040120361087,
+ "loss": 1.34,
+ "theoretical_loss": 3.4567355302876024,
+ "tokens_seen": 2014904320
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031478435305917757,
+ "loss": 1.2595,
+ "theoretical_loss": 3.456726246868512,
+ "tokens_seen": 2014969856
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031476830491474427,
+ "loss": 1.2001,
+ "theoretical_loss": 3.456716963835896,
+ "tokens_seen": 2015035392
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031475225677031097,
+ "loss": 1.2267,
+ "theoretical_loss": 3.4567076811897253,
+ "tokens_seen": 2015100928
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031473620862587767,
+ "loss": 1.2498,
+ "theoretical_loss": 3.456698398929971,
+ "tokens_seen": 2015166464
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 674624,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2796549797058105,
+ "objective/train/theoretical_loss": 3.456689117056605,
+ "objective/train/tokens_used": 2035692000,
+ "theoretical_loss": 3.456689117056605,
+ "tokens_seen": 2015232000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031472016048144437,
+ "loss": 1.1848,
+ "theoretical_loss": 3.456689117056605,
+ "tokens_seen": 2015232000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003147041123370111,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4566798355695987,
+ "tokens_seen": 2015297536
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146880641925778,
+ "loss": 1.185,
+ "theoretical_loss": 3.4566705544689222,
+ "tokens_seen": 2015363072
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146720160481444,
+ "loss": 1.3028,
+ "theoretical_loss": 3.4566612737545483,
+ "tokens_seen": 2015428608
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146559679037111,
+ "loss": 1.2858,
+ "theoretical_loss": 3.4566519934264477,
+ "tokens_seen": 2015494144
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146399197592778,
+ "loss": 1.2488,
+ "theoretical_loss": 3.4566427134845923,
+ "tokens_seen": 2015559680
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146238716148445,
+ "loss": 1.2474,
+ "theoretical_loss": 3.456633433928953,
+ "tokens_seen": 2015625216
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003146078234704112,
+ "loss": 1.3715,
+ "theoretical_loss": 3.456624154759501,
+ "tokens_seen": 2015690752
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003145917753259779,
+ "loss": 1.2395,
+ "theoretical_loss": 3.456614875976208,
+ "tokens_seen": 2015756288
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003145757271815446,
+ "loss": 1.2891,
+ "theoretical_loss": 3.456605597579045,
+ "tokens_seen": 2015821824
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003145596790371113,
+ "loss": 1.2606,
+ "theoretical_loss": 3.456596319567984,
+ "tokens_seen": 2015887360
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000314543630892678,
+ "loss": 1.3007,
+ "theoretical_loss": 3.456587041942996,
+ "tokens_seen": 2015952896
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003145275827482447,
+ "loss": 1.2029,
+ "theoretical_loss": 3.4565777647040523,
+ "tokens_seen": 2016018432
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003145115346038114,
+ "loss": 1.1563,
+ "theoretical_loss": 3.4565684878511247,
+ "tokens_seen": 2016083968
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003144954864593781,
+ "loss": 1.2271,
+ "theoretical_loss": 3.4565592113841843,
+ "tokens_seen": 2016149504
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031447943831494483,
+ "loss": 1.2535,
+ "theoretical_loss": 3.456549935303202,
+ "tokens_seen": 2016215040
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031446339017051153,
+ "loss": 1.1762,
+ "theoretical_loss": 3.45654065960815,
+ "tokens_seen": 2016280576
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031444734202607823,
+ "loss": 1.2038,
+ "theoretical_loss": 3.4565313842989998,
+ "tokens_seen": 2016346112
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031443129388164493,
+ "loss": 1.1687,
+ "theoretical_loss": 3.4565221093757224,
+ "tokens_seen": 2016411648
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031441524573721163,
+ "loss": 1.3511,
+ "theoretical_loss": 3.4565128348382883,
+ "tokens_seen": 2016477184
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031439919759277833,
+ "loss": 1.296,
+ "theoretical_loss": 3.4565035606866705,
+ "tokens_seen": 2016542720
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031438314944834503,
+ "loss": 1.244,
+ "theoretical_loss": 3.4564942869208397,
+ "tokens_seen": 2016608256
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031436710130391173,
+ "loss": 1.2937,
+ "theoretical_loss": 3.4564850135407674,
+ "tokens_seen": 2016673792
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031435105315947843,
+ "loss": 1.25,
+ "theoretical_loss": 3.456475740546425,
+ "tokens_seen": 2016739328
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031433500501504513,
+ "loss": 1.3736,
+ "theoretical_loss": 3.4564664679377834,
+ "tokens_seen": 2016804864
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031431895687061183,
+ "loss": 1.391,
+ "theoretical_loss": 3.4564571957148154,
+ "tokens_seen": 2016870400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031430290872617853,
+ "loss": 1.2982,
+ "theoretical_loss": 3.456447923877491,
+ "tokens_seen": 2016935936
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031428686058174524,
+ "loss": 1.2943,
+ "theoretical_loss": 3.456438652425782,
+ "tokens_seen": 2017001472
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031427081243731194,
+ "loss": 1.315,
+ "theoretical_loss": 3.4564293813596603,
+ "tokens_seen": 2017067008
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031425476429287864,
+ "loss": 1.3477,
+ "theoretical_loss": 3.456420110679097,
+ "tokens_seen": 2017132544
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031423871614844534,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4564108403840637,
+ "tokens_seen": 2017198080
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031422266800401204,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4564015704745312,
+ "tokens_seen": 2017263616
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031420661985957874,
+ "loss": 1.2969,
+ "theoretical_loss": 3.4563923009504722,
+ "tokens_seen": 2017329152
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031419057171514544,
+ "loss": 1.2524,
+ "theoretical_loss": 3.456383031811857,
+ "tokens_seen": 2017394688
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031417452357071214,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4563737630586577,
+ "tokens_seen": 2017460224
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031415847542627884,
+ "loss": 1.2107,
+ "theoretical_loss": 3.4563644946908454,
+ "tokens_seen": 2017525760
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031414242728184554,
+ "loss": 1.1601,
+ "theoretical_loss": 3.456355226708392,
+ "tokens_seen": 2017591296
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031412637913741224,
+ "loss": 1.3043,
+ "theoretical_loss": 3.456345959111268,
+ "tokens_seen": 2017656832
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000314110330992979,
+ "loss": 1.3037,
+ "theoretical_loss": 3.4563366918994456,
+ "tokens_seen": 2017722368
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003140942828485457,
+ "loss": 1.3059,
+ "theoretical_loss": 3.456327425072897,
+ "tokens_seen": 2017787904
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003140782347041124,
+ "loss": 1.3482,
+ "theoretical_loss": 3.456318158631592,
+ "tokens_seen": 2017853440
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003140621865596791,
+ "loss": 1.3435,
+ "theoretical_loss": 3.4563088925755032,
+ "tokens_seen": 2017918976
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003140461384152458,
+ "loss": 1.3079,
+ "theoretical_loss": 3.456299626904602,
+ "tokens_seen": 2017984512
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031403009027081245,
+ "loss": 1.289,
+ "theoretical_loss": 3.4562903616188594,
+ "tokens_seen": 2018050048
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031401404212637915,
+ "loss": 1.2096,
+ "theoretical_loss": 3.4562810967182473,
+ "tokens_seen": 2018115584
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031399799398194585,
+ "loss": 1.2272,
+ "theoretical_loss": 3.456271832202737,
+ "tokens_seen": 2018181120
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031398194583751255,
+ "loss": 1.1956,
+ "theoretical_loss": 3.4562625680723,
+ "tokens_seen": 2018246656
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031396589769307925,
+ "loss": 1.2056,
+ "theoretical_loss": 3.456253304326908,
+ "tokens_seen": 2018312192
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031394984954864595,
+ "loss": 1.3314,
+ "theoretical_loss": 3.4562440409665323,
+ "tokens_seen": 2018377728
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031393380140421265,
+ "loss": 1.2872,
+ "theoretical_loss": 3.456234777991144,
+ "tokens_seen": 2018443264
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 675599,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.253434419631958,
+ "objective/train/theoretical_loss": 3.4562255154007158,
+ "objective/train/tokens_used": 2038968800,
+ "theoretical_loss": 3.4562255154007158,
+ "tokens_seen": 2018508800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031391775325977935,
+ "loss": 1.3645,
+ "theoretical_loss": 3.4562255154007158,
+ "tokens_seen": 2018508800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031390170511534605,
+ "loss": 1.2885,
+ "theoretical_loss": 3.4562162531952176,
+ "tokens_seen": 2018574336
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031388565697091275,
+ "loss": 1.199,
+ "theoretical_loss": 3.4562069913746223,
+ "tokens_seen": 2018639872
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031386960882647945,
+ "loss": 1.3032,
+ "theoretical_loss": 3.456197729938901,
+ "tokens_seen": 2018705408
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031385356068204615,
+ "loss": 1.4003,
+ "theoretical_loss": 3.4561884688880244,
+ "tokens_seen": 2018770944
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031383751253761285,
+ "loss": 1.2505,
+ "theoretical_loss": 3.4561792082219647,
+ "tokens_seen": 2018836480
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031382146439317955,
+ "loss": 1.1888,
+ "theoretical_loss": 3.456169947940694,
+ "tokens_seen": 2018902016
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031380541624874625,
+ "loss": 1.2354,
+ "theoretical_loss": 3.4561606880441826,
+ "tokens_seen": 2018967552
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031378936810431295,
+ "loss": 1.2211,
+ "theoretical_loss": 3.4561514285324026,
+ "tokens_seen": 2019033088
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031377331995987966,
+ "loss": 1.3486,
+ "theoretical_loss": 3.456142169405326,
+ "tokens_seen": 2019098624
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031375727181544636,
+ "loss": 1.2352,
+ "theoretical_loss": 3.456132910662924,
+ "tokens_seen": 2019164160
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031374122367101306,
+ "loss": 1.145,
+ "theoretical_loss": 3.456123652305168,
+ "tokens_seen": 2019229696
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031372517552657976,
+ "loss": 1.3951,
+ "theoretical_loss": 3.4561143943320287,
+ "tokens_seen": 2019295232
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031370912738214646,
+ "loss": 1.3111,
+ "theoretical_loss": 3.4561051367434796,
+ "tokens_seen": 2019360768
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031369307923771316,
+ "loss": 1.1926,
+ "theoretical_loss": 3.4560958795394905,
+ "tokens_seen": 2019426304
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031367703109327986,
+ "loss": 1.3605,
+ "theoretical_loss": 3.4560866227200338,
+ "tokens_seen": 2019491840
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031366098294884656,
+ "loss": 1.2575,
+ "theoretical_loss": 3.456077366285081,
+ "tokens_seen": 2019557376
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031364493480441326,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4560681102346034,
+ "tokens_seen": 2019622912
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031362888665997996,
+ "loss": 1.2136,
+ "theoretical_loss": 3.4560588545685724,
+ "tokens_seen": 2019688448
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031361283851554666,
+ "loss": 1.1928,
+ "theoretical_loss": 3.4560495992869598,
+ "tokens_seen": 2019753984
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031359679037111336,
+ "loss": 1.2462,
+ "theoretical_loss": 3.4560403443897374,
+ "tokens_seen": 2019819520
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031358074222668006,
+ "loss": 1.3372,
+ "theoretical_loss": 3.456031089876877,
+ "tokens_seen": 2019885056
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031356469408224676,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4560218357483485,
+ "tokens_seen": 2019950592
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031354864593781346,
+ "loss": 1.3099,
+ "theoretical_loss": 3.4560125820041256,
+ "tokens_seen": 2020016128
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031353259779338017,
+ "loss": 1.3239,
+ "theoretical_loss": 3.4560033286441785,
+ "tokens_seen": 2020081664
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031351654964894687,
+ "loss": 1.2715,
+ "theoretical_loss": 3.45599407566848,
+ "tokens_seen": 2020147200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031350050150451357,
+ "loss": 1.1761,
+ "theoretical_loss": 3.4559848230770003,
+ "tokens_seen": 2020212736
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031348445336008027,
+ "loss": 1.3067,
+ "theoretical_loss": 3.455975570869711,
+ "tokens_seen": 2020278272
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031346840521564697,
+ "loss": 1.3007,
+ "theoretical_loss": 3.4559663190465852,
+ "tokens_seen": 2020343808
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031345235707121367,
+ "loss": 1.3201,
+ "theoretical_loss": 3.455957067607593,
+ "tokens_seen": 2020409344
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031343630892678037,
+ "loss": 1.1481,
+ "theoretical_loss": 3.455947816552707,
+ "tokens_seen": 2020474880
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031342026078234707,
+ "loss": 1.3199,
+ "theoretical_loss": 3.4559385658818984,
+ "tokens_seen": 2020540416
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031340421263791377,
+ "loss": 1.2353,
+ "theoretical_loss": 3.4559293155951383,
+ "tokens_seen": 2020605952
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031338816449348047,
+ "loss": 1.2676,
+ "theoretical_loss": 3.455920065692399,
+ "tokens_seen": 2020671488
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031337211634904717,
+ "loss": 1.258,
+ "theoretical_loss": 3.4559108161736516,
+ "tokens_seen": 2020737024
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031335606820461387,
+ "loss": 1.2013,
+ "theoretical_loss": 3.455901567038868,
+ "tokens_seen": 2020802560
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031334002006018057,
+ "loss": 1.2853,
+ "theoretical_loss": 3.45589231828802,
+ "tokens_seen": 2020868096
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003133239719157473,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4558830699210787,
+ "tokens_seen": 2020933632
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000313307923771314,
+ "loss": 1.3942,
+ "theoretical_loss": 3.455873821938016,
+ "tokens_seen": 2020999168
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003132918756268807,
+ "loss": 1.1987,
+ "theoretical_loss": 3.4558645743388037,
+ "tokens_seen": 2021064704
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003132758274824474,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4558553271234134,
+ "tokens_seen": 2021130240
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000313259779338014,
+ "loss": 1.2775,
+ "theoretical_loss": 3.455846080291816,
+ "tokens_seen": 2021195776
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003132437311935807,
+ "loss": 1.199,
+ "theoretical_loss": 3.455836833843984,
+ "tokens_seen": 2021261312
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003132276830491474,
+ "loss": 1.2561,
+ "theoretical_loss": 3.455827587779889,
+ "tokens_seen": 2021326848
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003132116349047141,
+ "loss": 1.1722,
+ "theoretical_loss": 3.4558183420995015,
+ "tokens_seen": 2021392384
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003131955867602808,
+ "loss": 1.3859,
+ "theoretical_loss": 3.4558090968027946,
+ "tokens_seen": 2021457920
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003131795386158475,
+ "loss": 1.2778,
+ "theoretical_loss": 3.4557998518897395,
+ "tokens_seen": 2021523456
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003131634904714142,
+ "loss": 1.2392,
+ "theoretical_loss": 3.4557906073603073,
+ "tokens_seen": 2021588992
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003131474423269809,
+ "loss": 1.2568,
+ "theoretical_loss": 3.45578136321447,
+ "tokens_seen": 2021654528
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003131313941825476,
+ "loss": 1.3003,
+ "theoretical_loss": 3.4557721194521998,
+ "tokens_seen": 2021720064
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 676834,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.277672290802002,
+ "objective/train/theoretical_loss": 3.455762876073467,
+ "objective/train/tokens_used": 2042245600,
+ "theoretical_loss": 3.455762876073467,
+ "tokens_seen": 2021785600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031311534603811433,
+ "loss": 1.2835,
+ "theoretical_loss": 3.455762876073467,
+ "tokens_seen": 2021785600
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031309929789368103,
+ "loss": 1.2825,
+ "theoretical_loss": 3.4557536330782446,
+ "tokens_seen": 2021851136
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031308324974924773,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4557443904665037,
+ "tokens_seen": 2021916672
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031306720160481443,
+ "loss": 1.2218,
+ "theoretical_loss": 3.4557351482382153,
+ "tokens_seen": 2021982208
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031305115346038113,
+ "loss": 1.3394,
+ "theoretical_loss": 3.4557259063933525,
+ "tokens_seen": 2022047744
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031303510531594783,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4557166649318862,
+ "tokens_seen": 2022113280
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031301905717151453,
+ "loss": 1.1825,
+ "theoretical_loss": 3.455707423853788,
+ "tokens_seen": 2022178816
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031300300902708123,
+ "loss": 1.2714,
+ "theoretical_loss": 3.455698183159029,
+ "tokens_seen": 2022244352
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031298696088264793,
+ "loss": 1.2884,
+ "theoretical_loss": 3.4556889428475825,
+ "tokens_seen": 2022309888
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031297091273821463,
+ "loss": 1.2069,
+ "theoretical_loss": 3.4556797029194186,
+ "tokens_seen": 2022375424
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031295486459378133,
+ "loss": 1.2378,
+ "theoretical_loss": 3.45567046337451,
+ "tokens_seen": 2022440960
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031293881644934803,
+ "loss": 1.3573,
+ "theoretical_loss": 3.455661224212828,
+ "tokens_seen": 2022506496
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031292276830491473,
+ "loss": 1.2299,
+ "theoretical_loss": 3.4556519854343435,
+ "tokens_seen": 2022572032
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003129067201604815,
+ "loss": 1.3058,
+ "theoretical_loss": 3.45564274703903,
+ "tokens_seen": 2022637568
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003128906720160482,
+ "loss": 1.2403,
+ "theoretical_loss": 3.4556335090268573,
+ "tokens_seen": 2022703104
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003128746238716149,
+ "loss": 1.2602,
+ "theoretical_loss": 3.4556242713977983,
+ "tokens_seen": 2022768640
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003128585757271816,
+ "loss": 1.2551,
+ "theoretical_loss": 3.4556150341518244,
+ "tokens_seen": 2022834176
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003128425275827483,
+ "loss": 1.3021,
+ "theoretical_loss": 3.455605797288907,
+ "tokens_seen": 2022899712
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000312826479438315,
+ "loss": 1.2694,
+ "theoretical_loss": 3.4555965608090187,
+ "tokens_seen": 2022965248
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003128104312938817,
+ "loss": 1.3417,
+ "theoretical_loss": 3.4555873247121296,
+ "tokens_seen": 2023030784
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127943831494484,
+ "loss": 1.3842,
+ "theoretical_loss": 3.4555780889982133,
+ "tokens_seen": 2023096320
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127783350050151,
+ "loss": 1.2525,
+ "theoretical_loss": 3.45556885366724,
+ "tokens_seen": 2023161856
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127622868605818,
+ "loss": 1.2779,
+ "theoretical_loss": 3.455559618719182,
+ "tokens_seen": 2023227392
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127462387161485,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4555503841540114,
+ "tokens_seen": 2023292928
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127301905717152,
+ "loss": 1.2426,
+ "theoretical_loss": 3.4555411499716993,
+ "tokens_seen": 2023358464
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003127141424272819,
+ "loss": 1.3509,
+ "theoretical_loss": 3.4555319161722178,
+ "tokens_seen": 2023424000
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003126980942828486,
+ "loss": 1.1984,
+ "theoretical_loss": 3.4555226827555385,
+ "tokens_seen": 2023489536
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003126820461384153,
+ "loss": 1.2994,
+ "theoretical_loss": 3.455513449721633,
+ "tokens_seen": 2023555072
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000312665997993982,
+ "loss": 1.1901,
+ "theoretical_loss": 3.4555042170704735,
+ "tokens_seen": 2023620608
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003126499498495487,
+ "loss": 1.2047,
+ "theoretical_loss": 3.4554949848020313,
+ "tokens_seen": 2023686144
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031263390170511535,
+ "loss": 1.3177,
+ "theoretical_loss": 3.455485752916278,
+ "tokens_seen": 2023751680
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031261785356068205,
+ "loss": 1.2972,
+ "theoretical_loss": 3.455476521413186,
+ "tokens_seen": 2023817216
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031260180541624875,
+ "loss": 1.1906,
+ "theoretical_loss": 3.4554672902927264,
+ "tokens_seen": 2023882752
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031258575727181545,
+ "loss": 1.1792,
+ "theoretical_loss": 3.4554580595548714,
+ "tokens_seen": 2023948288
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031256970912738215,
+ "loss": 1.1914,
+ "theoretical_loss": 3.455448829199592,
+ "tokens_seen": 2024013824
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031255366098294885,
+ "loss": 1.2878,
+ "theoretical_loss": 3.455439599226861,
+ "tokens_seen": 2024079360
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031253761283851555,
+ "loss": 1.2059,
+ "theoretical_loss": 3.45543036963665,
+ "tokens_seen": 2024144896
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031252156469408225,
+ "loss": 1.29,
+ "theoretical_loss": 3.45542114042893,
+ "tokens_seen": 2024210432
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031250551654964895,
+ "loss": 1.2594,
+ "theoretical_loss": 3.4554119116036732,
+ "tokens_seen": 2024275968
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031248946840521565,
+ "loss": 1.2785,
+ "theoretical_loss": 3.4554026831608518,
+ "tokens_seen": 2024341504
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031247342026078235,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4553934551004364,
+ "tokens_seen": 2024407040
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031245737211634905,
+ "loss": 1.3177,
+ "theoretical_loss": 3.4553842274224005,
+ "tokens_seen": 2024472576
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031244132397191575,
+ "loss": 1.2194,
+ "theoretical_loss": 3.455375000126714,
+ "tokens_seen": 2024538112
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031242527582748245,
+ "loss": 1.2977,
+ "theoretical_loss": 3.45536577321335,
+ "tokens_seen": 2024603648
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031240922768304916,
+ "loss": 1.2189,
+ "theoretical_loss": 3.45535654668228,
+ "tokens_seen": 2024669184
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031239317953861586,
+ "loss": 1.2709,
+ "theoretical_loss": 3.455347320533475,
+ "tokens_seen": 2024734720
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031237713139418256,
+ "loss": 1.2646,
+ "theoretical_loss": 3.455338094766908,
+ "tokens_seen": 2024800256
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031236108324974926,
+ "loss": 1.3225,
+ "theoretical_loss": 3.45532886938255,
+ "tokens_seen": 2024865792
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031234503510531596,
+ "loss": 1.3044,
+ "theoretical_loss": 3.4553196443803733,
+ "tokens_seen": 2024931328
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031232898696088266,
+ "loss": 1.201,
+ "theoretical_loss": 3.455310419760349,
+ "tokens_seen": 2024996864
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 678168,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2667977809906006,
+ "objective/train/theoretical_loss": 3.4553011955224497,
+ "objective/train/tokens_used": 2045522400,
+ "theoretical_loss": 3.4553011955224497,
+ "tokens_seen": 2025062400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031231293881644936,
+ "loss": 1.3154,
+ "theoretical_loss": 3.4553011955224497,
+ "tokens_seen": 2025062400
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031229689067201606,
+ "loss": 1.1993,
+ "theoretical_loss": 3.455291971666647,
+ "tokens_seen": 2025127936
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031228084252758276,
+ "loss": 1.3423,
+ "theoretical_loss": 3.4552827481929125,
+ "tokens_seen": 2025193472
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031226479438314946,
+ "loss": 1.3148,
+ "theoretical_loss": 3.4552735251012177,
+ "tokens_seen": 2025259008
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031224874623871616,
+ "loss": 1.3603,
+ "theoretical_loss": 3.4552643023915346,
+ "tokens_seen": 2025324544
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031223269809428286,
+ "loss": 1.3334,
+ "theoretical_loss": 3.4552550800638357,
+ "tokens_seen": 2025390080
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031221664994984956,
+ "loss": 1.2204,
+ "theoretical_loss": 3.4552458581180923,
+ "tokens_seen": 2025455616
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031220060180541626,
+ "loss": 1.2288,
+ "theoretical_loss": 3.455236636554276,
+ "tokens_seen": 2025521152
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031218455366098296,
+ "loss": 1.3527,
+ "theoretical_loss": 3.455227415372359,
+ "tokens_seen": 2025586688
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031216850551654966,
+ "loss": 1.1527,
+ "theoretical_loss": 3.455218194572313,
+ "tokens_seen": 2025652224
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031215245737211637,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4552089741541097,
+ "tokens_seen": 2025717760
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031213640922768307,
+ "loss": 1.2347,
+ "theoretical_loss": 3.4551997541177206,
+ "tokens_seen": 2025783296
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031212036108324977,
+ "loss": 1.2124,
+ "theoretical_loss": 3.455190534463119,
+ "tokens_seen": 2025848832
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031210431293881647,
+ "loss": 1.1825,
+ "theoretical_loss": 3.4551813151902753,
+ "tokens_seen": 2025914368
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031208826479438317,
+ "loss": 1.1782,
+ "theoretical_loss": 3.4551720962991617,
+ "tokens_seen": 2025979904
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031207221664994987,
+ "loss": 1.205,
+ "theoretical_loss": 3.45516287778975,
+ "tokens_seen": 2026045440
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031205616850551657,
+ "loss": 1.3805,
+ "theoretical_loss": 3.4551536596620123,
+ "tokens_seen": 2026110976
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031204012036108327,
+ "loss": 1.3957,
+ "theoretical_loss": 3.45514444191592,
+ "tokens_seen": 2026176512
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031202407221664997,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4551352245514453,
+ "tokens_seen": 2026242048
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031200802407221667,
+ "loss": 1.3274,
+ "theoretical_loss": 3.455126007568561,
+ "tokens_seen": 2026307584
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031199197592778337,
+ "loss": 1.2468,
+ "theoretical_loss": 3.455116790967237,
+ "tokens_seen": 2026373120
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031197592778335007,
+ "loss": 1.1995,
+ "theoretical_loss": 3.4551075747474465,
+ "tokens_seen": 2026438656
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003119598796389168,
+ "loss": 1.2539,
+ "theoretical_loss": 3.455098358909161,
+ "tokens_seen": 2026504192
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003119438314944835,
+ "loss": 1.2121,
+ "theoretical_loss": 3.4550891434523523,
+ "tokens_seen": 2026569728
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003119277833500502,
+ "loss": 1.202,
+ "theoretical_loss": 3.4550799283769926,
+ "tokens_seen": 2026635264
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003119117352056169,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4550707136830536,
+ "tokens_seen": 2026700800
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003118956870611836,
+ "loss": 1.2331,
+ "theoretical_loss": 3.4550614993705073,
+ "tokens_seen": 2026766336
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003118796389167503,
+ "loss": 1.2344,
+ "theoretical_loss": 3.455052285439325,
+ "tokens_seen": 2026831872
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003118635907723169,
+ "loss": 1.2727,
+ "theoretical_loss": 3.455043071889479,
+ "tokens_seen": 2026897408
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003118475426278836,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4550338587209413,
+ "tokens_seen": 2026962944
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003118314944834503,
+ "loss": 1.2247,
+ "theoretical_loss": 3.4550246459336833,
+ "tokens_seen": 2027028480
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000311815446339017,
+ "loss": 1.3285,
+ "theoretical_loss": 3.455015433527678,
+ "tokens_seen": 2027094016
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003117993981945837,
+ "loss": 1.3078,
+ "theoretical_loss": 3.455006221502896,
+ "tokens_seen": 2027159552
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003117833500501504,
+ "loss": 1.1593,
+ "theoretical_loss": 3.4549970098593104,
+ "tokens_seen": 2027225088
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003117673019057171,
+ "loss": 1.2019,
+ "theoretical_loss": 3.4549877985968918,
+ "tokens_seen": 2027290624
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003117512537612838,
+ "loss": 1.2882,
+ "theoretical_loss": 3.4549785877156136,
+ "tokens_seen": 2027356160
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031173520561685053,
+ "loss": 1.3323,
+ "theoretical_loss": 3.454969377215446,
+ "tokens_seen": 2027421696
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031171915747241723,
+ "loss": 1.25,
+ "theoretical_loss": 3.4549601670963623,
+ "tokens_seen": 2027487232
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000311703109327984,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4549509573583337,
+ "tokens_seen": 2027552768
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116870611835507,
+ "loss": 1.2289,
+ "theoretical_loss": 3.4549417480013327,
+ "tokens_seen": 2027618304
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116710130391174,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4549325390253305,
+ "tokens_seen": 2027683840
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116549648946841,
+ "loss": 1.1596,
+ "theoretical_loss": 3.4549233304302995,
+ "tokens_seen": 2027749376
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116389167502508,
+ "loss": 1.3987,
+ "theoretical_loss": 3.454914122216212,
+ "tokens_seen": 2027814912
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116228686058175,
+ "loss": 1.2311,
+ "theoretical_loss": 3.4549049143830386,
+ "tokens_seen": 2027880448
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003116068204613842,
+ "loss": 1.3269,
+ "theoretical_loss": 3.4548957069307527,
+ "tokens_seen": 2027945984
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003115907723169509,
+ "loss": 1.3215,
+ "theoretical_loss": 3.454886499859325,
+ "tokens_seen": 2028011520
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003115747241725176,
+ "loss": 1.2291,
+ "theoretical_loss": 3.4548772931687286,
+ "tokens_seen": 2028077056
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003115586760280843,
+ "loss": 1.3623,
+ "theoretical_loss": 3.4548680868589345,
+ "tokens_seen": 2028142592
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000311542627883651,
+ "loss": 1.3173,
+ "theoretical_loss": 3.4548588809299154,
+ "tokens_seen": 2028208128
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003115265797392177,
+ "loss": 1.2773,
+ "theoretical_loss": 3.454849675381643,
+ "tokens_seen": 2028273664
+ },
+ {
+ "epoch": 0.61,
+ "objective/train/docs_used": 679103,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3795264959335327,
+ "objective/train/theoretical_loss": 3.454840470214089,
+ "objective/train/tokens_used": 2048799200,
+ "theoretical_loss": 3.454840470214089,
+ "tokens_seen": 2028339200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003115105315947844,
+ "loss": 1.2951,
+ "theoretical_loss": 3.454840470214089,
+ "tokens_seen": 2028339200
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114944834503511,
+ "loss": 1.2206,
+ "theoretical_loss": 3.454831265427225,
+ "tokens_seen": 2028404736
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114784353059178,
+ "loss": 1.2835,
+ "theoretical_loss": 3.454822061021024,
+ "tokens_seen": 2028470272
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114623871614845,
+ "loss": 1.2287,
+ "theoretical_loss": 3.4548128569954573,
+ "tokens_seen": 2028535808
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114463390170512,
+ "loss": 1.2621,
+ "theoretical_loss": 3.454803653350497,
+ "tokens_seen": 2028601344
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114302908726179,
+ "loss": 1.1892,
+ "theoretical_loss": 3.4547944500861147,
+ "tokens_seen": 2028666880
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003114142427281846,
+ "loss": 1.2138,
+ "theoretical_loss": 3.454785247202283,
+ "tokens_seen": 2028732416
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113981945837513,
+ "loss": 1.2948,
+ "theoretical_loss": 3.4547760446989737,
+ "tokens_seen": 2028797952
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.000311382146439318,
+ "loss": 1.2765,
+ "theoretical_loss": 3.4547668425761584,
+ "tokens_seen": 2028863488
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113660982948847,
+ "loss": 1.3274,
+ "theoretical_loss": 3.45475764083381,
+ "tokens_seen": 2028929024
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113500501504514,
+ "loss": 1.3042,
+ "theoretical_loss": 3.454748439471899,
+ "tokens_seen": 2028994560
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113340020060181,
+ "loss": 1.1862,
+ "theoretical_loss": 3.454739238490398,
+ "tokens_seen": 2029060096
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113179538615848,
+ "loss": 1.3196,
+ "theoretical_loss": 3.45473003788928,
+ "tokens_seen": 2029125632
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003113019057171515,
+ "loss": 1.1663,
+ "theoretical_loss": 3.454720837668516,
+ "tokens_seen": 2029191168
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003112858575727182,
+ "loss": 1.2578,
+ "theoretical_loss": 3.454711637828078,
+ "tokens_seen": 2029256704
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003112698094282849,
+ "loss": 1.2119,
+ "theoretical_loss": 3.454702438367938,
+ "tokens_seen": 2029322240
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.0003112537612838516,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4546932392880683,
+ "tokens_seen": 2029387776
+ },
+ {
+ "epoch": 0.61,
+ "learning_rate": 0.00031123771313941825,
+ "loss": 1.1998,
+ "theoretical_loss": 3.454684040588441,
+ "tokens_seen": 2029453312
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031122166499498495,
+ "loss": 1.2048,
+ "theoretical_loss": 3.4546748422690277,
+ "tokens_seen": 2029518848
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031120561685055165,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4546656443298005,
+ "tokens_seen": 2029584384
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031118956870611835,
+ "loss": 1.187,
+ "theoretical_loss": 3.4546564467707315,
+ "tokens_seen": 2029649920
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031117352056168505,
+ "loss": 1.3663,
+ "theoretical_loss": 3.454647249591793,
+ "tokens_seen": 2029715456
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031115747241725175,
+ "loss": 1.3059,
+ "theoretical_loss": 3.454638052792956,
+ "tokens_seen": 2029780992
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031114142427281845,
+ "loss": 1.3897,
+ "theoretical_loss": 3.454628856374194,
+ "tokens_seen": 2029846528
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031112537612838515,
+ "loss": 1.3425,
+ "theoretical_loss": 3.454619660335478,
+ "tokens_seen": 2029912064
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031110932798395185,
+ "loss": 1.261,
+ "theoretical_loss": 3.45461046467678,
+ "tokens_seen": 2029977600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031109327983951855,
+ "loss": 1.2895,
+ "theoretical_loss": 3.4546012693980725,
+ "tokens_seen": 2030043136
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031107723169508525,
+ "loss": 1.2439,
+ "theoretical_loss": 3.4545920744993275,
+ "tokens_seen": 2030108672
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031106118355065195,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4545828799805167,
+ "tokens_seen": 2030174208
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031104513540621866,
+ "loss": 1.2177,
+ "theoretical_loss": 3.4545736858416127,
+ "tokens_seen": 2030239744
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031102908726178536,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4545644920825866,
+ "tokens_seen": 2030305280
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031101303911735206,
+ "loss": 1.2976,
+ "theoretical_loss": 3.4545552987034114,
+ "tokens_seen": 2030370816
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031099699097291876,
+ "loss": 1.2443,
+ "theoretical_loss": 3.4545461057040585,
+ "tokens_seen": 2030436352
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031098094282848546,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4545369130845,
+ "tokens_seen": 2030501888
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031096489468405216,
+ "loss": 1.2728,
+ "theoretical_loss": 3.454527720844709,
+ "tokens_seen": 2030567424
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031094884653961886,
+ "loss": 1.2414,
+ "theoretical_loss": 3.454518528984656,
+ "tokens_seen": 2030632960
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031093279839518556,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4545093375043137,
+ "tokens_seen": 2030698496
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031091675025075226,
+ "loss": 1.264,
+ "theoretical_loss": 3.4545001464036544,
+ "tokens_seen": 2030764032
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031090070210631896,
+ "loss": 1.2105,
+ "theoretical_loss": 3.45449095568265,
+ "tokens_seen": 2030829568
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031090070210631896,
+ "loss": 1.145,
+ "theoretical_loss": 3.4544817653412725,
+ "tokens_seen": 2030895104
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031088465396188566,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4544725753794943,
+ "tokens_seen": 2030960640
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031086860581745236,
+ "loss": 1.2781,
+ "theoretical_loss": 3.4544633857972866,
+ "tokens_seen": 2031026176
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031085255767301906,
+ "loss": 1.2642,
+ "theoretical_loss": 3.4544541965946225,
+ "tokens_seen": 2031091712
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031083650952858576,
+ "loss": 1.1627,
+ "theoretical_loss": 3.454445007771474,
+ "tokens_seen": 2031157248
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031082046138415246,
+ "loss": 1.2297,
+ "theoretical_loss": 3.454435819327812,
+ "tokens_seen": 2031222784
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031080441323971916,
+ "loss": 1.3006,
+ "theoretical_loss": 3.45442663126361,
+ "tokens_seen": 2031288320
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031078836509528587,
+ "loss": 1.2529,
+ "theoretical_loss": 3.454417443578839,
+ "tokens_seen": 2031353856
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031077231695085257,
+ "loss": 1.1862,
+ "theoretical_loss": 3.4544082562734717,
+ "tokens_seen": 2031419392
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031075626880641927,
+ "loss": 1.2043,
+ "theoretical_loss": 3.4543990693474806,
+ "tokens_seen": 2031484928
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031074022066198597,
+ "loss": 1.2927,
+ "theoretical_loss": 3.454389882800837,
+ "tokens_seen": 2031550464
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 680422,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2129366397857666,
+ "objective/train/theoretical_loss": 3.454380696633513,
+ "objective/train/tokens_used": 2052076000,
+ "theoretical_loss": 3.454380696633513,
+ "tokens_seen": 2031616000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031072417251755267,
+ "loss": 1.2718,
+ "theoretical_loss": 3.454380696633513,
+ "tokens_seen": 2031616000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031070812437311937,
+ "loss": 1.3017,
+ "theoretical_loss": 3.4543715108454816,
+ "tokens_seen": 2031681536
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031069207622868607,
+ "loss": 1.2935,
+ "theoretical_loss": 3.4543623254367137,
+ "tokens_seen": 2031747072
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031067602808425277,
+ "loss": 1.3038,
+ "theoretical_loss": 3.454353140407182,
+ "tokens_seen": 2031812608
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031065997993981947,
+ "loss": 1.2507,
+ "theoretical_loss": 3.454343955756859,
+ "tokens_seen": 2031878144
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031064393179538617,
+ "loss": 1.3444,
+ "theoretical_loss": 3.4543347714857164,
+ "tokens_seen": 2031943680
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031062788365095287,
+ "loss": 1.2736,
+ "theoretical_loss": 3.454325587593726,
+ "tokens_seen": 2032009216
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031061183550651957,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4543164040808603,
+ "tokens_seen": 2032074752
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031059578736208627,
+ "loss": 1.4047,
+ "theoretical_loss": 3.4543072209470917,
+ "tokens_seen": 2032140288
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000310579739217653,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4542980381923916,
+ "tokens_seen": 2032205824
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003105636910732197,
+ "loss": 1.2748,
+ "theoretical_loss": 3.454288855816733,
+ "tokens_seen": 2032271360
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003105476429287864,
+ "loss": 1.237,
+ "theoretical_loss": 3.4542796738200874,
+ "tokens_seen": 2032336896
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003105315947843531,
+ "loss": 1.2959,
+ "theoretical_loss": 3.4542704922024265,
+ "tokens_seen": 2032402432
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003105155466399198,
+ "loss": 1.1759,
+ "theoretical_loss": 3.454261310963724,
+ "tokens_seen": 2032467968
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104994984954865,
+ "loss": 1.2335,
+ "theoretical_loss": 3.45425213010395,
+ "tokens_seen": 2032533504
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104834503510532,
+ "loss": 1.1578,
+ "theoretical_loss": 3.4542429496230787,
+ "tokens_seen": 2032599040
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104674022066199,
+ "loss": 1.2512,
+ "theoretical_loss": 3.454233769521081,
+ "tokens_seen": 2032664576
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104513540621866,
+ "loss": 1.2635,
+ "theoretical_loss": 3.454224589797929,
+ "tokens_seen": 2032730112
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104353059177533,
+ "loss": 1.2513,
+ "theoretical_loss": 3.4542154104535956,
+ "tokens_seen": 2032795648
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00031041925777332,
+ "loss": 1.2434,
+ "theoretical_loss": 3.4542062314880524,
+ "tokens_seen": 2032861184
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003104032096288867,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4541970529012715,
+ "tokens_seen": 2032926720
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103871614844534,
+ "loss": 1.3071,
+ "theoretical_loss": 3.4541878746932255,
+ "tokens_seen": 2032992256
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103711133400201,
+ "loss": 1.327,
+ "theoretical_loss": 3.4541786968638863,
+ "tokens_seen": 2033057792
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103550651955868,
+ "loss": 1.334,
+ "theoretical_loss": 3.4541695194132256,
+ "tokens_seen": 2033123328
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103390170511535,
+ "loss": 1.2841,
+ "theoretical_loss": 3.4541603423412166,
+ "tokens_seen": 2033188864
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103229689067202,
+ "loss": 1.2503,
+ "theoretical_loss": 3.454151165647831,
+ "tokens_seen": 2033254400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003103069207622869,
+ "loss": 1.1638,
+ "theoretical_loss": 3.4541419893330403,
+ "tokens_seen": 2033319936
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003102908726178536,
+ "loss": 1.2743,
+ "theoretical_loss": 3.454132813396818,
+ "tokens_seen": 2033385472
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003102748244734203,
+ "loss": 1.3382,
+ "theoretical_loss": 3.4541236378391345,
+ "tokens_seen": 2033451008
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000310258776328987,
+ "loss": 1.3748,
+ "theoretical_loss": 3.454114462659964,
+ "tokens_seen": 2033516544
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003102427281845537,
+ "loss": 1.1973,
+ "theoretical_loss": 3.454105287859277,
+ "tokens_seen": 2033582080
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003102266800401204,
+ "loss": 1.2616,
+ "theoretical_loss": 3.454096113437047,
+ "tokens_seen": 2033647616
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003102106318956871,
+ "loss": 1.27,
+ "theoretical_loss": 3.454086939393245,
+ "tokens_seen": 2033713152
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101945837512538,
+ "loss": 1.214,
+ "theoretical_loss": 3.4540777657278445,
+ "tokens_seen": 2033778688
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101785356068205,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4540685924408168,
+ "tokens_seen": 2033844224
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101624874623872,
+ "loss": 1.3315,
+ "theoretical_loss": 3.4540594195321344,
+ "tokens_seen": 2033909760
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101464393179539,
+ "loss": 1.2739,
+ "theoretical_loss": 3.4540502470017693,
+ "tokens_seen": 2033975296
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101303911735206,
+ "loss": 1.3093,
+ "theoretical_loss": 3.4540410748496937,
+ "tokens_seen": 2034040832
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003101143430290873,
+ "loss": 1.2014,
+ "theoretical_loss": 3.4540319030758804,
+ "tokens_seen": 2034106368
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000310098294884654,
+ "loss": 1.3909,
+ "theoretical_loss": 3.4540227316803005,
+ "tokens_seen": 2034171904
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100822467402207,
+ "loss": 1.2074,
+ "theoretical_loss": 3.4540135606629274,
+ "tokens_seen": 2034237440
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100661985957874,
+ "loss": 1.2799,
+ "theoretical_loss": 3.4540043900237327,
+ "tokens_seen": 2034302976
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100501504513541,
+ "loss": 1.3112,
+ "theoretical_loss": 3.453995219762688,
+ "tokens_seen": 2034368512
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100341023069208,
+ "loss": 1.1292,
+ "theoretical_loss": 3.453986049879767,
+ "tokens_seen": 2034434048
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100180541624875,
+ "loss": 1.3172,
+ "theoretical_loss": 3.453976880374941,
+ "tokens_seen": 2034499584
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003100020060180542,
+ "loss": 1.2884,
+ "theoretical_loss": 3.4539677112481826,
+ "tokens_seen": 2034565120
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003099859578736209,
+ "loss": 1.2518,
+ "theoretical_loss": 3.453958542499463,
+ "tokens_seen": 2034630656
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003099699097291876,
+ "loss": 1.2859,
+ "theoretical_loss": 3.453949374128756,
+ "tokens_seen": 2034696192
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003099538615847543,
+ "loss": 1.2139,
+ "theoretical_loss": 3.453940206136033,
+ "tokens_seen": 2034761728
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000309937813440321,
+ "loss": 1.4163,
+ "theoretical_loss": 3.453931038521266,
+ "tokens_seen": 2034827264
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 681328,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.229004144668579,
+ "objective/train/theoretical_loss": 3.453921871284428,
+ "objective/train/tokens_used": 2055352800,
+ "theoretical_loss": 3.453921871284428,
+ "tokens_seen": 2034892800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003099217652958877,
+ "loss": 1.1501,
+ "theoretical_loss": 3.453921871284428,
+ "tokens_seen": 2034892800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003099057171514544,
+ "loss": 1.1942,
+ "theoretical_loss": 3.453912704425491,
+ "tokens_seen": 2034958336
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003098896690070211,
+ "loss": 1.346,
+ "theoretical_loss": 3.4539035379444267,
+ "tokens_seen": 2035023872
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003098736208625878,
+ "loss": 1.2499,
+ "theoretical_loss": 3.4538943718412076,
+ "tokens_seen": 2035089408
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003098575727181545,
+ "loss": 1.2287,
+ "theoretical_loss": 3.4538852061158067,
+ "tokens_seen": 2035154944
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003098415245737212,
+ "loss": 1.2177,
+ "theoretical_loss": 3.453876040768195,
+ "tokens_seen": 2035220480
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030982547642928785,
+ "loss": 1.2186,
+ "theoretical_loss": 3.4538668757983464,
+ "tokens_seen": 2035286016
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030980942828485455,
+ "loss": 1.2747,
+ "theoretical_loss": 3.4538577112062314,
+ "tokens_seen": 2035351552
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030979338014042125,
+ "loss": 1.3151,
+ "theoretical_loss": 3.453848546991823,
+ "tokens_seen": 2035417088
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030977733199598795,
+ "loss": 1.291,
+ "theoretical_loss": 3.453839383155094,
+ "tokens_seen": 2035482624
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030976128385155465,
+ "loss": 1.2808,
+ "theoretical_loss": 3.453830219696016,
+ "tokens_seen": 2035548160
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030974523570712135,
+ "loss": 1.2123,
+ "theoretical_loss": 3.453821056614562,
+ "tokens_seen": 2035613696
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030972918756268805,
+ "loss": 1.3032,
+ "theoretical_loss": 3.453811893910703,
+ "tokens_seen": 2035679232
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030971313941825475,
+ "loss": 1.3071,
+ "theoretical_loss": 3.4538027315844126,
+ "tokens_seen": 2035744768
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030969709127382145,
+ "loss": 1.181,
+ "theoretical_loss": 3.4537935696356623,
+ "tokens_seen": 2035810304
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030968104312938815,
+ "loss": 1.272,
+ "theoretical_loss": 3.4537844080644247,
+ "tokens_seen": 2035875840
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030966499498495486,
+ "loss": 1.2686,
+ "theoretical_loss": 3.453775246870672,
+ "tokens_seen": 2035941376
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030964894684052156,
+ "loss": 1.3049,
+ "theoretical_loss": 3.453766086054377,
+ "tokens_seen": 2036006912
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030963289869608826,
+ "loss": 1.3939,
+ "theoretical_loss": 3.4537569256155107,
+ "tokens_seen": 2036072448
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030961685055165496,
+ "loss": 1.171,
+ "theoretical_loss": 3.4537477655540467,
+ "tokens_seen": 2036137984
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030960080240722166,
+ "loss": 1.2487,
+ "theoretical_loss": 3.453738605869957,
+ "tokens_seen": 2036203520
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030958475426278836,
+ "loss": 1.3922,
+ "theoretical_loss": 3.4537294465632136,
+ "tokens_seen": 2036269056
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030956870611835506,
+ "loss": 1.2367,
+ "theoretical_loss": 3.4537202876337885,
+ "tokens_seen": 2036334592
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030955265797392176,
+ "loss": 1.2443,
+ "theoretical_loss": 3.453711129081655,
+ "tokens_seen": 2036400128
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030953660982948846,
+ "loss": 1.1168,
+ "theoretical_loss": 3.453701970906785,
+ "tokens_seen": 2036465664
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030952056168505516,
+ "loss": 1.3242,
+ "theoretical_loss": 3.4536928131091504,
+ "tokens_seen": 2036531200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030950451354062186,
+ "loss": 1.1845,
+ "theoretical_loss": 3.453683655688724,
+ "tokens_seen": 2036596736
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030948846539618856,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4536744986454777,
+ "tokens_seen": 2036662272
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030947241725175526,
+ "loss": 1.2069,
+ "theoretical_loss": 3.4536653419793844,
+ "tokens_seen": 2036727808
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030945636910732196,
+ "loss": 1.2663,
+ "theoretical_loss": 3.4536561856904155,
+ "tokens_seen": 2036793344
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030944032096288866,
+ "loss": 1.3526,
+ "theoretical_loss": 3.4536470297785447,
+ "tokens_seen": 2036858880
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030942427281845536,
+ "loss": 1.217,
+ "theoretical_loss": 3.4536378742437432,
+ "tokens_seen": 2036924416
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030940822467402207,
+ "loss": 1.3023,
+ "theoretical_loss": 3.453628719085984,
+ "tokens_seen": 2036989952
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030939217652958877,
+ "loss": 1.2773,
+ "theoretical_loss": 3.4536195643052388,
+ "tokens_seen": 2037055488
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030937612838515547,
+ "loss": 1.1703,
+ "theoretical_loss": 3.4536104099014806,
+ "tokens_seen": 2037121024
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030936008024072217,
+ "loss": 1.3722,
+ "theoretical_loss": 3.4536012558746814,
+ "tokens_seen": 2037186560
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030934403209628887,
+ "loss": 1.2888,
+ "theoretical_loss": 3.4535921022248135,
+ "tokens_seen": 2037252096
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030932798395185557,
+ "loss": 1.2585,
+ "theoretical_loss": 3.4535829489518495,
+ "tokens_seen": 2037317632
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030931193580742227,
+ "loss": 1.3362,
+ "theoretical_loss": 3.4535737960557613,
+ "tokens_seen": 2037383168
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000309295887662989,
+ "loss": 1.2632,
+ "theoretical_loss": 3.453564643536522,
+ "tokens_seen": 2037448704
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003092798395185557,
+ "loss": 1.2392,
+ "theoretical_loss": 3.453555491394104,
+ "tokens_seen": 2037514240
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003092637913741224,
+ "loss": 1.4172,
+ "theoretical_loss": 3.4535463396284785,
+ "tokens_seen": 2037579776
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003092477432296891,
+ "loss": 1.3359,
+ "theoretical_loss": 3.4535371882396184,
+ "tokens_seen": 2037645312
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003092316950852558,
+ "loss": 1.2016,
+ "theoretical_loss": 3.453528037227497,
+ "tokens_seen": 2037710848
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030921564694082253,
+ "loss": 1.3472,
+ "theoretical_loss": 3.453518886592085,
+ "tokens_seen": 2037776384
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091995987963892,
+ "loss": 1.2386,
+ "theoretical_loss": 3.4535097363333565,
+ "tokens_seen": 2037841920
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091835506519559,
+ "loss": 1.2378,
+ "theoretical_loss": 3.4535005864512827,
+ "tokens_seen": 2037907456
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091675025075226,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4534914369458365,
+ "tokens_seen": 2037972992
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091514543630893,
+ "loss": 1.3474,
+ "theoretical_loss": 3.4534822878169904,
+ "tokens_seen": 2038038528
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000309135406218656,
+ "loss": 1.3072,
+ "theoretical_loss": 3.453473139064716,
+ "tokens_seen": 2038104064
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 682560,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.257143497467041,
+ "objective/train/theoretical_loss": 3.453463990688987,
+ "objective/train/tokens_used": 2058629600,
+ "theoretical_loss": 3.453463990688987,
+ "tokens_seen": 2038169600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091193580742227,
+ "loss": 1.2463,
+ "theoretical_loss": 3.453463990688987,
+ "tokens_seen": 2038169600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003091033099297894,
+ "loss": 1.3082,
+ "theoretical_loss": 3.4534548426897747,
+ "tokens_seen": 2038235136
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090872617853561,
+ "loss": 1.2113,
+ "theoretical_loss": 3.4534456950670513,
+ "tokens_seen": 2038300672
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090712136409228,
+ "loss": 1.3351,
+ "theoretical_loss": 3.4534365478207905,
+ "tokens_seen": 2038366208
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090551654964895,
+ "loss": 1.2689,
+ "theoretical_loss": 3.4534274009509636,
+ "tokens_seen": 2038431744
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090391173520562,
+ "loss": 1.21,
+ "theoretical_loss": 3.453418254457543,
+ "tokens_seen": 2038497280
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090230692076229,
+ "loss": 1.2668,
+ "theoretical_loss": 3.453409108340502,
+ "tokens_seen": 2038562816
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003090070210631896,
+ "loss": 1.3273,
+ "theoretical_loss": 3.4533999625998124,
+ "tokens_seen": 2038628352
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003089909729187563,
+ "loss": 1.3636,
+ "theoretical_loss": 3.4533908172354466,
+ "tokens_seen": 2038693888
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000308974924774323,
+ "loss": 1.2644,
+ "theoretical_loss": 3.453381672247377,
+ "tokens_seen": 2038759424
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003089588766298897,
+ "loss": 1.2004,
+ "theoretical_loss": 3.4533725276355764,
+ "tokens_seen": 2038824960
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003089428284854564,
+ "loss": 1.2006,
+ "theoretical_loss": 3.453363383400017,
+ "tokens_seen": 2038890496
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003089267803410231,
+ "loss": 1.2727,
+ "theoretical_loss": 3.453354239540671,
+ "tokens_seen": 2038956032
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003089107321965898,
+ "loss": 1.36,
+ "theoretical_loss": 3.453345096057511,
+ "tokens_seen": 2039021568
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003088946840521565,
+ "loss": 1.3566,
+ "theoretical_loss": 3.453335952950509,
+ "tokens_seen": 2039087104
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003088786359077232,
+ "loss": 1.2552,
+ "theoretical_loss": 3.4533268102196386,
+ "tokens_seen": 2039152640
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003088625877632899,
+ "loss": 1.1587,
+ "theoretical_loss": 3.453317667864871,
+ "tokens_seen": 2039218176
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003088465396188566,
+ "loss": 1.3309,
+ "theoretical_loss": 3.4533085258861798,
+ "tokens_seen": 2039283712
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003088304914744233,
+ "loss": 1.3319,
+ "theoretical_loss": 3.453299384283536,
+ "tokens_seen": 2039349248
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030881444332999,
+ "loss": 1.3868,
+ "theoretical_loss": 3.4532902430569132,
+ "tokens_seen": 2039414784
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087983951855567,
+ "loss": 1.344,
+ "theoretical_loss": 3.453281102206284,
+ "tokens_seen": 2039480320
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087823470411234,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4532719617316197,
+ "tokens_seen": 2039545856
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087662988966901,
+ "loss": 1.34,
+ "theoretical_loss": 3.4532628216328938,
+ "tokens_seen": 2039611392
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087502507522568,
+ "loss": 1.3847,
+ "theoretical_loss": 3.453253681910078,
+ "tokens_seen": 2039676928
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087342026078235,
+ "loss": 1.328,
+ "theoretical_loss": 3.4532445425631453,
+ "tokens_seen": 2039742464
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087181544633902,
+ "loss": 1.3318,
+ "theoretical_loss": 3.4532354035920685,
+ "tokens_seen": 2039808000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003087021063189569,
+ "loss": 1.2651,
+ "theoretical_loss": 3.453226264996819,
+ "tokens_seen": 2039873536
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003086860581745236,
+ "loss": 1.2475,
+ "theoretical_loss": 3.45321712677737,
+ "tokens_seen": 2039939072
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003086700100300903,
+ "loss": 1.3265,
+ "theoretical_loss": 3.4532079889336935,
+ "tokens_seen": 2040004608
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000308653961885657,
+ "loss": 1.4194,
+ "theoretical_loss": 3.4531988514657628,
+ "tokens_seen": 2040070144
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003086379137412237,
+ "loss": 1.2217,
+ "theoretical_loss": 3.4531897143735497,
+ "tokens_seen": 2040135680
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003086218655967904,
+ "loss": 1.2882,
+ "theoretical_loss": 3.453180577657027,
+ "tokens_seen": 2040201216
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003086058174523571,
+ "loss": 1.2884,
+ "theoretical_loss": 3.4531714413161665,
+ "tokens_seen": 2040266752
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085897693079238,
+ "loss": 1.2864,
+ "theoretical_loss": 3.4531623053509417,
+ "tokens_seen": 2040332288
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085737211634905,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4531531697613245,
+ "tokens_seen": 2040397824
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085576730190572,
+ "loss": 1.3145,
+ "theoretical_loss": 3.4531440345472877,
+ "tokens_seen": 2040463360
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085416248746239,
+ "loss": 1.2018,
+ "theoretical_loss": 3.453134899708803,
+ "tokens_seen": 2040528896
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085255767301906,
+ "loss": 1.2823,
+ "theoretical_loss": 3.453125765245844,
+ "tokens_seen": 2040594432
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003085095285857573,
+ "loss": 1.2244,
+ "theoretical_loss": 3.4531166311583825,
+ "tokens_seen": 2040659968
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000308493480441324,
+ "loss": 1.273,
+ "theoretical_loss": 3.453107497446392,
+ "tokens_seen": 2040725504
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003084774322968907,
+ "loss": 1.1891,
+ "theoretical_loss": 3.453098364109843,
+ "tokens_seen": 2040791040
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003084613841524574,
+ "loss": 1.3062,
+ "theoretical_loss": 3.45308923114871,
+ "tokens_seen": 2040856576
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003084453360080241,
+ "loss": 1.2875,
+ "theoretical_loss": 3.4530800985629644,
+ "tokens_seen": 2040922112
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030842928786359075,
+ "loss": 1.2361,
+ "theoretical_loss": 3.453070966352579,
+ "tokens_seen": 2040987648
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030841323971915745,
+ "loss": 1.2962,
+ "theoretical_loss": 3.453061834517527,
+ "tokens_seen": 2041053184
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030839719157472415,
+ "loss": 1.3262,
+ "theoretical_loss": 3.4530527030577796,
+ "tokens_seen": 2041118720
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030838114343029085,
+ "loss": 1.214,
+ "theoretical_loss": 3.4530435719733106,
+ "tokens_seen": 2041184256
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030836509528585755,
+ "loss": 1.1722,
+ "theoretical_loss": 3.4530344412640916,
+ "tokens_seen": 2041249792
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030834904714142425,
+ "loss": 1.3134,
+ "theoretical_loss": 3.4530253109300957,
+ "tokens_seen": 2041315328
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030833299899699095,
+ "loss": 1.3019,
+ "theoretical_loss": 3.453016180971295,
+ "tokens_seen": 2041380864
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 683764,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.5399019718170166,
+ "objective/train/theoretical_loss": 3.4530070513876625,
+ "objective/train/tokens_used": 2061906400,
+ "theoretical_loss": 3.4530070513876625,
+ "tokens_seen": 2041446400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030831695085255765,
+ "loss": 1.3933,
+ "theoretical_loss": 3.4530070513876625,
+ "tokens_seen": 2041446400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030830090270812436,
+ "loss": 1.1885,
+ "theoretical_loss": 3.4529979221791707,
+ "tokens_seen": 2041511936
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030828485456369106,
+ "loss": 1.2352,
+ "theoretical_loss": 3.4529887933457917,
+ "tokens_seen": 2041577472
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030826880641925776,
+ "loss": 1.2362,
+ "theoretical_loss": 3.452979664887498,
+ "tokens_seen": 2041643008
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030825275827482446,
+ "loss": 1.3164,
+ "theoretical_loss": 3.452970536804263,
+ "tokens_seen": 2041708544
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030823671013039116,
+ "loss": 1.2778,
+ "theoretical_loss": 3.4529614090960585,
+ "tokens_seen": 2041774080
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030822066198595786,
+ "loss": 1.331,
+ "theoretical_loss": 3.4529522817628573,
+ "tokens_seen": 2041839616
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030820461384152456,
+ "loss": 1.3027,
+ "theoretical_loss": 3.4529431548046317,
+ "tokens_seen": 2041905152
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030818856569709126,
+ "loss": 1.3107,
+ "theoretical_loss": 3.452934028221355,
+ "tokens_seen": 2041970688
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030817251755265796,
+ "loss": 1.2855,
+ "theoretical_loss": 3.452924902012999,
+ "tokens_seen": 2042036224
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030815646940822466,
+ "loss": 1.1502,
+ "theoretical_loss": 3.4529157761795357,
+ "tokens_seen": 2042101760
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030814042126379136,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4529066507209394,
+ "tokens_seen": 2042167296
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030812437311935806,
+ "loss": 1.273,
+ "theoretical_loss": 3.4528975256371814,
+ "tokens_seen": 2042232832
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030810832497492476,
+ "loss": 1.2352,
+ "theoretical_loss": 3.452888400928235,
+ "tokens_seen": 2042298368
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003080922768304915,
+ "loss": 1.2075,
+ "theoretical_loss": 3.452879276594072,
+ "tokens_seen": 2042363904
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003080762286860582,
+ "loss": 1.3181,
+ "theoretical_loss": 3.4528701526346657,
+ "tokens_seen": 2042429440
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003080601805416249,
+ "loss": 1.2302,
+ "theoretical_loss": 3.4528610290499877,
+ "tokens_seen": 2042494976
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003080441323971916,
+ "loss": 1.2892,
+ "theoretical_loss": 3.452851905840012,
+ "tokens_seen": 2042560512
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003080280842527583,
+ "loss": 1.2011,
+ "theoretical_loss": 3.45284278300471,
+ "tokens_seen": 2042626048
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000308012036108325,
+ "loss": 1.3596,
+ "theoretical_loss": 3.452833660544055,
+ "tokens_seen": 2042691584
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079959879638917,
+ "loss": 1.2907,
+ "theoretical_loss": 3.452824538458019,
+ "tokens_seen": 2042757120
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079799398194584,
+ "loss": 1.2684,
+ "theoretical_loss": 3.452815416746575,
+ "tokens_seen": 2042822656
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079638916750251,
+ "loss": 1.3933,
+ "theoretical_loss": 3.452806295409696,
+ "tokens_seen": 2042888192
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079478435305918,
+ "loss": 1.3091,
+ "theoretical_loss": 3.452797174447354,
+ "tokens_seen": 2042953728
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079317953861585,
+ "loss": 1.28,
+ "theoretical_loss": 3.4527880538595213,
+ "tokens_seen": 2043019264
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003079157472417252,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4527789336461714,
+ "tokens_seen": 2043084800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003078996990972919,
+ "loss": 1.2647,
+ "theoretical_loss": 3.452769813807276,
+ "tokens_seen": 2043150336
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003078836509528586,
+ "loss": 1.2771,
+ "theoretical_loss": 3.4527606943428086,
+ "tokens_seen": 2043215872
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003078676028084253,
+ "loss": 1.2049,
+ "theoretical_loss": 3.452751575252741,
+ "tokens_seen": 2043281408
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000307851554663992,
+ "loss": 1.3456,
+ "theoretical_loss": 3.452742456537047,
+ "tokens_seen": 2043346944
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030783550651955873,
+ "loss": 1.3662,
+ "theoretical_loss": 3.4527333381956975,
+ "tokens_seen": 2043412480
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030781945837512543,
+ "loss": 1.3178,
+ "theoretical_loss": 3.4527242202286663,
+ "tokens_seen": 2043478016
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030780341023069213,
+ "loss": 1.3491,
+ "theoretical_loss": 3.452715102635926,
+ "tokens_seen": 2043543552
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077873620862588,
+ "loss": 1.3203,
+ "theoretical_loss": 3.452705985417449,
+ "tokens_seen": 2043609088
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077713139418255,
+ "loss": 1.2479,
+ "theoretical_loss": 3.452696868573208,
+ "tokens_seen": 2043674624
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077552657973922,
+ "loss": 1.1678,
+ "theoretical_loss": 3.452687752103176,
+ "tokens_seen": 2043740160
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077392176529589,
+ "loss": 1.2707,
+ "theoretical_loss": 3.4526786360073247,
+ "tokens_seen": 2043805696
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077231695085256,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4526695202856272,
+ "tokens_seen": 2043871232
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003077071213640923,
+ "loss": 1.32,
+ "theoretical_loss": 3.4526604049380563,
+ "tokens_seen": 2043936768
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000307691073219659,
+ "loss": 1.2236,
+ "theoretical_loss": 3.452651289964585,
+ "tokens_seen": 2044002304
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003076750250752257,
+ "loss": 1.3187,
+ "theoretical_loss": 3.4526421753651855,
+ "tokens_seen": 2044067840
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003076589769307924,
+ "loss": 1.2508,
+ "theoretical_loss": 3.4526330611398297,
+ "tokens_seen": 2044133376
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003076429287863591,
+ "loss": 1.2905,
+ "theoretical_loss": 3.452623947288492,
+ "tokens_seen": 2044198912
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003076268806419258,
+ "loss": 1.1559,
+ "theoretical_loss": 3.4526148338111433,
+ "tokens_seen": 2044264448
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003076108324974925,
+ "loss": 1.2295,
+ "theoretical_loss": 3.4526057207077576,
+ "tokens_seen": 2044329984
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003075947843530592,
+ "loss": 1.2918,
+ "theoretical_loss": 3.4525966079783075,
+ "tokens_seen": 2044395520
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003075787362086259,
+ "loss": 1.2014,
+ "theoretical_loss": 3.4525874956227645,
+ "tokens_seen": 2044461056
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003075626880641926,
+ "loss": 1.2849,
+ "theoretical_loss": 3.452578383641102,
+ "tokens_seen": 2044526592
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003075466399197593,
+ "loss": 1.3276,
+ "theoretical_loss": 3.4525692720332932,
+ "tokens_seen": 2044592128
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000307530591775326,
+ "loss": 1.2813,
+ "theoretical_loss": 3.45256016079931,
+ "tokens_seen": 2044657664
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 684631,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2299048900604248,
+ "objective/train/theoretical_loss": 3.4525510499391245,
+ "objective/train/tokens_used": 2065183200,
+ "theoretical_loss": 3.4525510499391245,
+ "tokens_seen": 2044723200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003075145436308927,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4525510499391245,
+ "tokens_seen": 2044723200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074984954864594,
+ "loss": 1.3623,
+ "theoretical_loss": 3.4525419394527113,
+ "tokens_seen": 2044788736
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074824473420261,
+ "loss": 1.2515,
+ "theoretical_loss": 3.4525328293400417,
+ "tokens_seen": 2044854272
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074663991975928,
+ "loss": 1.1655,
+ "theoretical_loss": 3.4525237196010883,
+ "tokens_seen": 2044919808
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074503510531595,
+ "loss": 1.3252,
+ "theoretical_loss": 3.4525146102358244,
+ "tokens_seen": 2044985344
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074343029087262,
+ "loss": 1.3235,
+ "theoretical_loss": 3.452505501244223,
+ "tokens_seen": 2045050880
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074182547642929,
+ "loss": 1.292,
+ "theoretical_loss": 3.452496392626256,
+ "tokens_seen": 2045116416
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003074022066198596,
+ "loss": 1.2388,
+ "theoretical_loss": 3.452487284381896,
+ "tokens_seen": 2045181952
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003073861584754263,
+ "loss": 1.2679,
+ "theoretical_loss": 3.452478176511116,
+ "tokens_seen": 2045247488
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000307370110330993,
+ "loss": 1.2823,
+ "theoretical_loss": 3.4524690690138895,
+ "tokens_seen": 2045313024
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003073540621865597,
+ "loss": 1.2521,
+ "theoretical_loss": 3.452459961890188,
+ "tokens_seen": 2045378560
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003073380140421264,
+ "loss": 1.2804,
+ "theoretical_loss": 3.452450855139985,
+ "tokens_seen": 2045444096
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003073219658976931,
+ "loss": 1.253,
+ "theoretical_loss": 3.4524417487632526,
+ "tokens_seen": 2045509632
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003073059177532598,
+ "loss": 1.3117,
+ "theoretical_loss": 3.4524326427599643,
+ "tokens_seen": 2045575168
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003072898696088265,
+ "loss": 1.2996,
+ "theoretical_loss": 3.452423537130092,
+ "tokens_seen": 2045640704
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003072738214643932,
+ "loss": 1.2578,
+ "theoretical_loss": 3.452414431873609,
+ "tokens_seen": 2045706240
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003072577733199599,
+ "loss": 1.1737,
+ "theoretical_loss": 3.4524053269904873,
+ "tokens_seen": 2045771776
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003072417251755266,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4523962224807008,
+ "tokens_seen": 2045837312
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003072256770310933,
+ "loss": 1.2415,
+ "theoretical_loss": 3.4523871183442214,
+ "tokens_seen": 2045902848
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030720962888666,
+ "loss": 1.3258,
+ "theoretical_loss": 3.4523780145810217,
+ "tokens_seen": 2045968384
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071935807422267,
+ "loss": 1.2345,
+ "theoretical_loss": 3.4523689111910754,
+ "tokens_seen": 2046033920
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071775325977934,
+ "loss": 1.237,
+ "theoretical_loss": 3.452359808174354,
+ "tokens_seen": 2046099456
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071614844533601,
+ "loss": 1.2516,
+ "theoretical_loss": 3.452350705530831,
+ "tokens_seen": 2046164992
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071454363089268,
+ "loss": 1.2471,
+ "theoretical_loss": 3.452341603260479,
+ "tokens_seen": 2046230528
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071293881644935,
+ "loss": 1.3384,
+ "theoretical_loss": 3.452332501363271,
+ "tokens_seen": 2046296064
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003071133400200602,
+ "loss": 1.3427,
+ "theoretical_loss": 3.4523233998391794,
+ "tokens_seen": 2046361600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003070972918756269,
+ "loss": 1.2222,
+ "theoretical_loss": 3.4523142986881767,
+ "tokens_seen": 2046427136
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003070812437311936,
+ "loss": 1.3624,
+ "theoretical_loss": 3.4523051979102366,
+ "tokens_seen": 2046492672
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003070651955867603,
+ "loss": 1.2361,
+ "theoretical_loss": 3.452296097505331,
+ "tokens_seen": 2046558208
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000307049147442327,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4522869974734327,
+ "tokens_seen": 2046623744
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003070330992978937,
+ "loss": 1.2026,
+ "theoretical_loss": 3.4522778978145148,
+ "tokens_seen": 2046689280
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030701705115346035,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4522687985285505,
+ "tokens_seen": 2046754816
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030700100300902705,
+ "loss": 1.2662,
+ "theoretical_loss": 3.4522596996155115,
+ "tokens_seen": 2046820352
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030698495486459375,
+ "loss": 1.3028,
+ "theoretical_loss": 3.4522506010753715,
+ "tokens_seen": 2046885888
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030696890672016045,
+ "loss": 1.3296,
+ "theoretical_loss": 3.4522415029081026,
+ "tokens_seen": 2046951424
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030695285857572715,
+ "loss": 1.3201,
+ "theoretical_loss": 3.452232405113678,
+ "tokens_seen": 2047016960
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030693681043129385,
+ "loss": 1.2371,
+ "theoretical_loss": 3.45222330769207,
+ "tokens_seen": 2047082496
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030692076228686056,
+ "loss": 1.319,
+ "theoretical_loss": 3.4522142106432527,
+ "tokens_seen": 2047148032
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003069047141424273,
+ "loss": 1.2867,
+ "theoretical_loss": 3.452205113967197,
+ "tokens_seen": 2047213568
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000306888665997994,
+ "loss": 1.2849,
+ "theoretical_loss": 3.4521960176638773,
+ "tokens_seen": 2047279104
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003068726178535607,
+ "loss": 1.3054,
+ "theoretical_loss": 3.4521869217332655,
+ "tokens_seen": 2047344640
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003068565697091274,
+ "loss": 1.3061,
+ "theoretical_loss": 3.4521778261753346,
+ "tokens_seen": 2047410176
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003068405215646941,
+ "loss": 1.3928,
+ "theoretical_loss": 3.4521687309900577,
+ "tokens_seen": 2047475712
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003068244734202608,
+ "loss": 1.3086,
+ "theoretical_loss": 3.4521596361774067,
+ "tokens_seen": 2047541248
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003068084252758275,
+ "loss": 1.3244,
+ "theoretical_loss": 3.452150541737356,
+ "tokens_seen": 2047606784
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003067923771313942,
+ "loss": 1.2439,
+ "theoretical_loss": 3.4521414476698764,
+ "tokens_seen": 2047672320
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003067763289869609,
+ "loss": 1.3025,
+ "theoretical_loss": 3.452132353974943,
+ "tokens_seen": 2047737856
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003067602808425276,
+ "loss": 1.2693,
+ "theoretical_loss": 3.4521232606525265,
+ "tokens_seen": 2047803392
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003067442326980943,
+ "loss": 1.3337,
+ "theoretical_loss": 3.452114167702601,
+ "tokens_seen": 2047868928
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000306728184553661,
+ "loss": 1.2676,
+ "theoretical_loss": 3.4521050751251385,
+ "tokens_seen": 2047934464
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 685800,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1857306957244873,
+ "objective/train/theoretical_loss": 3.4520959829201128,
+ "objective/train/tokens_used": 2068460000,
+ "theoretical_loss": 3.4520959829201128,
+ "tokens_seen": 2048000000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003067121364092277,
+ "loss": 1.2659,
+ "theoretical_loss": 3.4520959829201128,
+ "tokens_seen": 2048000000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066960882647944,
+ "loss": 1.3608,
+ "theoretical_loss": 3.452086891087496,
+ "tokens_seen": 2048065536
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066800401203611,
+ "loss": 1.2833,
+ "theoretical_loss": 3.452077799627261,
+ "tokens_seen": 2048131072
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066639919759278,
+ "loss": 1.2188,
+ "theoretical_loss": 3.4520687085393806,
+ "tokens_seen": 2048196608
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066479438314945,
+ "loss": 1.2748,
+ "theoretical_loss": 3.452059617823828,
+ "tokens_seen": 2048262144
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066318956870612,
+ "loss": 1.3237,
+ "theoretical_loss": 3.4520505274805764,
+ "tokens_seen": 2048327680
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003066158475426279,
+ "loss": 1.2631,
+ "theoretical_loss": 3.452041437509597,
+ "tokens_seen": 2048393216
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065997993981946,
+ "loss": 1.1814,
+ "theoretical_loss": 3.4520323479108646,
+ "tokens_seen": 2048458752
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065837512537613,
+ "loss": 1.3237,
+ "theoretical_loss": 3.452023258684351,
+ "tokens_seen": 2048524288
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000306567703109328,
+ "loss": 1.3989,
+ "theoretical_loss": 3.4520141698300284,
+ "tokens_seen": 2048589824
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065516549648947,
+ "loss": 1.2769,
+ "theoretical_loss": 3.4520050813478713,
+ "tokens_seen": 2048655360
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065356068204614,
+ "loss": 1.2536,
+ "theoretical_loss": 3.4519959932378512,
+ "tokens_seen": 2048720896
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065195586760281,
+ "loss": 1.2378,
+ "theoretical_loss": 3.4519869054999424,
+ "tokens_seen": 2048786432
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003065035105315948,
+ "loss": 1.2761,
+ "theoretical_loss": 3.451977818134116,
+ "tokens_seen": 2048851968
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003064874623871615,
+ "loss": 1.2268,
+ "theoretical_loss": 3.451968731140346,
+ "tokens_seen": 2048917504
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030647141424272823,
+ "loss": 1.3496,
+ "theoretical_loss": 3.4519596445186047,
+ "tokens_seen": 2048983040
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030645536609829493,
+ "loss": 1.3143,
+ "theoretical_loss": 3.451950558268866,
+ "tokens_seen": 2049048576
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030643931795386163,
+ "loss": 1.2292,
+ "theoretical_loss": 3.451941472391101,
+ "tokens_seen": 2049114112
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030642326980942833,
+ "loss": 1.356,
+ "theoretical_loss": 3.451932386885284,
+ "tokens_seen": 2049179648
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030640722166499503,
+ "loss": 1.1521,
+ "theoretical_loss": 3.4519233017513877,
+ "tokens_seen": 2049245184
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063911735205617,
+ "loss": 1.2458,
+ "theoretical_loss": 3.4519142169893846,
+ "tokens_seen": 2049310720
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063751253761284,
+ "loss": 1.2745,
+ "theoretical_loss": 3.451905132599248,
+ "tokens_seen": 2049376256
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063590772316951,
+ "loss": 1.2585,
+ "theoretical_loss": 3.45189604858095,
+ "tokens_seen": 2049441792
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063430290872618,
+ "loss": 1.3821,
+ "theoretical_loss": 3.4518869649344643,
+ "tokens_seen": 2049507328
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063269809428285,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4518778816597635,
+ "tokens_seen": 2049572864
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003063109327983952,
+ "loss": 1.3102,
+ "theoretical_loss": 3.451868798756821,
+ "tokens_seen": 2049638400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003062948846539619,
+ "loss": 1.2913,
+ "theoretical_loss": 3.4518597162256084,
+ "tokens_seen": 2049703936
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003062788365095286,
+ "loss": 1.284,
+ "theoretical_loss": 3.4518506340660995,
+ "tokens_seen": 2049769472
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003062627883650953,
+ "loss": 1.2228,
+ "theoretical_loss": 3.4518415522782675,
+ "tokens_seen": 2049835008
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000306246740220662,
+ "loss": 1.3518,
+ "theoretical_loss": 3.451832470862085,
+ "tokens_seen": 2049900544
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003062306920762287,
+ "loss": 1.3234,
+ "theoretical_loss": 3.4518233898175246,
+ "tokens_seen": 2049966080
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003062146439317954,
+ "loss": 1.1897,
+ "theoretical_loss": 3.4518143091445594,
+ "tokens_seen": 2050031616
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061985957873621,
+ "loss": 1.3907,
+ "theoretical_loss": 3.4518052288431624,
+ "tokens_seen": 2050097152
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061825476429288,
+ "loss": 1.2745,
+ "theoretical_loss": 3.4517961489133064,
+ "tokens_seen": 2050162688
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061664994984955,
+ "loss": 1.3842,
+ "theoretical_loss": 3.451787069354965,
+ "tokens_seen": 2050228224
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061504513540622,
+ "loss": 1.2386,
+ "theoretical_loss": 3.4517779901681096,
+ "tokens_seen": 2050293760
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061344032096289,
+ "loss": 1.2721,
+ "theoretical_loss": 3.4517689113527146,
+ "tokens_seen": 2050359296
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061183550651956,
+ "loss": 1.3492,
+ "theoretical_loss": 3.4517598329087518,
+ "tokens_seen": 2050424832
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003061023069207623,
+ "loss": 1.1884,
+ "theoretical_loss": 3.4517507548361954,
+ "tokens_seen": 2050490368
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000306086258776329,
+ "loss": 1.3203,
+ "theoretical_loss": 3.4517416771350176,
+ "tokens_seen": 2050555904
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003060702106318957,
+ "loss": 1.2926,
+ "theoretical_loss": 3.451732599805191,
+ "tokens_seen": 2050621440
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003060541624874624,
+ "loss": 1.3281,
+ "theoretical_loss": 3.451723522846689,
+ "tokens_seen": 2050686976
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003060381143430291,
+ "loss": 1.1767,
+ "theoretical_loss": 3.4517144462594844,
+ "tokens_seen": 2050752512
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003060220661985958,
+ "loss": 1.1938,
+ "theoretical_loss": 3.451705370043551,
+ "tokens_seen": 2050818048
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003060060180541625,
+ "loss": 1.2403,
+ "theoretical_loss": 3.45169629419886,
+ "tokens_seen": 2050883584
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003059899699097292,
+ "loss": 1.364,
+ "theoretical_loss": 3.451687218725386,
+ "tokens_seen": 2050949120
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003059739217652959,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4516781436231008,
+ "tokens_seen": 2051014656
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003059578736208626,
+ "loss": 1.2003,
+ "theoretical_loss": 3.451669068891978,
+ "tokens_seen": 2051080192
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003059418254764293,
+ "loss": 1.3335,
+ "theoretical_loss": 3.4516599945319903,
+ "tokens_seen": 2051145728
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000305925777331996,
+ "loss": 1.2301,
+ "theoretical_loss": 3.451650920543111,
+ "tokens_seen": 2051211264
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 686726,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3169500827789307,
+ "objective/train/theoretical_loss": 3.451641846925313,
+ "objective/train/tokens_used": 2071736800,
+ "theoretical_loss": 3.451641846925313,
+ "tokens_seen": 2051276800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003059097291875627,
+ "loss": 1.2802,
+ "theoretical_loss": 3.451641846925313,
+ "tokens_seen": 2051276800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058936810431294,
+ "loss": 1.3173,
+ "theoretical_loss": 3.4516327736785684,
+ "tokens_seen": 2051342336
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058776328986961,
+ "loss": 1.2207,
+ "theoretical_loss": 3.451623700802852,
+ "tokens_seen": 2051407872
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058615847542628,
+ "loss": 1.3639,
+ "theoretical_loss": 3.4516146282981346,
+ "tokens_seen": 2051473408
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058455366098295,
+ "loss": 1.3164,
+ "theoretical_loss": 3.451605556164391,
+ "tokens_seen": 2051538944
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058294884653962,
+ "loss": 1.4238,
+ "theoretical_loss": 3.451596484401592,
+ "tokens_seen": 2051604480
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003058134403209629,
+ "loss": 1.1302,
+ "theoretical_loss": 3.4515874130097135,
+ "tokens_seen": 2051670016
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057973921765296,
+ "loss": 1.2408,
+ "theoretical_loss": 3.4515783419887263,
+ "tokens_seen": 2051735552
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057813440320963,
+ "loss": 1.2066,
+ "theoretical_loss": 3.451569271338604,
+ "tokens_seen": 2051801088
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000305765295887663,
+ "loss": 1.3544,
+ "theoretical_loss": 3.45156020105932,
+ "tokens_seen": 2051866624
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057492477432297,
+ "loss": 1.3271,
+ "theoretical_loss": 3.451551131150847,
+ "tokens_seen": 2051932160
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057331995987964,
+ "loss": 1.3013,
+ "theoretical_loss": 3.4515420616131576,
+ "tokens_seen": 2051997696
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057171514543631,
+ "loss": 1.4297,
+ "theoretical_loss": 3.4515329924462255,
+ "tokens_seen": 2052063232
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003057011033099298,
+ "loss": 1.292,
+ "theoretical_loss": 3.451523923650023,
+ "tokens_seen": 2052128768
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003056850551654965,
+ "loss": 1.274,
+ "theoretical_loss": 3.4515148552245236,
+ "tokens_seen": 2052194304
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003056690070210632,
+ "loss": 1.2692,
+ "theoretical_loss": 3.4515057871697,
+ "tokens_seen": 2052259840
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003056529588766299,
+ "loss": 1.2282,
+ "theoretical_loss": 3.451496719485526,
+ "tokens_seen": 2052325376
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003056369107321966,
+ "loss": 1.2513,
+ "theoretical_loss": 3.4514876521719735,
+ "tokens_seen": 2052390912
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003056208625877633,
+ "loss": 1.1837,
+ "theoretical_loss": 3.451478585229016,
+ "tokens_seen": 2052456448
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030560481444333,
+ "loss": 1.2053,
+ "theoretical_loss": 3.451469518656627,
+ "tokens_seen": 2052521984
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055887662988967,
+ "loss": 1.307,
+ "theoretical_loss": 3.4514604524547785,
+ "tokens_seen": 2052587520
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055727181544634,
+ "loss": 1.3516,
+ "theoretical_loss": 3.4514513866234444,
+ "tokens_seen": 2052653056
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055566700100301,
+ "loss": 1.2605,
+ "theoretical_loss": 3.451442321162597,
+ "tokens_seen": 2052718592
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055406218655968,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4514332560722103,
+ "tokens_seen": 2052784128
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055245737211635,
+ "loss": 1.3299,
+ "theoretical_loss": 3.451424191352257,
+ "tokens_seen": 2052849664
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003055085255767302,
+ "loss": 1.1915,
+ "theoretical_loss": 3.451415127002709,
+ "tokens_seen": 2052915200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003054924774322969,
+ "loss": 1.2641,
+ "theoretical_loss": 3.4514060630235406,
+ "tokens_seen": 2052980736
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003054764292878636,
+ "loss": 1.1938,
+ "theoretical_loss": 3.451396999414725,
+ "tokens_seen": 2053046272
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003054603811434303,
+ "loss": 1.2671,
+ "theoretical_loss": 3.4513879361762343,
+ "tokens_seen": 2053111808
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000305444332998997,
+ "loss": 1.182,
+ "theoretical_loss": 3.451378873308042,
+ "tokens_seen": 2053177344
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003054282848545637,
+ "loss": 1.3227,
+ "theoretical_loss": 3.4513698108101214,
+ "tokens_seen": 2053242880
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003054122367101304,
+ "loss": 1.3401,
+ "theoretical_loss": 3.451360748682445,
+ "tokens_seen": 2053308416
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053961885656971,
+ "loss": 1.2524,
+ "theoretical_loss": 3.451351686924986,
+ "tokens_seen": 2053373952
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053801404212638,
+ "loss": 1.3038,
+ "theoretical_loss": 3.4513426255377184,
+ "tokens_seen": 2053439488
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053640922768305,
+ "loss": 1.1896,
+ "theoretical_loss": 3.4513335645206142,
+ "tokens_seen": 2053505024
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053480441323972,
+ "loss": 1.3894,
+ "theoretical_loss": 3.4513245038736464,
+ "tokens_seen": 2053570560
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053319959879639,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4513154435967888,
+ "tokens_seen": 2053636096
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003053159478435306,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4513063836900137,
+ "tokens_seen": 2053701632
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052998996990973,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4512973241532947,
+ "tokens_seen": 2053767168
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000305283851554664,
+ "loss": 1.2792,
+ "theoretical_loss": 3.451288264986605,
+ "tokens_seen": 2053832704
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052678034102307,
+ "loss": 1.2409,
+ "theoretical_loss": 3.451279206189917,
+ "tokens_seen": 2053898240
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052517552657974,
+ "loss": 1.349,
+ "theoretical_loss": 3.4512701477632044,
+ "tokens_seen": 2053963776
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052357071213641,
+ "loss": 1.3268,
+ "theoretical_loss": 3.45126108970644,
+ "tokens_seen": 2054029312
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052196589769308,
+ "loss": 1.412,
+ "theoretical_loss": 3.4512520320195974,
+ "tokens_seen": 2054094848
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003052036108324975,
+ "loss": 1.2705,
+ "theoretical_loss": 3.451242974702649,
+ "tokens_seen": 2054160384
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003051875626880642,
+ "loss": 1.1753,
+ "theoretical_loss": 3.451233917755568,
+ "tokens_seen": 2054225920
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003051715145436309,
+ "loss": 1.2702,
+ "theoretical_loss": 3.451224861178328,
+ "tokens_seen": 2054291456
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003051554663991976,
+ "loss": 1.4642,
+ "theoretical_loss": 3.451215804970902,
+ "tokens_seen": 2054356992
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003051394182547643,
+ "loss": 1.2314,
+ "theoretical_loss": 3.451206749133262,
+ "tokens_seen": 2054422528
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000305123370110331,
+ "loss": 1.2388,
+ "theoretical_loss": 3.4511976936653825,
+ "tokens_seen": 2054488064
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 687773,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3332979679107666,
+ "objective/train/theoretical_loss": 3.4511886385672357,
+ "objective/train/tokens_used": 2075013600,
+ "theoretical_loss": 3.4511886385672357,
+ "tokens_seen": 2054553600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030510732196589773,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4511886385672357,
+ "tokens_seen": 2054553600
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030509127382146443,
+ "loss": 1.2771,
+ "theoretical_loss": 3.4511795838387958,
+ "tokens_seen": 2054619136
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030507522567703113,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4511705294800343,
+ "tokens_seen": 2054684672
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030505917753259783,
+ "loss": 1.3907,
+ "theoretical_loss": 3.4511614754909257,
+ "tokens_seen": 2054750208
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030504312938816453,
+ "loss": 1.4565,
+ "theoretical_loss": 3.451152421871443,
+ "tokens_seen": 2054815744
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030502708124373123,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4511433686215582,
+ "tokens_seen": 2054881280
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030501103309929793,
+ "loss": 1.2495,
+ "theoretical_loss": 3.451134315741246,
+ "tokens_seen": 2054946816
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003049949849548646,
+ "loss": 1.2772,
+ "theoretical_loss": 3.451125263230478,
+ "tokens_seen": 2055012352
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003049789368104313,
+ "loss": 1.3352,
+ "theoretical_loss": 3.4511162110892286,
+ "tokens_seen": 2055077888
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304962888665998,
+ "loss": 1.3278,
+ "theoretical_loss": 3.45110715931747,
+ "tokens_seen": 2055143424
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003049468405215647,
+ "loss": 1.3033,
+ "theoretical_loss": 3.451098107915176,
+ "tokens_seen": 2055208960
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003049307923771314,
+ "loss": 1.3228,
+ "theoretical_loss": 3.451089056882319,
+ "tokens_seen": 2055274496
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003049147442326981,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4510800062188727,
+ "tokens_seen": 2055340032
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048986960882648,
+ "loss": 1.2146,
+ "theoretical_loss": 3.451070955924811,
+ "tokens_seen": 2055405568
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048826479438315,
+ "loss": 1.3313,
+ "theoretical_loss": 3.4510619060001053,
+ "tokens_seen": 2055471104
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048665997993982,
+ "loss": 1.2213,
+ "theoretical_loss": 3.4510528564447296,
+ "tokens_seen": 2055536640
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048505516549649,
+ "loss": 1.1385,
+ "theoretical_loss": 3.4510438072586576,
+ "tokens_seen": 2055602176
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048345035105316,
+ "loss": 1.2984,
+ "theoretical_loss": 3.4510347584418612,
+ "tokens_seen": 2055667712
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003048184553660983,
+ "loss": 1.2993,
+ "theoretical_loss": 3.451025709994315,
+ "tokens_seen": 2055733248
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304802407221665,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4510166619159914,
+ "tokens_seen": 2055798784
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047863590772317,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4510076142068633,
+ "tokens_seen": 2055864320
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047703109327984,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4509985668669043,
+ "tokens_seen": 2055929856
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047542627883651,
+ "loss": 1.2306,
+ "theoretical_loss": 3.450989519896088,
+ "tokens_seen": 2055995392
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047382146439318,
+ "loss": 1.2293,
+ "theoretical_loss": 3.4509804732943863,
+ "tokens_seen": 2056060928
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047221664994985,
+ "loss": 1.3185,
+ "theoretical_loss": 3.4509714270617735,
+ "tokens_seen": 2056126464
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003047061183550652,
+ "loss": 1.3,
+ "theoretical_loss": 3.450962381198222,
+ "tokens_seen": 2056192000
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003046900702106319,
+ "loss": 1.2521,
+ "theoretical_loss": 3.450953335703706,
+ "tokens_seen": 2056257536
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003046740220661986,
+ "loss": 1.2354,
+ "theoretical_loss": 3.4509442905781973,
+ "tokens_seen": 2056323072
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003046579739217653,
+ "loss": 1.2061,
+ "theoretical_loss": 3.4509352458216704,
+ "tokens_seen": 2056388608
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304641925777332,
+ "loss": 1.3107,
+ "theoretical_loss": 3.450926201434098,
+ "tokens_seen": 2056454144
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003046258776328987,
+ "loss": 1.248,
+ "theoretical_loss": 3.4509171574154527,
+ "tokens_seen": 2056519680
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003046098294884654,
+ "loss": 1.2084,
+ "theoretical_loss": 3.4509081137657085,
+ "tokens_seen": 2056585216
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045937813440321,
+ "loss": 1.2186,
+ "theoretical_loss": 3.450899070484838,
+ "tokens_seen": 2056650752
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045777331995988,
+ "loss": 1.3615,
+ "theoretical_loss": 3.4508900275728154,
+ "tokens_seen": 2056716288
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045616850551655,
+ "loss": 1.29,
+ "theoretical_loss": 3.450880985029613,
+ "tokens_seen": 2056781824
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045456369107322,
+ "loss": 1.1823,
+ "theoretical_loss": 3.450871942855204,
+ "tokens_seen": 2056847360
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045295887662989,
+ "loss": 1.2327,
+ "theoretical_loss": 3.450862901049562,
+ "tokens_seen": 2056912896
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003045135406218656,
+ "loss": 1.2362,
+ "theoretical_loss": 3.45085385961266,
+ "tokens_seen": 2056978432
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030449749247743235,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4508448185444713,
+ "tokens_seen": 2057043968
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030448144433299905,
+ "loss": 1.2555,
+ "theoretical_loss": 3.4508357778449694,
+ "tokens_seen": 2057109504
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030446539618856575,
+ "loss": 1.2788,
+ "theoretical_loss": 3.4508267375141264,
+ "tokens_seen": 2057175040
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030444934804413245,
+ "loss": 1.2358,
+ "theoretical_loss": 3.450817697551917,
+ "tokens_seen": 2057240576
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030443329989969915,
+ "loss": 1.2054,
+ "theoretical_loss": 3.4508086579583135,
+ "tokens_seen": 2057306112
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030441725175526585,
+ "loss": 1.2856,
+ "theoretical_loss": 3.4507996187332894,
+ "tokens_seen": 2057371648
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030440120361083256,
+ "loss": 1.2238,
+ "theoretical_loss": 3.450790579876818,
+ "tokens_seen": 2057437184
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030438515546639926,
+ "loss": 1.2619,
+ "theoretical_loss": 3.4507815413888725,
+ "tokens_seen": 2057502720
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030436910732196596,
+ "loss": 1.2797,
+ "theoretical_loss": 3.450772503269426,
+ "tokens_seen": 2057568256
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003043530591775326,
+ "loss": 1.3414,
+ "theoretical_loss": 3.4507634655184516,
+ "tokens_seen": 2057633792
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003043370110330993,
+ "loss": 1.2458,
+ "theoretical_loss": 3.450754428135923,
+ "tokens_seen": 2057699328
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304320962888666,
+ "loss": 1.3248,
+ "theoretical_loss": 3.450745391121813,
+ "tokens_seen": 2057764864
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 688746,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3244448900222778,
+ "objective/train/theoretical_loss": 3.4507363544760956,
+ "objective/train/tokens_used": 2078290400,
+ "theoretical_loss": 3.4507363544760956,
+ "tokens_seen": 2057830400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003043049147442327,
+ "loss": 1.2471,
+ "theoretical_loss": 3.4507363544760956,
+ "tokens_seen": 2057830400
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042888665997994,
+ "loss": 1.2969,
+ "theoretical_loss": 3.4507273181987426,
+ "tokens_seen": 2057895936
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042728184553661,
+ "loss": 1.2506,
+ "theoretical_loss": 3.4507182822897287,
+ "tokens_seen": 2057961472
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042567703109328,
+ "loss": 1.2169,
+ "theoretical_loss": 3.4507092467490272,
+ "tokens_seen": 2058027008
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042407221664995,
+ "loss": 1.3024,
+ "theoretical_loss": 3.45070021157661,
+ "tokens_seen": 2058092544
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042246740220662,
+ "loss": 1.1788,
+ "theoretical_loss": 3.4506911767724513,
+ "tokens_seen": 2058158080
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003042086258776329,
+ "loss": 1.1999,
+ "theoretical_loss": 3.4506821423365244,
+ "tokens_seen": 2058223616
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003041925777331996,
+ "loss": 1.2391,
+ "theoretical_loss": 3.450673108268802,
+ "tokens_seen": 2058289152
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003041765295887663,
+ "loss": 1.3035,
+ "theoretical_loss": 3.450664074569258,
+ "tokens_seen": 2058354688
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304160481444333,
+ "loss": 1.2367,
+ "theoretical_loss": 3.4506550412378654,
+ "tokens_seen": 2058420224
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003041444332998997,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4506460082745978,
+ "tokens_seen": 2058485760
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003041283851554664,
+ "loss": 1.3235,
+ "theoretical_loss": 3.4506369756794277,
+ "tokens_seen": 2058551296
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003041123370110331,
+ "loss": 1.2073,
+ "theoretical_loss": 3.450627943452329,
+ "tokens_seen": 2058616832
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040962888665998,
+ "loss": 1.2638,
+ "theoretical_loss": 3.450618911593275,
+ "tokens_seen": 2058682368
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040802407221665,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4506098801022387,
+ "tokens_seen": 2058747904
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040641925777332,
+ "loss": 1.2289,
+ "theoretical_loss": 3.4506008489791933,
+ "tokens_seen": 2058813440
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040481444332999,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4505918182241127,
+ "tokens_seen": 2058878976
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040320962888666,
+ "loss": 1.3613,
+ "theoretical_loss": 3.4505827878369697,
+ "tokens_seen": 2058944512
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003040160481444333,
+ "loss": 1.2264,
+ "theoretical_loss": 3.450573757817738,
+ "tokens_seen": 2059010048
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000304,
+ "loss": 1.2085,
+ "theoretical_loss": 3.45056472816639,
+ "tokens_seen": 2059075584
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039839518555667,
+ "loss": 1.2419,
+ "theoretical_loss": 3.4505556988828996,
+ "tokens_seen": 2059141120
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039679037111334,
+ "loss": 1.2468,
+ "theoretical_loss": 3.4505466699672405,
+ "tokens_seen": 2059206656
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039518555667001,
+ "loss": 1.3193,
+ "theoretical_loss": 3.4505376414193853,
+ "tokens_seen": 2059272192
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039358074222668,
+ "loss": 1.4149,
+ "theoretical_loss": 3.450528613239308,
+ "tokens_seen": 2059337728
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039197592778335,
+ "loss": 1.2797,
+ "theoretical_loss": 3.4505195854269815,
+ "tokens_seen": 2059403264
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003039037111334002,
+ "loss": 1.2463,
+ "theoretical_loss": 3.450510557982379,
+ "tokens_seen": 2059468800
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003038876629889669,
+ "loss": 1.2455,
+ "theoretical_loss": 3.4505015309054743,
+ "tokens_seen": 2059534336
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003038716148445336,
+ "loss": 1.3298,
+ "theoretical_loss": 3.45049250419624,
+ "tokens_seen": 2059599872
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003038555667001003,
+ "loss": 1.4246,
+ "theoretical_loss": 3.4504834778546503,
+ "tokens_seen": 2059665408
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000303839518555667,
+ "loss": 1.236,
+ "theoretical_loss": 3.4504744518806776,
+ "tokens_seen": 2059730944
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003038234704112337,
+ "loss": 1.2963,
+ "theoretical_loss": 3.4504654262742958,
+ "tokens_seen": 2059796480
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003038074222668004,
+ "loss": 1.3202,
+ "theoretical_loss": 3.4504564010354786,
+ "tokens_seen": 2059862016
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003037913741223671,
+ "loss": 1.2154,
+ "theoretical_loss": 3.450447376164198,
+ "tokens_seen": 2059927552
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003037753259779338,
+ "loss": 1.2535,
+ "theoretical_loss": 3.450438351660429,
+ "tokens_seen": 2059993088
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003037592778335005,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4504293275241436,
+ "tokens_seen": 2060058624
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003037432296890672,
+ "loss": 1.2386,
+ "theoretical_loss": 3.4504203037553163,
+ "tokens_seen": 2060124160
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030372718154463393,
+ "loss": 1.4104,
+ "theoretical_loss": 3.4504112803539195,
+ "tokens_seen": 2060189696
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030371113340020063,
+ "loss": 1.349,
+ "theoretical_loss": 3.450402257319927,
+ "tokens_seen": 2060255232
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030369508525576733,
+ "loss": 1.2928,
+ "theoretical_loss": 3.450393234653312,
+ "tokens_seen": 2060320768
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030367903711133403,
+ "loss": 1.3125,
+ "theoretical_loss": 3.450384212354048,
+ "tokens_seen": 2060386304
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030366298896690073,
+ "loss": 1.3297,
+ "theoretical_loss": 3.4503751904221085,
+ "tokens_seen": 2060451840
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030364694082246743,
+ "loss": 1.3422,
+ "theoretical_loss": 3.450366168857466,
+ "tokens_seen": 2060517376
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030363089267803413,
+ "loss": 1.3286,
+ "theoretical_loss": 3.450357147660095,
+ "tokens_seen": 2060582912
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030361484453360083,
+ "loss": 1.2436,
+ "theoretical_loss": 3.4503481268299683,
+ "tokens_seen": 2060648448
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030359879638916753,
+ "loss": 1.3507,
+ "theoretical_loss": 3.4503391063670596,
+ "tokens_seen": 2060713984
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003035827482447342,
+ "loss": 1.2979,
+ "theoretical_loss": 3.4503300862713413,
+ "tokens_seen": 2060779520
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003035667001003009,
+ "loss": 1.2859,
+ "theoretical_loss": 3.450321066542788,
+ "tokens_seen": 2060845056
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003035506519558676,
+ "loss": 1.2971,
+ "theoretical_loss": 3.450312047181373,
+ "tokens_seen": 2060910592
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003035346038114343,
+ "loss": 1.3308,
+ "theoretical_loss": 3.4503030281870686,
+ "tokens_seen": 2060976128
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000303518555667001,
+ "loss": 1.3051,
+ "theoretical_loss": 3.4502940095598484,
+ "tokens_seen": 2061041664
+ },
+ {
+ "epoch": 0.62,
+ "objective/train/docs_used": 689953,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2605319023132324,
+ "objective/train/theoretical_loss": 3.450284991299687,
+ "objective/train/tokens_used": 2081567200,
+ "theoretical_loss": 3.450284991299687,
+ "tokens_seen": 2061107200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003035025075225677,
+ "loss": 1.2442,
+ "theoretical_loss": 3.450284991299687,
+ "tokens_seen": 2061107200
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034864593781344,
+ "loss": 1.2675,
+ "theoretical_loss": 3.4502759734065567,
+ "tokens_seen": 2061172736
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034704112337011,
+ "loss": 1.3474,
+ "theoretical_loss": 3.4502669558804318,
+ "tokens_seen": 2061238272
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034543630892678,
+ "loss": 1.3098,
+ "theoretical_loss": 3.4502579387212844,
+ "tokens_seen": 2061303808
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034383149448345,
+ "loss": 1.2708,
+ "theoretical_loss": 3.450248921929089,
+ "tokens_seen": 2061369344
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034222668004012,
+ "loss": 1.2995,
+ "theoretical_loss": 3.4502399055038184,
+ "tokens_seen": 2061434880
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003034062186559679,
+ "loss": 1.3767,
+ "theoretical_loss": 3.4502308894454465,
+ "tokens_seen": 2061500416
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003033901705115346,
+ "loss": 1.1735,
+ "theoretical_loss": 3.4502218737539456,
+ "tokens_seen": 2061565952
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003033741223671013,
+ "loss": 1.3899,
+ "theoretical_loss": 3.450212858429291,
+ "tokens_seen": 2061631488
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.000303358074222668,
+ "loss": 1.2221,
+ "theoretical_loss": 3.4502038434714546,
+ "tokens_seen": 2061697024
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003033420260782347,
+ "loss": 1.3501,
+ "theoretical_loss": 3.4501948288804103,
+ "tokens_seen": 2061762560
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003033259779338014,
+ "loss": 1.2522,
+ "theoretical_loss": 3.4501858146561313,
+ "tokens_seen": 2061828096
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.0003033099297893681,
+ "loss": 1.2616,
+ "theoretical_loss": 3.450176800798591,
+ "tokens_seen": 2061893632
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030329388164493484,
+ "loss": 1.2815,
+ "theoretical_loss": 3.4501677873077634,
+ "tokens_seen": 2061959168
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030327783350050155,
+ "loss": 1.121,
+ "theoretical_loss": 3.4501587741836213,
+ "tokens_seen": 2062024704
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030326178535606825,
+ "loss": 1.3171,
+ "theoretical_loss": 3.4501497614261387,
+ "tokens_seen": 2062090240
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030324573721163495,
+ "loss": 1.2913,
+ "theoretical_loss": 3.4501407490352887,
+ "tokens_seen": 2062155776
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030322968906720165,
+ "loss": 1.2652,
+ "theoretical_loss": 3.4501317370110445,
+ "tokens_seen": 2062221312
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030321364092276835,
+ "loss": 1.1489,
+ "theoretical_loss": 3.45012272535338,
+ "tokens_seen": 2062286848
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030319759277833505,
+ "loss": 1.2835,
+ "theoretical_loss": 3.450113714062268,
+ "tokens_seen": 2062352384
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030318154463390175,
+ "loss": 1.2961,
+ "theoretical_loss": 3.450104703137683,
+ "tokens_seen": 2062417920
+ },
+ {
+ "epoch": 0.62,
+ "learning_rate": 0.00030316549648946845,
+ "loss": 1.1276,
+ "theoretical_loss": 3.4500956925795974,
+ "tokens_seen": 2062483456
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030314944834503515,
+ "loss": 1.2574,
+ "theoretical_loss": 3.450086682387985,
+ "tokens_seen": 2062548992
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030313340020060185,
+ "loss": 1.3237,
+ "theoretical_loss": 3.450077672562819,
+ "tokens_seen": 2062614528
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030311735205616855,
+ "loss": 1.2853,
+ "theoretical_loss": 3.450068663104074,
+ "tokens_seen": 2062680064
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030310130391173525,
+ "loss": 1.248,
+ "theoretical_loss": 3.4500596540117217,
+ "tokens_seen": 2062745600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030308525576730195,
+ "loss": 1.344,
+ "theoretical_loss": 3.4500506452857373,
+ "tokens_seen": 2062811136
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030306920762286865,
+ "loss": 1.3345,
+ "theoretical_loss": 3.450041636926093,
+ "tokens_seen": 2062876672
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030305315947843535,
+ "loss": 1.1895,
+ "theoretical_loss": 3.450032628932763,
+ "tokens_seen": 2062942208
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030303711133400205,
+ "loss": 1.2672,
+ "theoretical_loss": 3.45002362130572,
+ "tokens_seen": 2063007744
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030302106318956876,
+ "loss": 1.2416,
+ "theoretical_loss": 3.4500146140449384,
+ "tokens_seen": 2063073280
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030300501504513546,
+ "loss": 1.3193,
+ "theoretical_loss": 3.450005607150391,
+ "tokens_seen": 2063138816
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030298896690070216,
+ "loss": 1.3452,
+ "theoretical_loss": 3.4499966006220513,
+ "tokens_seen": 2063204352
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030297291875626886,
+ "loss": 1.1495,
+ "theoretical_loss": 3.449987594459893,
+ "tokens_seen": 2063269888
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003029568706118355,
+ "loss": 1.3296,
+ "theoretical_loss": 3.44997858866389,
+ "tokens_seen": 2063335424
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003029408224674022,
+ "loss": 1.2986,
+ "theoretical_loss": 3.449969583234015,
+ "tokens_seen": 2063400960
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003029247743229689,
+ "loss": 1.2938,
+ "theoretical_loss": 3.4499605781702423,
+ "tokens_seen": 2063466496
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003029087261785356,
+ "loss": 1.2456,
+ "theoretical_loss": 3.4499515734725446,
+ "tokens_seen": 2063532032
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003028926780341023,
+ "loss": 1.2635,
+ "theoretical_loss": 3.4499425691408954,
+ "tokens_seen": 2063597568
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000302876629889669,
+ "loss": 1.3664,
+ "theoretical_loss": 3.449933565175269,
+ "tokens_seen": 2063663104
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003028605817452357,
+ "loss": 1.2576,
+ "theoretical_loss": 3.4499245615756378,
+ "tokens_seen": 2063728640
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003028445336008024,
+ "loss": 1.3084,
+ "theoretical_loss": 3.449915558341976,
+ "tokens_seen": 2063794176
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003028284854563691,
+ "loss": 1.2893,
+ "theoretical_loss": 3.449906555474257,
+ "tokens_seen": 2063859712
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003028124373119358,
+ "loss": 1.2733,
+ "theoretical_loss": 3.449897552972455,
+ "tokens_seen": 2063925248
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027963891675025,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4498885508365422,
+ "tokens_seen": 2063990784
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027803410230692,
+ "loss": 1.1852,
+ "theoretical_loss": 3.449879549066493,
+ "tokens_seen": 2064056320
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027642928786359,
+ "loss": 1.1761,
+ "theoretical_loss": 3.4498705476622806,
+ "tokens_seen": 2064121856
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027482447342026,
+ "loss": 1.2735,
+ "theoretical_loss": 3.4498615466238784,
+ "tokens_seen": 2064187392
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027321965897693,
+ "loss": 1.3544,
+ "theoretical_loss": 3.44985254595126,
+ "tokens_seen": 2064252928
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000302716148445336,
+ "loss": 1.3144,
+ "theoretical_loss": 3.4498435456443994,
+ "tokens_seen": 2064318464
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 690841,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.329054832458496,
+ "objective/train/theoretical_loss": 3.4498345457032698,
+ "objective/train/tokens_used": 2084844000,
+ "theoretical_loss": 3.4498345457032698,
+ "tokens_seen": 2064384000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003027001003009027,
+ "loss": 1.2285,
+ "theoretical_loss": 3.4498345457032698,
+ "tokens_seen": 2064384000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026840521564694,
+ "loss": 1.267,
+ "theoretical_loss": 3.449825546127844,
+ "tokens_seen": 2064449536
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026680040120361,
+ "loss": 1.2597,
+ "theoretical_loss": 3.449816546918097,
+ "tokens_seen": 2064515072
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026519558676028,
+ "loss": 1.249,
+ "theoretical_loss": 3.449807548074001,
+ "tokens_seen": 2064580608
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026359077231695,
+ "loss": 1.1722,
+ "theoretical_loss": 3.4497985495955303,
+ "tokens_seen": 2064646144
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026198595787362,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4497895514826578,
+ "tokens_seen": 2064711680
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003026038114343029,
+ "loss": 1.3092,
+ "theoretical_loss": 3.4497805537353576,
+ "tokens_seen": 2064777216
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003025877632898696,
+ "loss": 1.2339,
+ "theoretical_loss": 3.4497715563536033,
+ "tokens_seen": 2064842752
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003025717151454363,
+ "loss": 1.2411,
+ "theoretical_loss": 3.449762559337368,
+ "tokens_seen": 2064908288
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000302555667001003,
+ "loss": 1.273,
+ "theoretical_loss": 3.449753562686626,
+ "tokens_seen": 2064973824
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003025396188565697,
+ "loss": 1.2267,
+ "theoretical_loss": 3.4497445664013497,
+ "tokens_seen": 2065039360
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003025235707121364,
+ "loss": 1.3444,
+ "theoretical_loss": 3.449735570481514,
+ "tokens_seen": 2065104896
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003025075225677031,
+ "loss": 1.271,
+ "theoretical_loss": 3.4497265749270913,
+ "tokens_seen": 2065170432
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024914744232698,
+ "loss": 1.3509,
+ "theoretical_loss": 3.4497175797380555,
+ "tokens_seen": 2065235968
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024754262788365,
+ "loss": 1.3245,
+ "theoretical_loss": 3.4497085849143807,
+ "tokens_seen": 2065301504
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024593781344032,
+ "loss": 1.1995,
+ "theoretical_loss": 3.44969959045604,
+ "tokens_seen": 2065367040
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024433299899699,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4496905963630065,
+ "tokens_seen": 2065432576
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024272818455366,
+ "loss": 1.2995,
+ "theoretical_loss": 3.449681602635255,
+ "tokens_seen": 2065498112
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003024112337011033,
+ "loss": 1.2104,
+ "theoretical_loss": 3.449672609272758,
+ "tokens_seen": 2065563648
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030239518555667,
+ "loss": 1.2581,
+ "theoretical_loss": 3.4496636162754895,
+ "tokens_seen": 2065629184
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003023791374122367,
+ "loss": 1.3995,
+ "theoretical_loss": 3.449654623643423,
+ "tokens_seen": 2065694720
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030236308926780343,
+ "loss": 1.2915,
+ "theoretical_loss": 3.449645631376532,
+ "tokens_seen": 2065760256
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030234704112337013,
+ "loss": 1.2394,
+ "theoretical_loss": 3.4496366394747904,
+ "tokens_seen": 2065825792
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030233099297893683,
+ "loss": 1.226,
+ "theoretical_loss": 3.449627647938171,
+ "tokens_seen": 2065891328
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030231494483450353,
+ "loss": 1.2297,
+ "theoretical_loss": 3.449618656766649,
+ "tokens_seen": 2065956864
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030229889669007023,
+ "loss": 1.3147,
+ "theoretical_loss": 3.4496096659601965,
+ "tokens_seen": 2066022400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030228284854563693,
+ "loss": 1.297,
+ "theoretical_loss": 3.4496006755187874,
+ "tokens_seen": 2066087936
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030226680040120363,
+ "loss": 1.2629,
+ "theoretical_loss": 3.4495916854423956,
+ "tokens_seen": 2066153472
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030225075225677033,
+ "loss": 1.2883,
+ "theoretical_loss": 3.449582695730995,
+ "tokens_seen": 2066219008
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030223470411233703,
+ "loss": 1.2547,
+ "theoretical_loss": 3.449573706384558,
+ "tokens_seen": 2066284544
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030221865596790373,
+ "loss": 1.2043,
+ "theoretical_loss": 3.4495647174030593,
+ "tokens_seen": 2066350080
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030220260782347043,
+ "loss": 1.2756,
+ "theoretical_loss": 3.449555728786472,
+ "tokens_seen": 2066415616
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021865596790371,
+ "loss": 1.2408,
+ "theoretical_loss": 3.44954674053477,
+ "tokens_seen": 2066481152
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021705115346038,
+ "loss": 1.2913,
+ "theoretical_loss": 3.4495377526479274,
+ "tokens_seen": 2066546688
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021544633901705,
+ "loss": 1.2239,
+ "theoretical_loss": 3.449528765125917,
+ "tokens_seen": 2066612224
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021384152457372,
+ "loss": 1.3645,
+ "theoretical_loss": 3.449519777968712,
+ "tokens_seen": 2066677760
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021223671013039,
+ "loss": 1.256,
+ "theoretical_loss": 3.4495107911762877,
+ "tokens_seen": 2066743296
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003021063189568706,
+ "loss": 1.1682,
+ "theoretical_loss": 3.4495018047486163,
+ "tokens_seen": 2066808832
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030209027081243734,
+ "loss": 1.2273,
+ "theoretical_loss": 3.449492818685672,
+ "tokens_seen": 2066874368
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030207422266800404,
+ "loss": 1.3652,
+ "theoretical_loss": 3.4494838329874282,
+ "tokens_seen": 2066939904
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030205817452357074,
+ "loss": 1.2044,
+ "theoretical_loss": 3.449474847653858,
+ "tokens_seen": 2067005440
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030204212637913744,
+ "loss": 1.283,
+ "theoretical_loss": 3.4494658626849364,
+ "tokens_seen": 2067070976
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030202607823470414,
+ "loss": 1.1578,
+ "theoretical_loss": 3.4494568780806363,
+ "tokens_seen": 2067136512
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030201003009027084,
+ "loss": 1.2333,
+ "theoretical_loss": 3.449447893840931,
+ "tokens_seen": 2067202048
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030199398194583754,
+ "loss": 1.2334,
+ "theoretical_loss": 3.4494389099657945,
+ "tokens_seen": 2067267584
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030197793380140424,
+ "loss": 1.2486,
+ "theoretical_loss": 3.449429926455201,
+ "tokens_seen": 2067333120
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030196188565697094,
+ "loss": 1.2371,
+ "theoretical_loss": 3.4494209433091227,
+ "tokens_seen": 2067398656
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030194583751253764,
+ "loss": 1.2584,
+ "theoretical_loss": 3.4494119605275344,
+ "tokens_seen": 2067464192
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030192978936810434,
+ "loss": 1.2016,
+ "theoretical_loss": 3.44940297811041,
+ "tokens_seen": 2067529728
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030191374122367105,
+ "loss": 1.0863,
+ "theoretical_loss": 3.4493939960577222,
+ "tokens_seen": 2067595264
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 692057,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3954368829727173,
+ "objective/train/theoretical_loss": 3.4493850143694456,
+ "objective/train/tokens_used": 2088120800,
+ "theoretical_loss": 3.4493850143694456,
+ "tokens_seen": 2067660800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030189769307923775,
+ "loss": 1.2912,
+ "theoretical_loss": 3.4493850143694456,
+ "tokens_seen": 2067660800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030188164493480445,
+ "loss": 1.3383,
+ "theoretical_loss": 3.449376033045553,
+ "tokens_seen": 2067726336
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030186559679037115,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4493670520860187,
+ "tokens_seen": 2067791872
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030184954864593785,
+ "loss": 1.3128,
+ "theoretical_loss": 3.449358071490816,
+ "tokens_seen": 2067857408
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030183350050150455,
+ "loss": 1.2128,
+ "theoretical_loss": 3.4493490912599185,
+ "tokens_seen": 2067922944
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030181745235707125,
+ "loss": 1.3361,
+ "theoretical_loss": 3.4493401113933007,
+ "tokens_seen": 2067988480
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030180140421263795,
+ "loss": 1.1504,
+ "theoretical_loss": 3.4493311318909354,
+ "tokens_seen": 2068054016
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030178535606820465,
+ "loss": 1.3057,
+ "theoretical_loss": 3.4493221527527966,
+ "tokens_seen": 2068119552
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030176930792377135,
+ "loss": 1.2004,
+ "theoretical_loss": 3.449313173978857,
+ "tokens_seen": 2068185088
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030175325977933805,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4493041955690926,
+ "tokens_seen": 2068250624
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030173721163490475,
+ "loss": 1.3868,
+ "theoretical_loss": 3.449295217523475,
+ "tokens_seen": 2068316160
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030172116349047145,
+ "loss": 1.2875,
+ "theoretical_loss": 3.449286239841979,
+ "tokens_seen": 2068381696
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030170511534603815,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4492772625245776,
+ "tokens_seen": 2068447232
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030168906720160485,
+ "loss": 1.3147,
+ "theoretical_loss": 3.449268285571245,
+ "tokens_seen": 2068512768
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030167301905717155,
+ "loss": 1.2807,
+ "theoretical_loss": 3.4492593089819543,
+ "tokens_seen": 2068578304
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030165697091273826,
+ "loss": 1.3243,
+ "theoretical_loss": 3.44925033275668,
+ "tokens_seen": 2068643840
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030164092276830496,
+ "loss": 1.356,
+ "theoretical_loss": 3.449241356895395,
+ "tokens_seen": 2068709376
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030162487462387166,
+ "loss": 1.2533,
+ "theoretical_loss": 3.449232381398074,
+ "tokens_seen": 2068774912
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030160882647943836,
+ "loss": 1.293,
+ "theoretical_loss": 3.44922340626469,
+ "tokens_seen": 2068840448
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030159277833500506,
+ "loss": 1.1874,
+ "theoretical_loss": 3.4492144314952164,
+ "tokens_seen": 2068905984
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030157673019057176,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4492054570896276,
+ "tokens_seen": 2068971520
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030156068204613846,
+ "loss": 1.2675,
+ "theoretical_loss": 3.4491964830478974,
+ "tokens_seen": 2069037056
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003015446339017051,
+ "loss": 1.2156,
+ "theoretical_loss": 3.449187509369999,
+ "tokens_seen": 2069102592
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003015285857572718,
+ "loss": 1.2127,
+ "theoretical_loss": 3.449178536055906,
+ "tokens_seen": 2069168128
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003015125376128385,
+ "loss": 1.3288,
+ "theoretical_loss": 3.4491695631055928,
+ "tokens_seen": 2069233664
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014964894684052,
+ "loss": 1.2187,
+ "theoretical_loss": 3.449160590519033,
+ "tokens_seen": 2069299200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014804413239719,
+ "loss": 1.2864,
+ "theoretical_loss": 3.4491516182962,
+ "tokens_seen": 2069364736
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014643931795386,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4491426464370676,
+ "tokens_seen": 2069430272
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014483450351053,
+ "loss": 1.2983,
+ "theoretical_loss": 3.44913367494161,
+ "tokens_seen": 2069495808
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000301432296890672,
+ "loss": 1.1868,
+ "theoretical_loss": 3.4491247038098,
+ "tokens_seen": 2069561344
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014162487462387,
+ "loss": 1.2534,
+ "theoretical_loss": 3.4491157330416122,
+ "tokens_seen": 2069626880
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003014002006018054,
+ "loss": 1.2948,
+ "theoretical_loss": 3.4491067626370198,
+ "tokens_seen": 2069692416
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013841524573721,
+ "loss": 1.1721,
+ "theoretical_loss": 3.449097792595997,
+ "tokens_seen": 2069757952
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013681043129388,
+ "loss": 1.2437,
+ "theoretical_loss": 3.449088822918517,
+ "tokens_seen": 2069823488
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013520561685055,
+ "loss": 1.1332,
+ "theoretical_loss": 3.4490798536045544,
+ "tokens_seen": 2069889024
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013360080240722,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4490708846540823,
+ "tokens_seen": 2069954560
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013199598796389,
+ "loss": 1.2069,
+ "theoretical_loss": 3.449061916067074,
+ "tokens_seen": 2070020096
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003013039117352056,
+ "loss": 1.1512,
+ "theoretical_loss": 3.449052947843505,
+ "tokens_seen": 2070085632
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003012878635907723,
+ "loss": 1.303,
+ "theoretical_loss": 3.4490439799833474,
+ "tokens_seen": 2070151168
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000301271815446339,
+ "loss": 1.2966,
+ "theoretical_loss": 3.4490350124865756,
+ "tokens_seen": 2070216704
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003012557673019057,
+ "loss": 1.2817,
+ "theoretical_loss": 3.449026045353163,
+ "tokens_seen": 2070282240
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003012397191574724,
+ "loss": 1.2739,
+ "theoretical_loss": 3.449017078583084,
+ "tokens_seen": 2070347776
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003012236710130391,
+ "loss": 1.3001,
+ "theoretical_loss": 3.449008112176312,
+ "tokens_seen": 2070413312
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003012076228686058,
+ "loss": 1.2974,
+ "theoretical_loss": 3.448999146132821,
+ "tokens_seen": 2070478848
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003011915747241725,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4489901804525847,
+ "tokens_seen": 2070544384
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003011755265797392,
+ "loss": 1.2141,
+ "theoretical_loss": 3.4489812151355763,
+ "tokens_seen": 2070609920
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003011594784353059,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4489722501817703,
+ "tokens_seen": 2070675456
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003011434302908726,
+ "loss": 1.2738,
+ "theoretical_loss": 3.44896328559114,
+ "tokens_seen": 2070740992
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003011273821464393,
+ "loss": 1.1986,
+ "theoretical_loss": 3.44895432136366,
+ "tokens_seen": 2070806528
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000301111334002006,
+ "loss": 1.2715,
+ "theoretical_loss": 3.448945357499303,
+ "tokens_seen": 2070872064
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 693021,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3823573589324951,
+ "objective/train/theoretical_loss": 3.4489363939980446,
+ "objective/train/tokens_used": 2091397600,
+ "theoretical_loss": 3.4489363939980446,
+ "tokens_seen": 2070937600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010952858575727,
+ "loss": 1.356,
+ "theoretical_loss": 3.4489363939980446,
+ "tokens_seen": 2070937600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010792377131394,
+ "loss": 1.2463,
+ "theoretical_loss": 3.4489274308598565,
+ "tokens_seen": 2071003136
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010631895687061,
+ "loss": 1.1544,
+ "theoretical_loss": 3.448918468084713,
+ "tokens_seen": 2071068672
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010471414242728,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4489095056725887,
+ "tokens_seen": 2071134208
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010310932798395,
+ "loss": 1.1969,
+ "theoretical_loss": 3.4489005436234574,
+ "tokens_seen": 2071199744
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003010150451354062,
+ "loss": 1.181,
+ "theoretical_loss": 3.448891581937292,
+ "tokens_seen": 2071265280
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003009989969909729,
+ "loss": 1.2674,
+ "theoretical_loss": 3.448882620614067,
+ "tokens_seen": 2071330816
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030098294884653963,
+ "loss": 1.251,
+ "theoretical_loss": 3.4488736596537564,
+ "tokens_seen": 2071396352
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030096690070210633,
+ "loss": 1.3765,
+ "theoretical_loss": 3.448864699056333,
+ "tokens_seen": 2071461888
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030095085255767303,
+ "loss": 1.2401,
+ "theoretical_loss": 3.4488557388217718,
+ "tokens_seen": 2071527424
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030093480441323973,
+ "loss": 1.2398,
+ "theoretical_loss": 3.4488467789500454,
+ "tokens_seen": 2071592960
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030091875626880643,
+ "loss": 1.2332,
+ "theoretical_loss": 3.4488378194411293,
+ "tokens_seen": 2071658496
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030090270812437313,
+ "loss": 1.2895,
+ "theoretical_loss": 3.448828860294996,
+ "tokens_seen": 2071724032
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030088665997993983,
+ "loss": 1.1977,
+ "theoretical_loss": 3.4488199015116194,
+ "tokens_seen": 2071789568
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030087061183550653,
+ "loss": 1.2498,
+ "theoretical_loss": 3.448810943090974,
+ "tokens_seen": 2071855104
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030085456369107323,
+ "loss": 1.2028,
+ "theoretical_loss": 3.4488019850330334,
+ "tokens_seen": 2071920640
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030083851554663993,
+ "loss": 1.2523,
+ "theoretical_loss": 3.4487930273377714,
+ "tokens_seen": 2071986176
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030082246740220663,
+ "loss": 1.3033,
+ "theoretical_loss": 3.4487840700051615,
+ "tokens_seen": 2072051712
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030080641925777333,
+ "loss": 1.2357,
+ "theoretical_loss": 3.4487751130351776,
+ "tokens_seen": 2072117248
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030079037111334004,
+ "loss": 1.2496,
+ "theoretical_loss": 3.4487661564277943,
+ "tokens_seen": 2072182784
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030077432296890674,
+ "loss": 1.3466,
+ "theoretical_loss": 3.4487572001829845,
+ "tokens_seen": 2072248320
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030075827482447344,
+ "loss": 1.3033,
+ "theoretical_loss": 3.448748244300723,
+ "tokens_seen": 2072313856
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030074222668004014,
+ "loss": 1.1555,
+ "theoretical_loss": 3.4487392887809825,
+ "tokens_seen": 2072379392
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030072617853560684,
+ "loss": 1.1323,
+ "theoretical_loss": 3.448730333623738,
+ "tokens_seen": 2072444928
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030071013039117354,
+ "loss": 1.2852,
+ "theoretical_loss": 3.448721378828963,
+ "tokens_seen": 2072510464
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030069408224674024,
+ "loss": 1.2393,
+ "theoretical_loss": 3.4487124243966307,
+ "tokens_seen": 2072576000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030067803410230694,
+ "loss": 1.3238,
+ "theoretical_loss": 3.448703470326716,
+ "tokens_seen": 2072641536
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030066198595787364,
+ "loss": 1.3059,
+ "theoretical_loss": 3.448694516619192,
+ "tokens_seen": 2072707072
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030064593781344034,
+ "loss": 1.1547,
+ "theoretical_loss": 3.448685563274033,
+ "tokens_seen": 2072772608
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030062988966900704,
+ "loss": 1.2905,
+ "theoretical_loss": 3.448676610291213,
+ "tokens_seen": 2072838144
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030061384152457374,
+ "loss": 1.2688,
+ "theoretical_loss": 3.448667657670705,
+ "tokens_seen": 2072903680
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030059779338014044,
+ "loss": 1.2954,
+ "theoretical_loss": 3.448658705412484,
+ "tokens_seen": 2072969216
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030058174523570714,
+ "loss": 1.3422,
+ "theoretical_loss": 3.4486497535165235,
+ "tokens_seen": 2073034752
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030056569709127384,
+ "loss": 1.1013,
+ "theoretical_loss": 3.4486408019827968,
+ "tokens_seen": 2073100288
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030054964894684054,
+ "loss": 1.193,
+ "theoretical_loss": 3.4486318508112785,
+ "tokens_seen": 2073165824
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030053360080240725,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4486229000019426,
+ "tokens_seen": 2073231360
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030051755265797395,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4486139495547623,
+ "tokens_seen": 2073296896
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030050150451354065,
+ "loss": 1.3084,
+ "theoretical_loss": 3.448604999469712,
+ "tokens_seen": 2073362432
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030048545636910735,
+ "loss": 1.3016,
+ "theoretical_loss": 3.4485960497467656,
+ "tokens_seen": 2073427968
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030046940822467405,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4485871003858968,
+ "tokens_seen": 2073493504
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030045336008024075,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4485781513870792,
+ "tokens_seen": 2073559040
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030043731193580745,
+ "loss": 1.2323,
+ "theoretical_loss": 3.4485692027502877,
+ "tokens_seen": 2073624576
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030042126379137415,
+ "loss": 1.2912,
+ "theoretical_loss": 3.4485602544754954,
+ "tokens_seen": 2073690112
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030040521564694085,
+ "loss": 1.2085,
+ "theoretical_loss": 3.448551306562676,
+ "tokens_seen": 2073755648
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030038916750250755,
+ "loss": 1.2638,
+ "theoretical_loss": 3.448542359011804,
+ "tokens_seen": 2073821184
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030037311935807425,
+ "loss": 1.2204,
+ "theoretical_loss": 3.4485334118228534,
+ "tokens_seen": 2073886720
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030035707121364095,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4485244649957982,
+ "tokens_seen": 2073952256
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030034102306920765,
+ "loss": 1.2493,
+ "theoretical_loss": 3.448515518530612,
+ "tokens_seen": 2074017792
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030032497492477435,
+ "loss": 1.2684,
+ "theoretical_loss": 3.4485065724272683,
+ "tokens_seen": 2074083328
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030030892678034105,
+ "loss": 1.2903,
+ "theoretical_loss": 3.4484976266857412,
+ "tokens_seen": 2074148864
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 694242,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2040023803710938,
+ "objective/train/theoretical_loss": 3.4484886813060056,
+ "objective/train/tokens_used": 2094674400,
+ "theoretical_loss": 3.4484886813060056,
+ "tokens_seen": 2074214400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030029287863590775,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4484886813060056,
+ "tokens_seen": 2074214400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030027683049147446,
+ "loss": 1.3644,
+ "theoretical_loss": 3.448479736288035,
+ "tokens_seen": 2074279936
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030026078234704116,
+ "loss": 1.2882,
+ "theoretical_loss": 3.4484707916318023,
+ "tokens_seen": 2074345472
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030024473420260786,
+ "loss": 1.2555,
+ "theoretical_loss": 3.448461847337282,
+ "tokens_seen": 2074411008
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030022868605817456,
+ "loss": 1.205,
+ "theoretical_loss": 3.4484529034044495,
+ "tokens_seen": 2074476544
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030021263791374126,
+ "loss": 1.2684,
+ "theoretical_loss": 3.4484439598332766,
+ "tokens_seen": 2074542080
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030019658976930796,
+ "loss": 1.2296,
+ "theoretical_loss": 3.4484350166237387,
+ "tokens_seen": 2074607616
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030018054162487466,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4484260737758086,
+ "tokens_seen": 2074673152
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00030016449348044136,
+ "loss": 1.1925,
+ "theoretical_loss": 3.448417131289461,
+ "tokens_seen": 2074738688
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000300148445336008,
+ "loss": 1.2899,
+ "theoretical_loss": 3.4484081891646703,
+ "tokens_seen": 2074804224
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003001323971915747,
+ "loss": 1.3365,
+ "theoretical_loss": 3.44839924740141,
+ "tokens_seen": 2074869760
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003001163490471414,
+ "loss": 1.251,
+ "theoretical_loss": 3.4483903059996535,
+ "tokens_seen": 2074935296
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003001003009027081,
+ "loss": 1.2492,
+ "theoretical_loss": 3.448381364959375,
+ "tokens_seen": 2075000832
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000842527582748,
+ "loss": 1.2864,
+ "theoretical_loss": 3.4483724242805494,
+ "tokens_seen": 2075066368
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000682046138415,
+ "loss": 1.2724,
+ "theoretical_loss": 3.44836348396315,
+ "tokens_seen": 2075131904
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000521564694082,
+ "loss": 1.1596,
+ "theoretical_loss": 3.44835454400715,
+ "tokens_seen": 2075197440
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000361083249749,
+ "loss": 1.274,
+ "theoretical_loss": 3.448345604412525,
+ "tokens_seen": 2075262976
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000200601805416,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4483366651792475,
+ "tokens_seen": 2075328512
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0003000040120361083,
+ "loss": 1.2547,
+ "theoretical_loss": 3.4483277263072925,
+ "tokens_seen": 2075394048
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000299987963891675,
+ "loss": 1.2258,
+ "theoretical_loss": 3.4483187877966337,
+ "tokens_seen": 2075459584
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002999719157472417,
+ "loss": 1.1796,
+ "theoretical_loss": 3.4483098496472446,
+ "tokens_seen": 2075525120
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002999558676028084,
+ "loss": 1.1559,
+ "theoretical_loss": 3.4483009118591,
+ "tokens_seen": 2075590656
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002999398194583751,
+ "loss": 1.3152,
+ "theoretical_loss": 3.448291974432173,
+ "tokens_seen": 2075656192
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002999237713139418,
+ "loss": 1.1918,
+ "theoretical_loss": 3.448283037366438,
+ "tokens_seen": 2075721728
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002999077231695085,
+ "loss": 1.2346,
+ "theoretical_loss": 3.4482741006618696,
+ "tokens_seen": 2075787264
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002998916750250752,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4482651643184408,
+ "tokens_seen": 2075852800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002998756268806419,
+ "loss": 1.2301,
+ "theoretical_loss": 3.4482562283361267,
+ "tokens_seen": 2075918336
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002998595787362086,
+ "loss": 1.1728,
+ "theoretical_loss": 3.4482472927148997,
+ "tokens_seen": 2075983872
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002998435305917753,
+ "loss": 1.3189,
+ "theoretical_loss": 3.448238357454736,
+ "tokens_seen": 2076049408
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000299827482447342,
+ "loss": 1.2279,
+ "theoretical_loss": 3.448229422555608,
+ "tokens_seen": 2076114944
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002998114343029087,
+ "loss": 1.4536,
+ "theoretical_loss": 3.4482204880174896,
+ "tokens_seen": 2076180480
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997953861584754,
+ "loss": 1.1946,
+ "theoretical_loss": 3.4482115538403555,
+ "tokens_seen": 2076246016
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997793380140421,
+ "loss": 1.2129,
+ "theoretical_loss": 3.4482026200241798,
+ "tokens_seen": 2076311552
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997632898696088,
+ "loss": 1.232,
+ "theoretical_loss": 3.448193686568936,
+ "tokens_seen": 2076377088
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997472417251755,
+ "loss": 1.371,
+ "theoretical_loss": 3.4481847534745986,
+ "tokens_seen": 2076442624
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997311935807422,
+ "loss": 1.2729,
+ "theoretical_loss": 3.4481758207411417,
+ "tokens_seen": 2076508160
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002997151454363089,
+ "loss": 1.2036,
+ "theoretical_loss": 3.4481668883685392,
+ "tokens_seen": 2076573696
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996990972918757,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4481579563567646,
+ "tokens_seen": 2076639232
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996830491474424,
+ "loss": 1.2059,
+ "theoretical_loss": 3.448149024705792,
+ "tokens_seen": 2076704768
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996670010030091,
+ "loss": 1.2239,
+ "theoretical_loss": 3.4481400934155966,
+ "tokens_seen": 2076770304
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996509528585758,
+ "loss": 1.1539,
+ "theoretical_loss": 3.4481311624861513,
+ "tokens_seen": 2076835840
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996349047141425,
+ "loss": 1.231,
+ "theoretical_loss": 3.4481222319174307,
+ "tokens_seen": 2076901376
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996188565697092,
+ "loss": 1.2545,
+ "theoretical_loss": 3.448113301709408,
+ "tokens_seen": 2076966912
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002996028084252759,
+ "loss": 1.243,
+ "theoretical_loss": 3.4481043718620583,
+ "tokens_seen": 2077032448
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002995867602808426,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4480954423753554,
+ "tokens_seen": 2077097984
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002995707121364093,
+ "loss": 1.3025,
+ "theoretical_loss": 3.448086513249273,
+ "tokens_seen": 2077163520
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000299554663991976,
+ "loss": 1.3754,
+ "theoretical_loss": 3.4480775844837854,
+ "tokens_seen": 2077229056
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002995386158475427,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4480686560788665,
+ "tokens_seen": 2077294592
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029952256770310933,
+ "loss": 1.1699,
+ "theoretical_loss": 3.4480597280344902,
+ "tokens_seen": 2077360128
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029950651955867603,
+ "loss": 1.281,
+ "theoretical_loss": 3.4480508003506314,
+ "tokens_seen": 2077425664
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 695255,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.125720739364624,
+ "objective/train/theoretical_loss": 3.4480418730272637,
+ "objective/train/tokens_used": 2097951200,
+ "theoretical_loss": 3.4480418730272637,
+ "tokens_seen": 2077491200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029949047141424273,
+ "loss": 1.237,
+ "theoretical_loss": 3.4480418730272637,
+ "tokens_seen": 2077491200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029947442326980943,
+ "loss": 1.22,
+ "theoretical_loss": 3.4480329460643606,
+ "tokens_seen": 2077556736
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029945837512537613,
+ "loss": 1.254,
+ "theoretical_loss": 3.448024019461897,
+ "tokens_seen": 2077622272
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029944232698094283,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4480150932198463,
+ "tokens_seen": 2077687808
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029942627883650953,
+ "loss": 1.1979,
+ "theoretical_loss": 3.448006167338183,
+ "tokens_seen": 2077753344
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029941023069207624,
+ "loss": 1.2815,
+ "theoretical_loss": 3.447997241816881,
+ "tokens_seen": 2077818880
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029939418254764294,
+ "loss": 1.1726,
+ "theoretical_loss": 3.447988316655915,
+ "tokens_seen": 2077884416
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029937813440320964,
+ "loss": 1.2554,
+ "theoretical_loss": 3.447979391855258,
+ "tokens_seen": 2077949952
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029936208625877634,
+ "loss": 1.3344,
+ "theoretical_loss": 3.447970467414885,
+ "tokens_seen": 2078015488
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029934603811434304,
+ "loss": 1.1866,
+ "theoretical_loss": 3.44796154333477,
+ "tokens_seen": 2078081024
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029932998996990974,
+ "loss": 1.1973,
+ "theoretical_loss": 3.4479526196148864,
+ "tokens_seen": 2078146560
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029931394182547644,
+ "loss": 1.2373,
+ "theoretical_loss": 3.447943696255209,
+ "tokens_seen": 2078212096
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029929789368104314,
+ "loss": 1.1871,
+ "theoretical_loss": 3.447934773255712,
+ "tokens_seen": 2078277632
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029928184553660984,
+ "loss": 1.276,
+ "theoretical_loss": 3.447925850616368,
+ "tokens_seen": 2078343168
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029926579739217654,
+ "loss": 1.2682,
+ "theoretical_loss": 3.447916928337153,
+ "tokens_seen": 2078408704
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029924974924774324,
+ "loss": 1.1884,
+ "theoretical_loss": 3.4479080064180407,
+ "tokens_seen": 2078474240
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029923370110330994,
+ "loss": 1.3241,
+ "theoretical_loss": 3.4478990848590048,
+ "tokens_seen": 2078539776
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029921765295887664,
+ "loss": 1.3157,
+ "theoretical_loss": 3.447890163660019,
+ "tokens_seen": 2078605312
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029920160481444334,
+ "loss": 1.3705,
+ "theoretical_loss": 3.447881242821058,
+ "tokens_seen": 2078670848
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029918555667001004,
+ "loss": 1.2891,
+ "theoretical_loss": 3.4478723223420964,
+ "tokens_seen": 2078736384
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029916950852557675,
+ "loss": 1.2647,
+ "theoretical_loss": 3.447863402223107,
+ "tokens_seen": 2078801920
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029915346038114345,
+ "loss": 1.3147,
+ "theoretical_loss": 3.4478544824640656,
+ "tokens_seen": 2078867456
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029913741223671015,
+ "loss": 1.24,
+ "theoretical_loss": 3.447845563064945,
+ "tokens_seen": 2078932992
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029912136409227685,
+ "loss": 1.3141,
+ "theoretical_loss": 3.4478366440257195,
+ "tokens_seen": 2078998528
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029910531594784355,
+ "loss": 1.2183,
+ "theoretical_loss": 3.447827725346364,
+ "tokens_seen": 2079064064
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029908926780341025,
+ "loss": 1.2379,
+ "theoretical_loss": 3.447818807026852,
+ "tokens_seen": 2079129600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029907321965897695,
+ "loss": 1.2783,
+ "theoretical_loss": 3.447809889067157,
+ "tokens_seen": 2079195136
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029905717151454365,
+ "loss": 1.2371,
+ "theoretical_loss": 3.447800971467255,
+ "tokens_seen": 2079260672
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029904112337011035,
+ "loss": 1.2518,
+ "theoretical_loss": 3.4477920542271185,
+ "tokens_seen": 2079326208
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029902507522567705,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4477831373467223,
+ "tokens_seen": 2079391744
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029900902708124375,
+ "loss": 1.2536,
+ "theoretical_loss": 3.4477742208260405,
+ "tokens_seen": 2079457280
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029899297893681045,
+ "loss": 1.2268,
+ "theoretical_loss": 3.4477653046650474,
+ "tokens_seen": 2079522816
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029897693079237715,
+ "loss": 1.322,
+ "theoretical_loss": 3.4477563888637164,
+ "tokens_seen": 2079588352
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029896088264794385,
+ "loss": 1.2246,
+ "theoretical_loss": 3.4477474734220226,
+ "tokens_seen": 2079653888
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029894483450351055,
+ "loss": 1.2457,
+ "theoretical_loss": 3.44773855833994,
+ "tokens_seen": 2079719424
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029892878635907725,
+ "loss": 1.2413,
+ "theoretical_loss": 3.447729643617442,
+ "tokens_seen": 2079784960
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029891273821464396,
+ "loss": 1.2093,
+ "theoretical_loss": 3.4477207292545033,
+ "tokens_seen": 2079850496
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029889669007021066,
+ "loss": 1.3034,
+ "theoretical_loss": 3.4477118152510986,
+ "tokens_seen": 2079916032
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029888064192577736,
+ "loss": 1.273,
+ "theoretical_loss": 3.447702901607201,
+ "tokens_seen": 2079981568
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029886459378134406,
+ "loss": 1.2364,
+ "theoretical_loss": 3.4476939883227855,
+ "tokens_seen": 2080047104
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029884854563691076,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4476850753978265,
+ "tokens_seen": 2080112640
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029883249749247746,
+ "loss": 1.2876,
+ "theoretical_loss": 3.447676162832297,
+ "tokens_seen": 2080178176
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029881644934804416,
+ "loss": 1.2884,
+ "theoretical_loss": 3.447667250626172,
+ "tokens_seen": 2080243712
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029880040120361086,
+ "loss": 1.3132,
+ "theoretical_loss": 3.4476583387794255,
+ "tokens_seen": 2080309248
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029878435305917756,
+ "loss": 1.3254,
+ "theoretical_loss": 3.447649427292032,
+ "tokens_seen": 2080374784
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029876830491474426,
+ "loss": 1.4085,
+ "theoretical_loss": 3.447640516163965,
+ "tokens_seen": 2080440320
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002987522567703109,
+ "loss": 1.2418,
+ "theoretical_loss": 3.4476316053951996,
+ "tokens_seen": 2080505856
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002987362086258776,
+ "loss": 1.1698,
+ "theoretical_loss": 3.447622694985709,
+ "tokens_seen": 2080571392
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002987201604814443,
+ "loss": 1.2114,
+ "theoretical_loss": 3.447613784935468,
+ "tokens_seen": 2080636928
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000298704112337011,
+ "loss": 1.198,
+ "theoretical_loss": 3.4476048752444513,
+ "tokens_seen": 2080702464
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 696439,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2649786472320557,
+ "objective/train/theoretical_loss": 3.4475959659126323,
+ "objective/train/tokens_used": 2101228000,
+ "theoretical_loss": 3.4475959659126323,
+ "tokens_seen": 2080768000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986880641925777,
+ "loss": 1.2952,
+ "theoretical_loss": 3.4475959659126323,
+ "tokens_seen": 2080768000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986720160481444,
+ "loss": 1.3234,
+ "theoretical_loss": 3.447587056939985,
+ "tokens_seen": 2080833536
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986559679037111,
+ "loss": 1.1959,
+ "theoretical_loss": 3.4475781483264845,
+ "tokens_seen": 2080899072
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986399197592778,
+ "loss": 1.1853,
+ "theoretical_loss": 3.447569240072104,
+ "tokens_seen": 2080964608
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986238716148445,
+ "loss": 1.22,
+ "theoretical_loss": 3.4475603321768187,
+ "tokens_seen": 2081030144
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002986078234704112,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4475514246406025,
+ "tokens_seen": 2081095680
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002985917753259779,
+ "loss": 1.2263,
+ "theoretical_loss": 3.447542517463429,
+ "tokens_seen": 2081161216
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002985757271815446,
+ "loss": 1.2903,
+ "theoretical_loss": 3.447533610645273,
+ "tokens_seen": 2081226752
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002985596790371113,
+ "loss": 1.3758,
+ "theoretical_loss": 3.447524704186109,
+ "tokens_seen": 2081292288
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000298543630892678,
+ "loss": 1.2131,
+ "theoretical_loss": 3.4475157980859104,
+ "tokens_seen": 2081357824
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002985275827482447,
+ "loss": 1.2638,
+ "theoretical_loss": 3.447506892344652,
+ "tokens_seen": 2081423360
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002985115346038114,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4474979869623086,
+ "tokens_seen": 2081488896
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029849548645937817,
+ "loss": 1.3461,
+ "theoretical_loss": 3.447489081938853,
+ "tokens_seen": 2081554432
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029847943831494487,
+ "loss": 1.3332,
+ "theoretical_loss": 3.4474801772742607,
+ "tokens_seen": 2081619968
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002984633901705116,
+ "loss": 1.2435,
+ "theoretical_loss": 3.447471272968505,
+ "tokens_seen": 2081685504
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002984473420260783,
+ "loss": 1.192,
+ "theoretical_loss": 3.4474623690215607,
+ "tokens_seen": 2081751040
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000298431293881645,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4474534654334024,
+ "tokens_seen": 2081816576
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002984152457372117,
+ "loss": 1.1925,
+ "theoretical_loss": 3.4474445622040033,
+ "tokens_seen": 2081882112
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983991975927784,
+ "loss": 1.2782,
+ "theoretical_loss": 3.4474356593333386,
+ "tokens_seen": 2081947648
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983831494483451,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4474267568213817,
+ "tokens_seen": 2082013184
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983671013039118,
+ "loss": 1.2486,
+ "theoretical_loss": 3.4474178546681076,
+ "tokens_seen": 2082078720
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983510531594785,
+ "loss": 1.333,
+ "theoretical_loss": 3.4474089528734906,
+ "tokens_seen": 2082144256
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983350050150452,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4474000514375045,
+ "tokens_seen": 2082209792
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983189568706119,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4473911503601236,
+ "tokens_seen": 2082275328
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002983029087261786,
+ "loss": 1.1468,
+ "theoretical_loss": 3.4473822496413224,
+ "tokens_seen": 2082340864
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002982868605817453,
+ "loss": 1.2998,
+ "theoretical_loss": 3.447373349281075,
+ "tokens_seen": 2082406400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000298270812437312,
+ "loss": 1.2871,
+ "theoretical_loss": 3.447364449279356,
+ "tokens_seen": 2082471936
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002982547642928787,
+ "loss": 1.2146,
+ "theoretical_loss": 3.447355549636139,
+ "tokens_seen": 2082537472
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002982387161484454,
+ "loss": 1.2798,
+ "theoretical_loss": 3.447346650351399,
+ "tokens_seen": 2082603008
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002982226680040121,
+ "loss": 1.1781,
+ "theoretical_loss": 3.44733775142511,
+ "tokens_seen": 2082668544
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002982066198595788,
+ "loss": 1.2442,
+ "theoretical_loss": 3.447328852857246,
+ "tokens_seen": 2082734080
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002981905717151455,
+ "loss": 1.291,
+ "theoretical_loss": 3.4473199546477815,
+ "tokens_seen": 2082799616
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002981745235707122,
+ "loss": 1.2626,
+ "theoretical_loss": 3.447311056796691,
+ "tokens_seen": 2082865152
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002981584754262789,
+ "loss": 1.206,
+ "theoretical_loss": 3.4473021593039483,
+ "tokens_seen": 2082930688
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002981424272818456,
+ "loss": 1.3288,
+ "theoretical_loss": 3.4472932621695285,
+ "tokens_seen": 2082996224
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002981263791374123,
+ "loss": 1.2887,
+ "theoretical_loss": 3.447284365393405,
+ "tokens_seen": 2083061760
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029811033099297893,
+ "loss": 1.3171,
+ "theoretical_loss": 3.447275468975553,
+ "tokens_seen": 2083127296
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029809428284854563,
+ "loss": 1.2533,
+ "theoretical_loss": 3.447266572915946,
+ "tokens_seen": 2083192832
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029807823470411233,
+ "loss": 1.2579,
+ "theoretical_loss": 3.4472576772145587,
+ "tokens_seen": 2083258368
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029806218655967903,
+ "loss": 1.3818,
+ "theoretical_loss": 3.4472487818713655,
+ "tokens_seen": 2083323904
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029804613841524574,
+ "loss": 1.265,
+ "theoretical_loss": 3.44723988688634,
+ "tokens_seen": 2083389440
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029803009027081244,
+ "loss": 1.1827,
+ "theoretical_loss": 3.447230992259458,
+ "tokens_seen": 2083454976
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029801404212637914,
+ "loss": 1.2305,
+ "theoretical_loss": 3.447222097990692,
+ "tokens_seen": 2083520512
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029799799398194584,
+ "loss": 1.2945,
+ "theoretical_loss": 3.4472132040800174,
+ "tokens_seen": 2083586048
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029798194583751254,
+ "loss": 1.2065,
+ "theoretical_loss": 3.4472043105274084,
+ "tokens_seen": 2083651584
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029796589769307924,
+ "loss": 1.2726,
+ "theoretical_loss": 3.447195417332839,
+ "tokens_seen": 2083717120
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029794984954864594,
+ "loss": 1.2113,
+ "theoretical_loss": 3.447186524496284,
+ "tokens_seen": 2083782656
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029793380140421264,
+ "loss": 1.2808,
+ "theoretical_loss": 3.447177632017717,
+ "tokens_seen": 2083848192
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029791775325977934,
+ "loss": 1.1871,
+ "theoretical_loss": 3.447168739897114,
+ "tokens_seen": 2083913728
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029790170511534604,
+ "loss": 1.249,
+ "theoretical_loss": 3.447159848134447,
+ "tokens_seen": 2083979264
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 697706,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2984927892684937,
+ "objective/train/theoretical_loss": 3.4471509567296916,
+ "objective/train/tokens_used": 2104504800,
+ "theoretical_loss": 3.4471509567296916,
+ "tokens_seen": 2084044800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029788565697091274,
+ "loss": 1.3036,
+ "theoretical_loss": 3.4471509567296916,
+ "tokens_seen": 2084044800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029786960882647944,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4471420656828227,
+ "tokens_seen": 2084110336
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029785356068204614,
+ "loss": 1.1548,
+ "theoretical_loss": 3.447133174993813,
+ "tokens_seen": 2084175872
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029783751253761284,
+ "loss": 1.2451,
+ "theoretical_loss": 3.447124284662639,
+ "tokens_seen": 2084241408
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029782146439317954,
+ "loss": 1.2719,
+ "theoretical_loss": 3.447115394689273,
+ "tokens_seen": 2084306944
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029780541624874624,
+ "loss": 1.3192,
+ "theoretical_loss": 3.44710650507369,
+ "tokens_seen": 2084372480
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029778936810431295,
+ "loss": 1.3094,
+ "theoretical_loss": 3.4470976158158653,
+ "tokens_seen": 2084438016
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029777331995987965,
+ "loss": 1.2804,
+ "theoretical_loss": 3.447088726915772,
+ "tokens_seen": 2084503552
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029775727181544635,
+ "loss": 1.2738,
+ "theoretical_loss": 3.4470798383733854,
+ "tokens_seen": 2084569088
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029774122367101305,
+ "loss": 1.2337,
+ "theoretical_loss": 3.447070950188679,
+ "tokens_seen": 2084634624
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029772517552657975,
+ "loss": 1.2368,
+ "theoretical_loss": 3.4470620623616277,
+ "tokens_seen": 2084700160
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029770912738214645,
+ "loss": 1.3068,
+ "theoretical_loss": 3.447053174892206,
+ "tokens_seen": 2084765696
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029769307923771315,
+ "loss": 1.3187,
+ "theoretical_loss": 3.447044287780387,
+ "tokens_seen": 2084831232
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029767703109327985,
+ "loss": 1.3644,
+ "theoretical_loss": 3.447035401026147,
+ "tokens_seen": 2084896768
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029766098294884655,
+ "loss": 1.2888,
+ "theoretical_loss": 3.4470265146294596,
+ "tokens_seen": 2084962304
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029764493480441325,
+ "loss": 1.2274,
+ "theoretical_loss": 3.4470176285902987,
+ "tokens_seen": 2085027840
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029762888665997995,
+ "loss": 1.26,
+ "theoretical_loss": 3.4470087429086393,
+ "tokens_seen": 2085093376
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029761283851554665,
+ "loss": 1.1482,
+ "theoretical_loss": 3.446999857584455,
+ "tokens_seen": 2085158912
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029759679037111335,
+ "loss": 1.29,
+ "theoretical_loss": 3.446990972617721,
+ "tokens_seen": 2085224448
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029758074222668005,
+ "loss": 1.1949,
+ "theoretical_loss": 3.4469820880084114,
+ "tokens_seen": 2085289984
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029756469408224675,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4469732037565004,
+ "tokens_seen": 2085355520
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029754864593781345,
+ "loss": 1.212,
+ "theoretical_loss": 3.446964319861963,
+ "tokens_seen": 2085421056
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029753259779338016,
+ "loss": 1.1884,
+ "theoretical_loss": 3.4469554363247727,
+ "tokens_seen": 2085486592
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029751654964894686,
+ "loss": 1.2054,
+ "theoretical_loss": 3.4469465531449046,
+ "tokens_seen": 2085552128
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029750050150451356,
+ "loss": 1.2183,
+ "theoretical_loss": 3.446937670322333,
+ "tokens_seen": 2085617664
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029748445336008026,
+ "loss": 1.1213,
+ "theoretical_loss": 3.446928787857032,
+ "tokens_seen": 2085683200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029746840521564696,
+ "loss": 1.1907,
+ "theoretical_loss": 3.4469199057489757,
+ "tokens_seen": 2085748736
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029745235707121366,
+ "loss": 1.2878,
+ "theoretical_loss": 3.4469110239981395,
+ "tokens_seen": 2085814272
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029743630892678036,
+ "loss": 1.1109,
+ "theoretical_loss": 3.446902142604497,
+ "tokens_seen": 2085879808
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029742026078234706,
+ "loss": 1.2331,
+ "theoretical_loss": 3.4468932615680226,
+ "tokens_seen": 2085945344
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029740421263791376,
+ "loss": 1.3025,
+ "theoretical_loss": 3.4468843808886915,
+ "tokens_seen": 2086010880
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029738816449348046,
+ "loss": 1.2117,
+ "theoretical_loss": 3.4468755005664775,
+ "tokens_seen": 2086076416
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029737211634904716,
+ "loss": 1.3032,
+ "theoretical_loss": 3.446866620601355,
+ "tokens_seen": 2086141952
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029735606820461386,
+ "loss": 1.2764,
+ "theoretical_loss": 3.446857740993299,
+ "tokens_seen": 2086207488
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002973400200601805,
+ "loss": 1.268,
+ "theoretical_loss": 3.446848861742283,
+ "tokens_seen": 2086273024
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002973239719157472,
+ "loss": 1.303,
+ "theoretical_loss": 3.4468399828482816,
+ "tokens_seen": 2086338560
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002973079237713139,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4468311043112703,
+ "tokens_seen": 2086404096
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029729187562688067,
+ "loss": 1.3145,
+ "theoretical_loss": 3.4468222261312222,
+ "tokens_seen": 2086469632
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029727582748244737,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4468133483081127,
+ "tokens_seen": 2086535168
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029725977933801407,
+ "loss": 1.3114,
+ "theoretical_loss": 3.446804470841916,
+ "tokens_seen": 2086600704
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029724373119358077,
+ "loss": 1.3037,
+ "theoretical_loss": 3.446795593732606,
+ "tokens_seen": 2086666240
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029722768304914747,
+ "loss": 1.3834,
+ "theoretical_loss": 3.4467867169801574,
+ "tokens_seen": 2086731776
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029721163490471417,
+ "loss": 1.2186,
+ "theoretical_loss": 3.446777840584545,
+ "tokens_seen": 2086797312
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029719558676028087,
+ "loss": 1.2076,
+ "theoretical_loss": 3.4467689645457433,
+ "tokens_seen": 2086862848
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029717953861584757,
+ "loss": 1.2087,
+ "theoretical_loss": 3.4467600888637264,
+ "tokens_seen": 2086928384
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029716349047141427,
+ "loss": 1.1611,
+ "theoretical_loss": 3.4467512135384686,
+ "tokens_seen": 2086993920
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029714744232698097,
+ "loss": 1.2285,
+ "theoretical_loss": 3.446742338569945,
+ "tokens_seen": 2087059456
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029713139418254767,
+ "loss": 1.1244,
+ "theoretical_loss": 3.4467334639581293,
+ "tokens_seen": 2087124992
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029711534603811437,
+ "loss": 1.3254,
+ "theoretical_loss": 3.4467245897029963,
+ "tokens_seen": 2087190528
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029709929789368107,
+ "loss": 1.287,
+ "theoretical_loss": 3.4467157158045207,
+ "tokens_seen": 2087256064
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 698576,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2330121994018555,
+ "objective/train/theoretical_loss": 3.446706842262677,
+ "objective/train/tokens_used": 2107781600,
+ "theoretical_loss": 3.446706842262677,
+ "tokens_seen": 2087321600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970832497492478,
+ "loss": 1.237,
+ "theoretical_loss": 3.446706842262677,
+ "tokens_seen": 2087321600
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970672016048145,
+ "loss": 1.21,
+ "theoretical_loss": 3.446697969077439,
+ "tokens_seen": 2087387136
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970511534603812,
+ "loss": 1.1829,
+ "theoretical_loss": 3.4466890962487815,
+ "tokens_seen": 2087452672
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970351053159479,
+ "loss": 1.2723,
+ "theoretical_loss": 3.4466802237766796,
+ "tokens_seen": 2087518208
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970190571715146,
+ "loss": 1.2567,
+ "theoretical_loss": 3.4466713516611067,
+ "tokens_seen": 2087583744
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002970030090270813,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4466624799020384,
+ "tokens_seen": 2087649280
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000296986960882648,
+ "loss": 1.2271,
+ "theoretical_loss": 3.4466536084994486,
+ "tokens_seen": 2087714816
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002969709127382147,
+ "loss": 1.2229,
+ "theoretical_loss": 3.4466447374533113,
+ "tokens_seen": 2087780352
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002969548645937814,
+ "loss": 1.2971,
+ "theoretical_loss": 3.446635866763602,
+ "tokens_seen": 2087845888
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002969388164493481,
+ "loss": 1.2552,
+ "theoretical_loss": 3.4466269964302945,
+ "tokens_seen": 2087911424
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002969227683049148,
+ "loss": 1.2638,
+ "theoretical_loss": 3.4466181264533633,
+ "tokens_seen": 2087976960
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002969067201604815,
+ "loss": 1.2966,
+ "theoretical_loss": 3.446609256832783,
+ "tokens_seen": 2088042496
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002968906720160482,
+ "loss": 1.2428,
+ "theoretical_loss": 3.4466003875685285,
+ "tokens_seen": 2088108032
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002968746238716149,
+ "loss": 1.2396,
+ "theoretical_loss": 3.446591518660574,
+ "tokens_seen": 2088173568
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002968585757271816,
+ "loss": 1.2942,
+ "theoretical_loss": 3.446582650108894,
+ "tokens_seen": 2088239104
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002968425275827483,
+ "loss": 1.3371,
+ "theoretical_loss": 3.4465737819134628,
+ "tokens_seen": 2088304640
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000296826479438315,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4465649140742554,
+ "tokens_seen": 2088370176
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002968104312938817,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4465560465912457,
+ "tokens_seen": 2088435712
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002967943831494484,
+ "loss": 1.2756,
+ "theoretical_loss": 3.446547179464409,
+ "tokens_seen": 2088501248
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002967783350050151,
+ "loss": 1.3114,
+ "theoretical_loss": 3.446538312693719,
+ "tokens_seen": 2088566784
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002967622868605818,
+ "loss": 1.2946,
+ "theoretical_loss": 3.4465294462791505,
+ "tokens_seen": 2088632320
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002967462387161485,
+ "loss": 1.1559,
+ "theoretical_loss": 3.446520580220678,
+ "tokens_seen": 2088697856
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002967301905717152,
+ "loss": 1.288,
+ "theoretical_loss": 3.4465117145182766,
+ "tokens_seen": 2088763392
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029671414242728183,
+ "loss": 1.4131,
+ "theoretical_loss": 3.44650284917192,
+ "tokens_seen": 2088828928
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029669809428284853,
+ "loss": 1.2796,
+ "theoretical_loss": 3.446493984181583,
+ "tokens_seen": 2088894464
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029668204613841524,
+ "loss": 1.3392,
+ "theoretical_loss": 3.4464851195472406,
+ "tokens_seen": 2088960000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029666599799398194,
+ "loss": 1.2528,
+ "theoretical_loss": 3.446476255268867,
+ "tokens_seen": 2089025536
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029664994984954864,
+ "loss": 1.2454,
+ "theoretical_loss": 3.4464673913464363,
+ "tokens_seen": 2089091072
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029663390170511534,
+ "loss": 1.2182,
+ "theoretical_loss": 3.4464585277799236,
+ "tokens_seen": 2089156608
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029661785356068204,
+ "loss": 1.2263,
+ "theoretical_loss": 3.446449664569303,
+ "tokens_seen": 2089222144
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029660180541624874,
+ "loss": 1.2434,
+ "theoretical_loss": 3.44644080171455,
+ "tokens_seen": 2089287680
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029658575727181544,
+ "loss": 1.0842,
+ "theoretical_loss": 3.446431939215638,
+ "tokens_seen": 2089353216
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029656970912738214,
+ "loss": 1.2815,
+ "theoretical_loss": 3.4464230770725424,
+ "tokens_seen": 2089418752
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029655366098294884,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4464142152852366,
+ "tokens_seen": 2089484288
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029653761283851554,
+ "loss": 1.2542,
+ "theoretical_loss": 3.446405353853697,
+ "tokens_seen": 2089549824
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029652156469408224,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4463964927778963,
+ "tokens_seen": 2089615360
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029650551654964894,
+ "loss": 1.1499,
+ "theoretical_loss": 3.44638763205781,
+ "tokens_seen": 2089680896
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029648946840521564,
+ "loss": 1.2379,
+ "theoretical_loss": 3.446378771693413,
+ "tokens_seen": 2089746432
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029647342026078234,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4463699116846787,
+ "tokens_seen": 2089811968
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029645737211634904,
+ "loss": 1.2244,
+ "theoretical_loss": 3.446361052031583,
+ "tokens_seen": 2089877504
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029644132397191574,
+ "loss": 1.386,
+ "theoretical_loss": 3.4463521927340994,
+ "tokens_seen": 2089943040
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029642527582748245,
+ "loss": 1.3183,
+ "theoretical_loss": 3.4463433337922034,
+ "tokens_seen": 2090008576
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029640922768304915,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4463344752058687,
+ "tokens_seen": 2090074112
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029639317953861585,
+ "loss": 1.0941,
+ "theoretical_loss": 3.44632561697507,
+ "tokens_seen": 2090139648
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029637713139418255,
+ "loss": 1.2311,
+ "theoretical_loss": 3.4463167590997825,
+ "tokens_seen": 2090205184
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029636108324974925,
+ "loss": 1.2813,
+ "theoretical_loss": 3.4463079015799805,
+ "tokens_seen": 2090270720
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029634503510531595,
+ "loss": 1.2843,
+ "theoretical_loss": 3.446299044415639,
+ "tokens_seen": 2090336256
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029632898696088265,
+ "loss": 1.1622,
+ "theoretical_loss": 3.4462901876067313,
+ "tokens_seen": 2090401792
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029631293881644935,
+ "loss": 1.2731,
+ "theoretical_loss": 3.446281331153233,
+ "tokens_seen": 2090467328
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029629689067201605,
+ "loss": 1.3292,
+ "theoretical_loss": 3.446272475055119,
+ "tokens_seen": 2090532864
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 699756,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1224358081817627,
+ "objective/train/theoretical_loss": 3.446263619312363,
+ "objective/train/tokens_used": 2111058400,
+ "theoretical_loss": 3.446263619312363,
+ "tokens_seen": 2090598400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029628084252758275,
+ "loss": 1.2192,
+ "theoretical_loss": 3.446263619312363,
+ "tokens_seen": 2090598400
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029626479438314945,
+ "loss": 1.191,
+ "theoretical_loss": 3.44625476392494,
+ "tokens_seen": 2090663936
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029624874623871615,
+ "loss": 1.2315,
+ "theoretical_loss": 3.446245908892825,
+ "tokens_seen": 2090729472
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029623269809428285,
+ "loss": 1.2851,
+ "theoretical_loss": 3.4462370542159917,
+ "tokens_seen": 2090795008
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029621664994984955,
+ "loss": 1.2403,
+ "theoretical_loss": 3.4462281998944153,
+ "tokens_seen": 2090860544
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029620060180541625,
+ "loss": 1.1767,
+ "theoretical_loss": 3.4462193459280708,
+ "tokens_seen": 2090926080
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029618455366098295,
+ "loss": 1.2788,
+ "theoretical_loss": 3.446210492316932,
+ "tokens_seen": 2090991616
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029616850551654966,
+ "loss": 1.2252,
+ "theoretical_loss": 3.446201639060974,
+ "tokens_seen": 2091057152
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029615245737211636,
+ "loss": 1.1338,
+ "theoretical_loss": 3.4461927861601716,
+ "tokens_seen": 2091122688
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029613640922768306,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4461839336144986,
+ "tokens_seen": 2091188224
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029612036108324976,
+ "loss": 1.3678,
+ "theoretical_loss": 3.4461750814239305,
+ "tokens_seen": 2091253760
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029610431293881646,
+ "loss": 1.2843,
+ "theoretical_loss": 3.4461662295884414,
+ "tokens_seen": 2091319296
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029608826479438316,
+ "loss": 1.1918,
+ "theoretical_loss": 3.4461573781080066,
+ "tokens_seen": 2091384832
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029607221664994986,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4461485269825993,
+ "tokens_seen": 2091450368
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029605616850551656,
+ "loss": 1.2525,
+ "theoretical_loss": 3.446139676212196,
+ "tokens_seen": 2091515904
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029604012036108326,
+ "loss": 1.1929,
+ "theoretical_loss": 3.4461308257967698,
+ "tokens_seen": 2091581440
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029602407221664996,
+ "loss": 1.3232,
+ "theoretical_loss": 3.446121975736296,
+ "tokens_seen": 2091646976
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029600802407221666,
+ "loss": 1.3202,
+ "theoretical_loss": 3.44611312603075,
+ "tokens_seen": 2091712512
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029599197592778336,
+ "loss": 1.2232,
+ "theoretical_loss": 3.4461042766801047,
+ "tokens_seen": 2091778048
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029597592778335006,
+ "loss": 1.2833,
+ "theoretical_loss": 3.446095427684336,
+ "tokens_seen": 2091843584
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029595987963891676,
+ "loss": 1.2782,
+ "theoretical_loss": 3.4460865790434183,
+ "tokens_seen": 2091909120
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029594383149448346,
+ "loss": 1.1673,
+ "theoretical_loss": 3.4460777307573265,
+ "tokens_seen": 2091974656
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029592778335005016,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4460688828260344,
+ "tokens_seen": 2092040192
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029591173520561687,
+ "loss": 1.2642,
+ "theoretical_loss": 3.446060035249517,
+ "tokens_seen": 2092105728
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029589568706118357,
+ "loss": 1.286,
+ "theoretical_loss": 3.44605118802775,
+ "tokens_seen": 2092171264
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029587963891675027,
+ "loss": 1.1543,
+ "theoretical_loss": 3.446042341160707,
+ "tokens_seen": 2092236800
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029586359077231697,
+ "loss": 1.2497,
+ "theoretical_loss": 3.446033494648363,
+ "tokens_seen": 2092302336
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029584754262788367,
+ "loss": 1.3391,
+ "theoretical_loss": 3.4460246484906922,
+ "tokens_seen": 2092367872
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029583149448345037,
+ "loss": 1.1767,
+ "theoretical_loss": 3.44601580268767,
+ "tokens_seen": 2092433408
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029581544633901707,
+ "loss": 1.2437,
+ "theoretical_loss": 3.4460069572392706,
+ "tokens_seen": 2092498944
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029579939819458377,
+ "loss": 1.1901,
+ "theoretical_loss": 3.445998112145469,
+ "tokens_seen": 2092564480
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029578335005015047,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4459892674062393,
+ "tokens_seen": 2092630016
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029576730190571717,
+ "loss": 1.2451,
+ "theoretical_loss": 3.445980423021557,
+ "tokens_seen": 2092695552
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029575125376128387,
+ "loss": 1.2625,
+ "theoretical_loss": 3.445971578991396,
+ "tokens_seen": 2092761088
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029573520561685057,
+ "loss": 1.2259,
+ "theoretical_loss": 3.4459627353157316,
+ "tokens_seen": 2092826624
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002957191574724173,
+ "loss": 1.2279,
+ "theoretical_loss": 3.445953891994538,
+ "tokens_seen": 2092892160
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000295703109327984,
+ "loss": 1.2454,
+ "theoretical_loss": 3.44594504902779,
+ "tokens_seen": 2092957696
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956870611835507,
+ "loss": 1.2577,
+ "theoretical_loss": 3.4459362064154626,
+ "tokens_seen": 2093023232
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956710130391174,
+ "loss": 1.2545,
+ "theoretical_loss": 3.44592736415753,
+ "tokens_seen": 2093088768
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956549648946841,
+ "loss": 1.213,
+ "theoretical_loss": 3.445918522253968,
+ "tokens_seen": 2093154304
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956389167502508,
+ "loss": 1.2772,
+ "theoretical_loss": 3.4459096807047502,
+ "tokens_seen": 2093219840
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956228686058175,
+ "loss": 1.3243,
+ "theoretical_loss": 3.4459008395098514,
+ "tokens_seen": 2093285376
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002956068204613842,
+ "loss": 1.2862,
+ "theoretical_loss": 3.4458919986692464,
+ "tokens_seen": 2093350912
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002955907723169509,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4458831581829106,
+ "tokens_seen": 2093416448
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002955747241725176,
+ "loss": 1.1911,
+ "theoretical_loss": 3.445874318050818,
+ "tokens_seen": 2093481984
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002955586760280843,
+ "loss": 1.3106,
+ "theoretical_loss": 3.4458654782729434,
+ "tokens_seen": 2093547520
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000295542627883651,
+ "loss": 1.1755,
+ "theoretical_loss": 3.4458566388492615,
+ "tokens_seen": 2093613056
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002955265797392177,
+ "loss": 1.2403,
+ "theoretical_loss": 3.445847799779747,
+ "tokens_seen": 2093678592
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002955105315947844,
+ "loss": 1.1971,
+ "theoretical_loss": 3.445838961064375,
+ "tokens_seen": 2093744128
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954944834503511,
+ "loss": 1.315,
+ "theoretical_loss": 3.44583012270312,
+ "tokens_seen": 2093809664
+ },
+ {
+ "epoch": 0.63,
+ "objective/train/docs_used": 700782,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0766267776489258,
+ "objective/train/theoretical_loss": 3.4458212846959566,
+ "objective/train/tokens_used": 2114335200,
+ "theoretical_loss": 3.4458212846959566,
+ "tokens_seen": 2093875200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954784353059178,
+ "loss": 1.1144,
+ "theoretical_loss": 3.4458212846959566,
+ "tokens_seen": 2093875200
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954623871614845,
+ "loss": 1.1034,
+ "theoretical_loss": 3.4458124470428597,
+ "tokens_seen": 2093940736
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954463390170512,
+ "loss": 1.2526,
+ "theoretical_loss": 3.445803609743804,
+ "tokens_seen": 2094006272
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954302908726179,
+ "loss": 1.1599,
+ "theoretical_loss": 3.445794772798764,
+ "tokens_seen": 2094071808
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002954142427281846,
+ "loss": 1.1921,
+ "theoretical_loss": 3.445785936207715,
+ "tokens_seen": 2094137344
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002953981945837513,
+ "loss": 1.1896,
+ "theoretical_loss": 3.445777099970631,
+ "tokens_seen": 2094202880
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.000295382146439318,
+ "loss": 1.342,
+ "theoretical_loss": 3.445768264087487,
+ "tokens_seen": 2094268416
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002953660982948847,
+ "loss": 1.2053,
+ "theoretical_loss": 3.4457594285582585,
+ "tokens_seen": 2094333952
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002953500501504514,
+ "loss": 1.1623,
+ "theoretical_loss": 3.445750593382919,
+ "tokens_seen": 2094399488
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.0002953340020060181,
+ "loss": 1.339,
+ "theoretical_loss": 3.4457417585614447,
+ "tokens_seen": 2094465024
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029531795386158473,
+ "loss": 1.2398,
+ "theoretical_loss": 3.445732924093809,
+ "tokens_seen": 2094530560
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029530190571715144,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4457240899799872,
+ "tokens_seen": 2094596096
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029528585757271814,
+ "loss": 1.2589,
+ "theoretical_loss": 3.445715256219954,
+ "tokens_seen": 2094661632
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029526980942828484,
+ "loss": 1.3002,
+ "theoretical_loss": 3.445706422813684,
+ "tokens_seen": 2094727168
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029525376128385154,
+ "loss": 1.2714,
+ "theoretical_loss": 3.445697589761153,
+ "tokens_seen": 2094792704
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029523771313941824,
+ "loss": 1.2613,
+ "theoretical_loss": 3.4456887570623342,
+ "tokens_seen": 2094858240
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029522166499498494,
+ "loss": 1.3529,
+ "theoretical_loss": 3.4456799247172034,
+ "tokens_seen": 2094923776
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029520561685055164,
+ "loss": 1.2225,
+ "theoretical_loss": 3.445671092725735,
+ "tokens_seen": 2094989312
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029518956870611834,
+ "loss": 1.2477,
+ "theoretical_loss": 3.4456622610879037,
+ "tokens_seen": 2095054848
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029517352056168504,
+ "loss": 1.2897,
+ "theoretical_loss": 3.445653429803685,
+ "tokens_seen": 2095120384
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029515747241725174,
+ "loss": 1.205,
+ "theoretical_loss": 3.4456445988730526,
+ "tokens_seen": 2095185920
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029514142427281844,
+ "loss": 1.2753,
+ "theoretical_loss": 3.445635768295982,
+ "tokens_seen": 2095251456
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029512537612838514,
+ "loss": 1.252,
+ "theoretical_loss": 3.445626938072448,
+ "tokens_seen": 2095316992
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029510932798395184,
+ "loss": 1.1538,
+ "theoretical_loss": 3.4456181082024253,
+ "tokens_seen": 2095382528
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 0.00029509327983951854,
+ "loss": 1.2913,
+ "theoretical_loss": 3.445609278685888,
+ "tokens_seen": 2095448064
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029507723169508524,
+ "loss": 1.2468,
+ "theoretical_loss": 3.4456004495228116,
+ "tokens_seen": 2095513600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029506118355065194,
+ "loss": 1.2389,
+ "theoretical_loss": 3.445591620713171,
+ "tokens_seen": 2095579136
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029504513540621865,
+ "loss": 1.2258,
+ "theoretical_loss": 3.445582792256941,
+ "tokens_seen": 2095644672
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029502908726178535,
+ "loss": 1.2837,
+ "theoretical_loss": 3.4455739641540957,
+ "tokens_seen": 2095710208
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029501303911735205,
+ "loss": 1.2303,
+ "theoretical_loss": 3.445565136404611,
+ "tokens_seen": 2095775744
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029499699097291875,
+ "loss": 1.2866,
+ "theoretical_loss": 3.4455563090084604,
+ "tokens_seen": 2095841280
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029498094282848545,
+ "loss": 1.3051,
+ "theoretical_loss": 3.4455474819656198,
+ "tokens_seen": 2095906816
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029496489468405215,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4455386552760636,
+ "tokens_seen": 2095972352
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029494884653961885,
+ "loss": 1.2218,
+ "theoretical_loss": 3.4455298289397662,
+ "tokens_seen": 2096037888
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029493279839518555,
+ "loss": 1.1865,
+ "theoretical_loss": 3.4455210029567036,
+ "tokens_seen": 2096103424
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029491675025075225,
+ "loss": 1.3202,
+ "theoretical_loss": 3.445512177326849,
+ "tokens_seen": 2096168960
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029490070210631895,
+ "loss": 1.2313,
+ "theoretical_loss": 3.4455033520501788,
+ "tokens_seen": 2096234496
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948846539618857,
+ "loss": 1.1672,
+ "theoretical_loss": 3.445494527126667,
+ "tokens_seen": 2096300032
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948686058174524,
+ "loss": 1.1828,
+ "theoretical_loss": 3.4454857025562884,
+ "tokens_seen": 2096365568
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948525576730191,
+ "loss": 1.3516,
+ "theoretical_loss": 3.4454768783390177,
+ "tokens_seen": 2096431104
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948365095285858,
+ "loss": 1.2632,
+ "theoretical_loss": 3.4454680544748304,
+ "tokens_seen": 2096496640
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948204613841525,
+ "loss": 1.2106,
+ "theoretical_loss": 3.445459230963701,
+ "tokens_seen": 2096562176
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002948044132397192,
+ "loss": 1.2206,
+ "theoretical_loss": 3.4454504078056036,
+ "tokens_seen": 2096627712
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002947883650952859,
+ "loss": 1.2807,
+ "theoretical_loss": 3.4454415850005145,
+ "tokens_seen": 2096693248
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002947723169508526,
+ "loss": 1.1625,
+ "theoretical_loss": 3.4454327625484074,
+ "tokens_seen": 2096758784
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002947562688064193,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4454239404492575,
+ "tokens_seen": 2096824320
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000294740220661986,
+ "loss": 1.2591,
+ "theoretical_loss": 3.44541511870304,
+ "tokens_seen": 2096889856
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002947241725175527,
+ "loss": 1.3978,
+ "theoretical_loss": 3.445406297309729,
+ "tokens_seen": 2096955392
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002947081243731194,
+ "loss": 1.1998,
+ "theoretical_loss": 3.4453974762692994,
+ "tokens_seen": 2097020928
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002946920762286861,
+ "loss": 1.3152,
+ "theoretical_loss": 3.445388655581727,
+ "tokens_seen": 2097086464
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 702025,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.190444827079773,
+ "objective/train/theoretical_loss": 3.445379835246986,
+ "objective/train/tokens_used": 2117612000,
+ "theoretical_loss": 3.445379835246986,
+ "tokens_seen": 2097152000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029467602808425276,
+ "loss": 1.2753,
+ "theoretical_loss": 3.445379835246986,
+ "tokens_seen": 2097152000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029465997993981946,
+ "loss": 1.3138,
+ "theoretical_loss": 3.445371015265051,
+ "tokens_seen": 2097217536
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029464393179538616,
+ "loss": 1.249,
+ "theoretical_loss": 3.445362195635898,
+ "tokens_seen": 2097283072
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029462788365095286,
+ "loss": 1.2635,
+ "theoretical_loss": 3.4453533763595003,
+ "tokens_seen": 2097348608
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029461183550651956,
+ "loss": 1.2913,
+ "theoretical_loss": 3.445344557435834,
+ "tokens_seen": 2097414144
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029459578736208626,
+ "loss": 1.2187,
+ "theoretical_loss": 3.4453357388648733,
+ "tokens_seen": 2097479680
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029457973921765296,
+ "loss": 1.266,
+ "theoretical_loss": 3.4453269206465933,
+ "tokens_seen": 2097545216
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029456369107321966,
+ "loss": 1.2703,
+ "theoretical_loss": 3.4453181027809685,
+ "tokens_seen": 2097610752
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029454764292878637,
+ "loss": 1.245,
+ "theoretical_loss": 3.4453092852679745,
+ "tokens_seen": 2097676288
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029453159478435307,
+ "loss": 1.2037,
+ "theoretical_loss": 3.445300468107586,
+ "tokens_seen": 2097741824
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029451554663991977,
+ "loss": 1.315,
+ "theoretical_loss": 3.4452916512997773,
+ "tokens_seen": 2097807360
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029449949849548647,
+ "loss": 1.2049,
+ "theoretical_loss": 3.445282834844524,
+ "tokens_seen": 2097872896
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029448345035105317,
+ "loss": 1.1896,
+ "theoretical_loss": 3.4452740187418005,
+ "tokens_seen": 2097938432
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029446740220661987,
+ "loss": 1.2491,
+ "theoretical_loss": 3.445265202991582,
+ "tokens_seen": 2098003968
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029445135406218657,
+ "loss": 1.2117,
+ "theoretical_loss": 3.445256387593843,
+ "tokens_seen": 2098069504
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029443530591775327,
+ "loss": 1.2746,
+ "theoretical_loss": 3.445247572548559,
+ "tokens_seen": 2098135040
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029441925777331997,
+ "loss": 1.2843,
+ "theoretical_loss": 3.445238757855704,
+ "tokens_seen": 2098200576
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029440320962888667,
+ "loss": 1.2633,
+ "theoretical_loss": 3.445229943515254,
+ "tokens_seen": 2098266112
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029438716148445337,
+ "loss": 1.2815,
+ "theoretical_loss": 3.445221129527183,
+ "tokens_seen": 2098331648
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029437111334002007,
+ "loss": 1.2502,
+ "theoretical_loss": 3.4452123158914665,
+ "tokens_seen": 2098397184
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029435506519558677,
+ "loss": 1.3069,
+ "theoretical_loss": 3.4452035026080794,
+ "tokens_seen": 2098462720
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002943390170511535,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4451946896769963,
+ "tokens_seen": 2098528256
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002943229689067202,
+ "loss": 1.2281,
+ "theoretical_loss": 3.445185877098192,
+ "tokens_seen": 2098593792
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002943069207622869,
+ "loss": 1.2486,
+ "theoretical_loss": 3.4451770648716415,
+ "tokens_seen": 2098659328
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002942908726178536,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4451682529973198,
+ "tokens_seen": 2098724864
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002942748244734203,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4451594414752025,
+ "tokens_seen": 2098790400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000294258776328987,
+ "loss": 1.1652,
+ "theoretical_loss": 3.4451506303052635,
+ "tokens_seen": 2098855936
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002942427281845537,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4451418194874783,
+ "tokens_seen": 2098921472
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002942266800401204,
+ "loss": 1.4216,
+ "theoretical_loss": 3.4451330090218217,
+ "tokens_seen": 2098987008
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002942106318956871,
+ "loss": 1.3034,
+ "theoretical_loss": 3.4451241989082684,
+ "tokens_seen": 2099052544
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941945837512538,
+ "loss": 1.2486,
+ "theoretical_loss": 3.4451153891467934,
+ "tokens_seen": 2099118080
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941785356068205,
+ "loss": 1.2809,
+ "theoretical_loss": 3.445106579737372,
+ "tokens_seen": 2099183616
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941624874623872,
+ "loss": 1.2644,
+ "theoretical_loss": 3.4450977706799786,
+ "tokens_seen": 2099249152
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941464393179539,
+ "loss": 1.2968,
+ "theoretical_loss": 3.445088961974589,
+ "tokens_seen": 2099314688
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941303911735206,
+ "loss": 1.3076,
+ "theoretical_loss": 3.445080153621177,
+ "tokens_seen": 2099380224
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002941143430290873,
+ "loss": 1.2414,
+ "theoretical_loss": 3.4450713456197186,
+ "tokens_seen": 2099445760
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000294098294884654,
+ "loss": 1.2998,
+ "theoretical_loss": 3.445062537970188,
+ "tokens_seen": 2099511296
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940822467402207,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4450537306725604,
+ "tokens_seen": 2099576832
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940661985957874,
+ "loss": 1.3093,
+ "theoretical_loss": 3.4450449237268113,
+ "tokens_seen": 2099642368
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940501504513541,
+ "loss": 1.3336,
+ "theoretical_loss": 3.445036117132915,
+ "tokens_seen": 2099707904
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940341023069208,
+ "loss": 1.1623,
+ "theoretical_loss": 3.4450273108908465,
+ "tokens_seen": 2099773440
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940180541624875,
+ "loss": 1.3386,
+ "theoretical_loss": 3.4450185050005806,
+ "tokens_seen": 2099838976
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002940020060180542,
+ "loss": 1.2946,
+ "theoretical_loss": 3.445009699462093,
+ "tokens_seen": 2099904512
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002939859578736209,
+ "loss": 1.2569,
+ "theoretical_loss": 3.445000894275358,
+ "tokens_seen": 2099970048
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002939699097291876,
+ "loss": 1.3297,
+ "theoretical_loss": 3.444992089440351,
+ "tokens_seen": 2100035584
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002939538615847543,
+ "loss": 1.2595,
+ "theoretical_loss": 3.4449832849570465,
+ "tokens_seen": 2100101120
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000293937813440321,
+ "loss": 1.2265,
+ "theoretical_loss": 3.44497448082542,
+ "tokens_seen": 2100166656
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002939217652958877,
+ "loss": 1.3455,
+ "theoretical_loss": 3.444965677045446,
+ "tokens_seen": 2100232192
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029390571715145434,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4449568736171,
+ "tokens_seen": 2100297728
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029388966900702104,
+ "loss": 1.201,
+ "theoretical_loss": 3.444948070540357,
+ "tokens_seen": 2100363264
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 702928,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3683278560638428,
+ "objective/train/theoretical_loss": 3.4449392678151907,
+ "objective/train/tokens_used": 2120888800,
+ "theoretical_loss": 3.4449392678151907,
+ "tokens_seen": 2100428800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029387362086258774,
+ "loss": 1.3309,
+ "theoretical_loss": 3.4449392678151907,
+ "tokens_seen": 2100428800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029385757271815444,
+ "loss": 1.3092,
+ "theoretical_loss": 3.4449304654415775,
+ "tokens_seen": 2100494336
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029384152457372114,
+ "loss": 1.2743,
+ "theoretical_loss": 3.4449216634194926,
+ "tokens_seen": 2100559872
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029382547642928784,
+ "loss": 1.3148,
+ "theoretical_loss": 3.4449128617489095,
+ "tokens_seen": 2100625408
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029380942828485454,
+ "loss": 1.2783,
+ "theoretical_loss": 3.4449040604298045,
+ "tokens_seen": 2100690944
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029379338014042124,
+ "loss": 1.3382,
+ "theoretical_loss": 3.444895259462152,
+ "tokens_seen": 2100756480
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029377733199598794,
+ "loss": 1.1861,
+ "theoretical_loss": 3.444886458845927,
+ "tokens_seen": 2100822016
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029376128385155464,
+ "loss": 1.2768,
+ "theoretical_loss": 3.444877658581105,
+ "tokens_seen": 2100887552
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029374523570712134,
+ "loss": 1.2145,
+ "theoretical_loss": 3.4448688586676606,
+ "tokens_seen": 2100953088
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029372918756268804,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4448600591055687,
+ "tokens_seen": 2101018624
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029371313941825474,
+ "loss": 1.308,
+ "theoretical_loss": 3.444851259894805,
+ "tokens_seen": 2101084160
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029369709127382144,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4448424610353436,
+ "tokens_seen": 2101149696
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002936810431293882,
+ "loss": 1.2907,
+ "theoretical_loss": 3.44483366252716,
+ "tokens_seen": 2101215232
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002936649949849549,
+ "loss": 1.2855,
+ "theoretical_loss": 3.444824864370229,
+ "tokens_seen": 2101280768
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002936489468405216,
+ "loss": 1.2564,
+ "theoretical_loss": 3.4448160665645258,
+ "tokens_seen": 2101346304
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002936328986960883,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4448072691100258,
+ "tokens_seen": 2101411840
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000293616850551655,
+ "loss": 1.2523,
+ "theoretical_loss": 3.444798472006703,
+ "tokens_seen": 2101477376
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002936008024072217,
+ "loss": 1.3142,
+ "theoretical_loss": 3.4447896752545333,
+ "tokens_seen": 2101542912
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935847542627884,
+ "loss": 1.2352,
+ "theoretical_loss": 3.444780878853492,
+ "tokens_seen": 2101608448
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935687061183551,
+ "loss": 1.3131,
+ "theoretical_loss": 3.444772082803553,
+ "tokens_seen": 2101673984
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935526579739218,
+ "loss": 1.2058,
+ "theoretical_loss": 3.444763287104692,
+ "tokens_seen": 2101739520
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935366098294885,
+ "loss": 1.3306,
+ "theoretical_loss": 3.4447544917568846,
+ "tokens_seen": 2101805056
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935205616850552,
+ "loss": 1.3626,
+ "theoretical_loss": 3.4447456967601044,
+ "tokens_seen": 2101870592
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002935045135406219,
+ "loss": 1.2997,
+ "theoretical_loss": 3.444736902114328,
+ "tokens_seen": 2101936128
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002934884653961886,
+ "loss": 1.1453,
+ "theoretical_loss": 3.4447281078195293,
+ "tokens_seen": 2102001664
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002934724172517553,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4447193138756838,
+ "tokens_seen": 2102067200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000293456369107322,
+ "loss": 1.2993,
+ "theoretical_loss": 3.444710520282767,
+ "tokens_seen": 2102132736
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002934403209628887,
+ "loss": 1.3058,
+ "theoretical_loss": 3.444701727040753,
+ "tokens_seen": 2102198272
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002934242728184554,
+ "loss": 1.3147,
+ "theoretical_loss": 3.4446929341496175,
+ "tokens_seen": 2102263808
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002934082246740221,
+ "loss": 1.207,
+ "theoretical_loss": 3.4446841416093354,
+ "tokens_seen": 2102329344
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933921765295888,
+ "loss": 1.2134,
+ "theoretical_loss": 3.444675349419882,
+ "tokens_seen": 2102394880
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933761283851555,
+ "loss": 1.3701,
+ "theoretical_loss": 3.444666557581232,
+ "tokens_seen": 2102460416
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933600802407222,
+ "loss": 1.1788,
+ "theoretical_loss": 3.4446577660933606,
+ "tokens_seen": 2102525952
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933440320962889,
+ "loss": 1.265,
+ "theoretical_loss": 3.444648974956243,
+ "tokens_seen": 2102591488
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933279839518556,
+ "loss": 1.168,
+ "theoretical_loss": 3.444640184169854,
+ "tokens_seen": 2102657024
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002933119358074223,
+ "loss": 1.2341,
+ "theoretical_loss": 3.444631393734169,
+ "tokens_seen": 2102722560
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000293295887662989,
+ "loss": 1.2968,
+ "theoretical_loss": 3.444622603649163,
+ "tokens_seen": 2102788096
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029327983951855566,
+ "loss": 1.3049,
+ "theoretical_loss": 3.4446138139148106,
+ "tokens_seen": 2102853632
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029326379137412236,
+ "loss": 1.2331,
+ "theoretical_loss": 3.4446050245310875,
+ "tokens_seen": 2102919168
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029324774322968906,
+ "loss": 1.2285,
+ "theoretical_loss": 3.444596235497969,
+ "tokens_seen": 2102984704
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029323169508525576,
+ "loss": 1.3213,
+ "theoretical_loss": 3.4445874468154294,
+ "tokens_seen": 2103050240
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029321564694082246,
+ "loss": 1.2382,
+ "theoretical_loss": 3.444578658483444,
+ "tokens_seen": 2103115776
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029319959879638916,
+ "loss": 1.3081,
+ "theoretical_loss": 3.4445698705019883,
+ "tokens_seen": 2103181312
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029318355065195586,
+ "loss": 1.1937,
+ "theoretical_loss": 3.4445610828710373,
+ "tokens_seen": 2103246848
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029316750250752257,
+ "loss": 1.3231,
+ "theoretical_loss": 3.4445522955905656,
+ "tokens_seen": 2103312384
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029315145436308927,
+ "loss": 1.2709,
+ "theoretical_loss": 3.4445435086605487,
+ "tokens_seen": 2103377920
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029313540621865597,
+ "loss": 1.1468,
+ "theoretical_loss": 3.4445347220809617,
+ "tokens_seen": 2103443456
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029311935807422267,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4445259358517797,
+ "tokens_seen": 2103508992
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029310330992978937,
+ "loss": 1.2432,
+ "theoretical_loss": 3.444517149972978,
+ "tokens_seen": 2103574528
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029308726178535607,
+ "loss": 1.2176,
+ "theoretical_loss": 3.444508364444531,
+ "tokens_seen": 2103640064
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 704157,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2861976623535156,
+ "objective/train/theoretical_loss": 3.4444995792664144,
+ "objective/train/tokens_used": 2124165600,
+ "theoretical_loss": 3.4444995792664144,
+ "tokens_seen": 2103705600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029307121364092277,
+ "loss": 1.19,
+ "theoretical_loss": 3.4444995792664144,
+ "tokens_seen": 2103705600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029305516549648947,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4444907944386034,
+ "tokens_seen": 2103771136
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029303911735205617,
+ "loss": 1.2838,
+ "theoretical_loss": 3.4444820099610727,
+ "tokens_seen": 2103836672
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029302306920762287,
+ "loss": 1.2223,
+ "theoretical_loss": 3.4444732258337982,
+ "tokens_seen": 2103902208
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029300702106318957,
+ "loss": 1.3129,
+ "theoretical_loss": 3.444464442056754,
+ "tokens_seen": 2103967744
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029299097291875627,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4444556586299155,
+ "tokens_seen": 2104033280
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000292974924774323,
+ "loss": 1.2356,
+ "theoretical_loss": 3.4444468755532585,
+ "tokens_seen": 2104098816
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002929588766298897,
+ "loss": 1.2547,
+ "theoretical_loss": 3.4444380928267577,
+ "tokens_seen": 2104164352
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002929428284854564,
+ "loss": 1.2811,
+ "theoretical_loss": 3.4444293104503876,
+ "tokens_seen": 2104229888
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002929267803410231,
+ "loss": 1.3758,
+ "theoretical_loss": 3.444420528424125,
+ "tokens_seen": 2104295424
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002929107321965898,
+ "loss": 1.3592,
+ "theoretical_loss": 3.444411746747943,
+ "tokens_seen": 2104360960
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002928946840521565,
+ "loss": 1.2085,
+ "theoretical_loss": 3.4444029654218182,
+ "tokens_seen": 2104426496
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002928786359077232,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4443941844457253,
+ "tokens_seen": 2104492032
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002928625877632899,
+ "loss": 1.2905,
+ "theoretical_loss": 3.444385403819639,
+ "tokens_seen": 2104557568
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002928465396188566,
+ "loss": 1.2848,
+ "theoretical_loss": 3.444376623543535,
+ "tokens_seen": 2104623104
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002928304914744233,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4443678436173886,
+ "tokens_seen": 2104688640
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029281444332999,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4443590640411745,
+ "tokens_seen": 2104754176
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927983951855567,
+ "loss": 1.2977,
+ "theoretical_loss": 3.444350284814868,
+ "tokens_seen": 2104819712
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927823470411234,
+ "loss": 1.2106,
+ "theoretical_loss": 3.4443415059384446,
+ "tokens_seen": 2104885248
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927662988966901,
+ "loss": 1.2026,
+ "theoretical_loss": 3.444332727411879,
+ "tokens_seen": 2104950784
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927502507522568,
+ "loss": 1.228,
+ "theoretical_loss": 3.4443239492351463,
+ "tokens_seen": 2105016320
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927342026078235,
+ "loss": 1.2969,
+ "theoretical_loss": 3.4443151714082223,
+ "tokens_seen": 2105081856
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927181544633902,
+ "loss": 1.1926,
+ "theoretical_loss": 3.4443063939310816,
+ "tokens_seen": 2105147392
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002927021063189569,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4442976168036994,
+ "tokens_seen": 2105212928
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002926860581745236,
+ "loss": 1.165,
+ "theoretical_loss": 3.4442888400260507,
+ "tokens_seen": 2105278464
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002926700100300903,
+ "loss": 1.2043,
+ "theoretical_loss": 3.4442800635981117,
+ "tokens_seen": 2105344000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000292653961885657,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4442712875198565,
+ "tokens_seen": 2105409536
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002926379137412237,
+ "loss": 1.2781,
+ "theoretical_loss": 3.44426251179126,
+ "tokens_seen": 2105475072
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002926218655967904,
+ "loss": 1.2227,
+ "theoretical_loss": 3.444253736412299,
+ "tokens_seen": 2105540608
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002926058174523571,
+ "loss": 1.2487,
+ "theoretical_loss": 3.444244961382948,
+ "tokens_seen": 2105606144
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002925897693079238,
+ "loss": 1.303,
+ "theoretical_loss": 3.444236186703181,
+ "tokens_seen": 2105671680
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002925737211634905,
+ "loss": 1.287,
+ "theoretical_loss": 3.4442274123729746,
+ "tokens_seen": 2105737216
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002925576730190572,
+ "loss": 1.2965,
+ "theoretical_loss": 3.4442186383923032,
+ "tokens_seen": 2105802752
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002925416248746239,
+ "loss": 1.1241,
+ "theoretical_loss": 3.4442098647611425,
+ "tokens_seen": 2105868288
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002925255767301906,
+ "loss": 1.3987,
+ "theoretical_loss": 3.444201091479467,
+ "tokens_seen": 2105933824
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029250952858575724,
+ "loss": 1.2754,
+ "theoretical_loss": 3.4441923185472536,
+ "tokens_seen": 2105999360
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029249348044132394,
+ "loss": 1.2785,
+ "theoretical_loss": 3.444183545964475,
+ "tokens_seen": 2106064896
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002924774322968907,
+ "loss": 1.2474,
+ "theoretical_loss": 3.444174773731109,
+ "tokens_seen": 2106130432
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002924613841524574,
+ "loss": 1.2127,
+ "theoretical_loss": 3.4441660018471287,
+ "tokens_seen": 2106195968
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002924453360080241,
+ "loss": 1.2591,
+ "theoretical_loss": 3.44415723031251,
+ "tokens_seen": 2106261504
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002924292878635908,
+ "loss": 1.3267,
+ "theoretical_loss": 3.444148459127229,
+ "tokens_seen": 2106327040
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002924132397191575,
+ "loss": 1.2089,
+ "theoretical_loss": 3.4441396882912594,
+ "tokens_seen": 2106392576
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923971915747242,
+ "loss": 1.2744,
+ "theoretical_loss": 3.4441309178045776,
+ "tokens_seen": 2106458112
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923811434302909,
+ "loss": 1.2721,
+ "theoretical_loss": 3.4441221476671586,
+ "tokens_seen": 2106523648
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923650952858576,
+ "loss": 1.2363,
+ "theoretical_loss": 3.444113377878977,
+ "tokens_seen": 2106589184
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923490471414243,
+ "loss": 1.248,
+ "theoretical_loss": 3.444104608440009,
+ "tokens_seen": 2106654720
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000292332998996991,
+ "loss": 1.1822,
+ "theoretical_loss": 3.4440958393502292,
+ "tokens_seen": 2106720256
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923169508525577,
+ "loss": 1.2854,
+ "theoretical_loss": 3.444087070609613,
+ "tokens_seen": 2106785792
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002923009027081244,
+ "loss": 1.265,
+ "theoretical_loss": 3.444078302218135,
+ "tokens_seen": 2106851328
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922848545636911,
+ "loss": 1.2895,
+ "theoretical_loss": 3.444069534175772,
+ "tokens_seen": 2106916864
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 705118,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3771543502807617,
+ "objective/train/theoretical_loss": 3.4440607664824974,
+ "objective/train/tokens_used": 2127442400,
+ "theoretical_loss": 3.4440607664824974,
+ "tokens_seen": 2106982400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922688064192578,
+ "loss": 1.3712,
+ "theoretical_loss": 3.4440607664824974,
+ "tokens_seen": 2106982400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922527582748245,
+ "loss": 1.2341,
+ "theoretical_loss": 3.4440519991382876,
+ "tokens_seen": 2107047936
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922367101303912,
+ "loss": 1.2584,
+ "theoretical_loss": 3.4440432321431174,
+ "tokens_seen": 2107113472
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922206619859579,
+ "loss": 1.2007,
+ "theoretical_loss": 3.444034465496962,
+ "tokens_seen": 2107179008
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002922046138415246,
+ "loss": 1.2798,
+ "theoretical_loss": 3.4440256991997975,
+ "tokens_seen": 2107244544
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002921885656970913,
+ "loss": 1.227,
+ "theoretical_loss": 3.444016933251598,
+ "tokens_seen": 2107310080
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000292172517552658,
+ "loss": 1.3767,
+ "theoretical_loss": 3.4440081676523397,
+ "tokens_seen": 2107375616
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002921564694082247,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4439994024019973,
+ "tokens_seen": 2107441152
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002921404212637914,
+ "loss": 1.2753,
+ "theoretical_loss": 3.443990637500546,
+ "tokens_seen": 2107506688
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002921243731193581,
+ "loss": 1.2253,
+ "theoretical_loss": 3.4439818729479614,
+ "tokens_seen": 2107572224
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002921083249749248,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4439731087442187,
+ "tokens_seen": 2107637760
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002920922768304915,
+ "loss": 1.2582,
+ "theoretical_loss": 3.443964344889293,
+ "tokens_seen": 2107703296
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002920762286860582,
+ "loss": 1.1671,
+ "theoretical_loss": 3.4439555813831593,
+ "tokens_seen": 2107768832
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002920601805416249,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4439468182257937,
+ "tokens_seen": 2107834368
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002920441323971916,
+ "loss": 1.2867,
+ "theoretical_loss": 3.443938055417171,
+ "tokens_seen": 2107899904
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002920280842527583,
+ "loss": 1.2575,
+ "theoretical_loss": 3.4439292929572662,
+ "tokens_seen": 2107965440
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000292012036108325,
+ "loss": 1.1907,
+ "theoretical_loss": 3.4439205308460554,
+ "tokens_seen": 2108030976
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919959879638917,
+ "loss": 1.29,
+ "theoretical_loss": 3.443911769083513,
+ "tokens_seen": 2108096512
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919799398194584,
+ "loss": 1.3228,
+ "theoretical_loss": 3.4439030076696144,
+ "tokens_seen": 2108162048
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919638916750251,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4438942466043354,
+ "tokens_seen": 2108227584
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919478435305918,
+ "loss": 1.2715,
+ "theoretical_loss": 3.443885485887651,
+ "tokens_seen": 2108293120
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919317953861585,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4438767255195364,
+ "tokens_seen": 2108358656
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002919157472417252,
+ "loss": 1.2612,
+ "theoretical_loss": 3.443867965499967,
+ "tokens_seen": 2108424192
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002918996990972919,
+ "loss": 1.2924,
+ "theoretical_loss": 3.4438592058289186,
+ "tokens_seen": 2108489728
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002918836509528586,
+ "loss": 1.1931,
+ "theoretical_loss": 3.4438504465063655,
+ "tokens_seen": 2108555264
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029186760280842526,
+ "loss": 1.3789,
+ "theoretical_loss": 3.4438416875322835,
+ "tokens_seen": 2108620800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029185155466399196,
+ "loss": 1.2194,
+ "theoretical_loss": 3.443832928906648,
+ "tokens_seen": 2108686336
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029183550651955866,
+ "loss": 1.2794,
+ "theoretical_loss": 3.4438241706294344,
+ "tokens_seen": 2108751872
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029181945837512536,
+ "loss": 1.1751,
+ "theoretical_loss": 3.443815412700618,
+ "tokens_seen": 2108817408
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029180341023069207,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4438066551201736,
+ "tokens_seen": 2108882944
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029178736208625877,
+ "loss": 1.2454,
+ "theoretical_loss": 3.443797897888077,
+ "tokens_seen": 2108948480
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029177131394182547,
+ "loss": 1.3378,
+ "theoretical_loss": 3.4437891410043036,
+ "tokens_seen": 2109014016
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029175526579739217,
+ "loss": 1.3135,
+ "theoretical_loss": 3.4437803844688286,
+ "tokens_seen": 2109079552
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029173921765295887,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4437716282816266,
+ "tokens_seen": 2109145088
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029172316950852557,
+ "loss": 1.2716,
+ "theoretical_loss": 3.443762872442674,
+ "tokens_seen": 2109210624
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029170712136409227,
+ "loss": 1.3249,
+ "theoretical_loss": 3.443754116951946,
+ "tokens_seen": 2109276160
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029169107321965897,
+ "loss": 1.2818,
+ "theoretical_loss": 3.443745361809417,
+ "tokens_seen": 2109341696
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029167502507522567,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4437366070150635,
+ "tokens_seen": 2109407232
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029165897693079237,
+ "loss": 1.282,
+ "theoretical_loss": 3.44372785256886,
+ "tokens_seen": 2109472768
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029164292878635907,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4437190984707824,
+ "tokens_seen": 2109538304
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029162688064192577,
+ "loss": 1.2352,
+ "theoretical_loss": 3.4437103447208055,
+ "tokens_seen": 2109603840
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002916108324974925,
+ "loss": 1.2879,
+ "theoretical_loss": 3.443701591318905,
+ "tokens_seen": 2109669376
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002915947843530592,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4436928382650565,
+ "tokens_seen": 2109734912
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002915787362086259,
+ "loss": 1.1082,
+ "theoretical_loss": 3.4436840855592346,
+ "tokens_seen": 2109800448
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002915626880641926,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4436753332014156,
+ "tokens_seen": 2109865984
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002915466399197593,
+ "loss": 1.3113,
+ "theoretical_loss": 3.4436665811915734,
+ "tokens_seen": 2109931520
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000291530591775326,
+ "loss": 1.2668,
+ "theoretical_loss": 3.443657829529685,
+ "tokens_seen": 2109997056
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002915145436308927,
+ "loss": 1.3158,
+ "theoretical_loss": 3.443649078215725,
+ "tokens_seen": 2110062592
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914984954864594,
+ "loss": 1.1845,
+ "theoretical_loss": 3.443640327249669,
+ "tokens_seen": 2110128128
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914824473420261,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4436315766314918,
+ "tokens_seen": 2110193664
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 706481,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1775219440460205,
+ "objective/train/theoretical_loss": 3.4436228263611692,
+ "objective/train/tokens_used": 2130719200,
+ "theoretical_loss": 3.4436228263611692,
+ "tokens_seen": 2110259200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914663991975928,
+ "loss": 1.2052,
+ "theoretical_loss": 3.4436228263611692,
+ "tokens_seen": 2110259200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914503510531595,
+ "loss": 1.2148,
+ "theoretical_loss": 3.443614076438677,
+ "tokens_seen": 2110324736
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914343029087262,
+ "loss": 1.2983,
+ "theoretical_loss": 3.4436053268639895,
+ "tokens_seen": 2110390272
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914182547642929,
+ "loss": 1.1328,
+ "theoretical_loss": 3.443596577637083,
+ "tokens_seen": 2110455808
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002914022066198596,
+ "loss": 1.2008,
+ "theoretical_loss": 3.4435878287579325,
+ "tokens_seen": 2110521344
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002913861584754263,
+ "loss": 1.3581,
+ "theoretical_loss": 3.443579080226513,
+ "tokens_seen": 2110586880
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000291370110330993,
+ "loss": 1.2269,
+ "theoretical_loss": 3.4435703320428006,
+ "tokens_seen": 2110652416
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002913540621865597,
+ "loss": 1.3272,
+ "theoretical_loss": 3.443561584206771,
+ "tokens_seen": 2110717952
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002913380140421264,
+ "loss": 1.2745,
+ "theoretical_loss": 3.443552836718398,
+ "tokens_seen": 2110783488
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002913219658976931,
+ "loss": 1.3242,
+ "theoretical_loss": 3.4435440895776583,
+ "tokens_seen": 2110849024
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002913059177532598,
+ "loss": 1.284,
+ "theoretical_loss": 3.443535342784527,
+ "tokens_seen": 2110914560
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029128986960882654,
+ "loss": 1.2567,
+ "theoretical_loss": 3.44352659633898,
+ "tokens_seen": 2110980096
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029127382146439324,
+ "loss": 1.2878,
+ "theoretical_loss": 3.4435178502409913,
+ "tokens_seen": 2111045632
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029125777331995994,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4435091044905377,
+ "tokens_seen": 2111111168
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002912417251755266,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4435003590875937,
+ "tokens_seen": 2111176704
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002912256770310933,
+ "loss": 1.2432,
+ "theoretical_loss": 3.443491614032135,
+ "tokens_seen": 2111242240
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029120962888666,
+ "loss": 1.2587,
+ "theoretical_loss": 3.4434828693241375,
+ "tokens_seen": 2111307776
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911935807422267,
+ "loss": 1.2997,
+ "theoretical_loss": 3.443474124963576,
+ "tokens_seen": 2111373312
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911775325977934,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4434653809504265,
+ "tokens_seen": 2111438848
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911614844533601,
+ "loss": 1.2491,
+ "theoretical_loss": 3.4434566372846636,
+ "tokens_seen": 2111504384
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911454363089268,
+ "loss": 1.2458,
+ "theoretical_loss": 3.4434478939662627,
+ "tokens_seen": 2111569920
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911293881644935,
+ "loss": 1.3273,
+ "theoretical_loss": 3.4434391509952,
+ "tokens_seen": 2111635456
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002911133400200602,
+ "loss": 1.2392,
+ "theoretical_loss": 3.4434304083714506,
+ "tokens_seen": 2111700992
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910972918756269,
+ "loss": 1.366,
+ "theoretical_loss": 3.4434216660949897,
+ "tokens_seen": 2111766528
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910812437311936,
+ "loss": 1.2782,
+ "theoretical_loss": 3.443412924165793,
+ "tokens_seen": 2111832064
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910651955867603,
+ "loss": 1.1609,
+ "theoretical_loss": 3.443404182583836,
+ "tokens_seen": 2111897600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000291049147442327,
+ "loss": 1.31,
+ "theoretical_loss": 3.4433954413490944,
+ "tokens_seen": 2111963136
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910330992978937,
+ "loss": 1.342,
+ "theoretical_loss": 3.4433867004615424,
+ "tokens_seen": 2112028672
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910170511534604,
+ "loss": 1.1764,
+ "theoretical_loss": 3.4433779599211567,
+ "tokens_seen": 2112094208
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002910010030090271,
+ "loss": 1.387,
+ "theoretical_loss": 3.443369219727912,
+ "tokens_seen": 2112159744
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909849548645938,
+ "loss": 1.2732,
+ "theoretical_loss": 3.4433604798817843,
+ "tokens_seen": 2112225280
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909689067201605,
+ "loss": 1.3549,
+ "theoretical_loss": 3.4433517403827487,
+ "tokens_seen": 2112290816
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909528585757272,
+ "loss": 1.2359,
+ "theoretical_loss": 3.443343001230781,
+ "tokens_seen": 2112356352
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909368104312939,
+ "loss": 1.2515,
+ "theoretical_loss": 3.4433342624258563,
+ "tokens_seen": 2112421888
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909207622868606,
+ "loss": 1.3391,
+ "theoretical_loss": 3.4433255239679497,
+ "tokens_seen": 2112487424
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002909047141424273,
+ "loss": 1.239,
+ "theoretical_loss": 3.4433167858570375,
+ "tokens_seen": 2112552960
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000290888665997994,
+ "loss": 1.3362,
+ "theoretical_loss": 3.443308048093094,
+ "tokens_seen": 2112618496
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002908726178535607,
+ "loss": 1.3537,
+ "theoretical_loss": 3.443299310676096,
+ "tokens_seen": 2112684032
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002908565697091274,
+ "loss": 1.2473,
+ "theoretical_loss": 3.4432905736060184,
+ "tokens_seen": 2112749568
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002908405215646941,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4432818368828366,
+ "tokens_seen": 2112815104
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002908244734202608,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4432731005065262,
+ "tokens_seen": 2112880640
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002908084252758275,
+ "loss": 1.3238,
+ "theoretical_loss": 3.443264364477062,
+ "tokens_seen": 2112946176
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002907923771313942,
+ "loss": 1.3158,
+ "theoretical_loss": 3.4432556287944207,
+ "tokens_seen": 2113011712
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002907763289869609,
+ "loss": 1.3023,
+ "theoretical_loss": 3.4432468934585767,
+ "tokens_seen": 2113077248
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002907602808425276,
+ "loss": 1.3082,
+ "theoretical_loss": 3.443238158469506,
+ "tokens_seen": 2113142784
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002907442326980943,
+ "loss": 1.1697,
+ "theoretical_loss": 3.443229423827184,
+ "tokens_seen": 2113208320
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000290728184553661,
+ "loss": 1.2475,
+ "theoretical_loss": 3.443220689531586,
+ "tokens_seen": 2113273856
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002907121364092277,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4432119555826883,
+ "tokens_seen": 2113339392
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906960882647944,
+ "loss": 1.2493,
+ "theoretical_loss": 3.443203221980465,
+ "tokens_seen": 2113404928
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906800401203611,
+ "loss": 1.3285,
+ "theoretical_loss": 3.4431944887248926,
+ "tokens_seen": 2113470464
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 707433,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1259653568267822,
+ "objective/train/theoretical_loss": 3.4431857558159464,
+ "objective/train/tokens_used": 2133996000,
+ "theoretical_loss": 3.4431857558159464,
+ "tokens_seen": 2113536000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906639919759278,
+ "loss": 1.1662,
+ "theoretical_loss": 3.4431857558159464,
+ "tokens_seen": 2113536000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906479438314945,
+ "loss": 1.2663,
+ "theoretical_loss": 3.443177023253601,
+ "tokens_seen": 2113601536
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906318956870612,
+ "loss": 1.3668,
+ "theoretical_loss": 3.443168291037834,
+ "tokens_seen": 2113667072
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002906158475426279,
+ "loss": 1.1815,
+ "theoretical_loss": 3.4431595591686186,
+ "tokens_seen": 2113732608
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905997993981946,
+ "loss": 1.4156,
+ "theoretical_loss": 3.4431508276459315,
+ "tokens_seen": 2113798144
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905837512537613,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4431420964697486,
+ "tokens_seen": 2113863680
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000290567703109328,
+ "loss": 1.1691,
+ "theoretical_loss": 3.443133365640044,
+ "tokens_seen": 2113929216
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905516549648947,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4431246351567943,
+ "tokens_seen": 2113994752
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905356068204614,
+ "loss": 1.2157,
+ "theoretical_loss": 3.443115905019975,
+ "tokens_seen": 2114060288
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905195586760281,
+ "loss": 1.3268,
+ "theoretical_loss": 3.443107175229561,
+ "tokens_seen": 2114125824
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002905035105315948,
+ "loss": 1.2281,
+ "theoretical_loss": 3.4430984457855285,
+ "tokens_seen": 2114191360
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002904874623871615,
+ "loss": 1.2267,
+ "theoretical_loss": 3.4430897166878527,
+ "tokens_seen": 2114256896
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029047141424272816,
+ "loss": 1.283,
+ "theoretical_loss": 3.4430809879365087,
+ "tokens_seen": 2114322432
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029045536609829486,
+ "loss": 1.2251,
+ "theoretical_loss": 3.4430722595314727,
+ "tokens_seen": 2114387968
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029043931795386157,
+ "loss": 1.21,
+ "theoretical_loss": 3.4430635314727196,
+ "tokens_seen": 2114453504
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029042326980942827,
+ "loss": 1.241,
+ "theoretical_loss": 3.4430548037602255,
+ "tokens_seen": 2114519040
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029040722166499497,
+ "loss": 1.1837,
+ "theoretical_loss": 3.443046076393966,
+ "tokens_seen": 2114584576
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029039117352056167,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4430373493739164,
+ "tokens_seen": 2114650112
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029037512537612837,
+ "loss": 1.2205,
+ "theoretical_loss": 3.443028622700052,
+ "tokens_seen": 2114715648
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029035907723169507,
+ "loss": 1.2717,
+ "theoretical_loss": 3.4430198963723484,
+ "tokens_seen": 2114781184
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029034302908726177,
+ "loss": 1.2229,
+ "theoretical_loss": 3.4430111703907813,
+ "tokens_seen": 2114846720
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029032698094282847,
+ "loss": 1.3069,
+ "theoretical_loss": 3.4430024447553267,
+ "tokens_seen": 2114912256
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029031093279839517,
+ "loss": 1.25,
+ "theoretical_loss": 3.4429937194659592,
+ "tokens_seen": 2114977792
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029029488465396187,
+ "loss": 1.1909,
+ "theoretical_loss": 3.4429849945226545,
+ "tokens_seen": 2115043328
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029027883650952857,
+ "loss": 1.1593,
+ "theoretical_loss": 3.442976269925389,
+ "tokens_seen": 2115108864
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029026278836509527,
+ "loss": 1.3597,
+ "theoretical_loss": 3.4429675456741373,
+ "tokens_seen": 2115174400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029024674022066197,
+ "loss": 1.2895,
+ "theoretical_loss": 3.442958821768876,
+ "tokens_seen": 2115239936
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002902306920762287,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4429500982095798,
+ "tokens_seen": 2115305472
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002902146439317954,
+ "loss": 1.1805,
+ "theoretical_loss": 3.442941374996224,
+ "tokens_seen": 2115371008
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901985957873621,
+ "loss": 1.248,
+ "theoretical_loss": 3.4429326521287855,
+ "tokens_seen": 2115436544
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901825476429288,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4429239296072383,
+ "tokens_seen": 2115502080
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901664994984955,
+ "loss": 1.3209,
+ "theoretical_loss": 3.442915207431559,
+ "tokens_seen": 2115567616
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901504513540622,
+ "loss": 1.1991,
+ "theoretical_loss": 3.442906485601723,
+ "tokens_seen": 2115633152
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901344032096289,
+ "loss": 1.2545,
+ "theoretical_loss": 3.4428977641177054,
+ "tokens_seen": 2115698688
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901183550651956,
+ "loss": 1.2406,
+ "theoretical_loss": 3.4428890429794823,
+ "tokens_seen": 2115764224
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002901023069207623,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4428803221870297,
+ "tokens_seen": 2115829760
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029008625877632903,
+ "loss": 1.2678,
+ "theoretical_loss": 3.442871601740322,
+ "tokens_seen": 2115895296
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029007021063189573,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4428628816393347,
+ "tokens_seen": 2115960832
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029005416248746243,
+ "loss": 1.1676,
+ "theoretical_loss": 3.4428541618840454,
+ "tokens_seen": 2116026368
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029003811434302914,
+ "loss": 1.184,
+ "theoretical_loss": 3.4428454424744275,
+ "tokens_seen": 2116091904
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029002206619859584,
+ "loss": 1.3178,
+ "theoretical_loss": 3.4428367234104575,
+ "tokens_seen": 2116157440
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00029000601805416254,
+ "loss": 1.2162,
+ "theoretical_loss": 3.442828004692111,
+ "tokens_seen": 2116222976
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028998996990972924,
+ "loss": 1.2909,
+ "theoretical_loss": 3.442819286319364,
+ "tokens_seen": 2116288512
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028997392176529594,
+ "loss": 1.1848,
+ "theoretical_loss": 3.4428105682921912,
+ "tokens_seen": 2116354048
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028995787362086264,
+ "loss": 1.2543,
+ "theoretical_loss": 3.442801850610569,
+ "tokens_seen": 2116419584
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028994182547642934,
+ "loss": 1.1889,
+ "theoretical_loss": 3.442793133274472,
+ "tokens_seen": 2116485120
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028992577733199604,
+ "loss": 1.3355,
+ "theoretical_loss": 3.4427844162838768,
+ "tokens_seen": 2116550656
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028990972918756274,
+ "loss": 1.2685,
+ "theoretical_loss": 3.4427756996387586,
+ "tokens_seen": 2116616192
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028989368104312944,
+ "loss": 1.1742,
+ "theoretical_loss": 3.442766983339093,
+ "tokens_seen": 2116681728
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028987763289869614,
+ "loss": 1.2429,
+ "theoretical_loss": 3.442758267384856,
+ "tokens_seen": 2116747264
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 708587,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1421294212341309,
+ "objective/train/theoretical_loss": 3.4427495517760223,
+ "objective/train/tokens_used": 2137272800,
+ "theoretical_loss": 3.4427495517760223,
+ "tokens_seen": 2116812800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028986158475426284,
+ "loss": 1.1653,
+ "theoretical_loss": 3.4427495517760223,
+ "tokens_seen": 2116812800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002898455366098295,
+ "loss": 1.2893,
+ "theoretical_loss": 3.4427408365125687,
+ "tokens_seen": 2116878336
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002898294884653962,
+ "loss": 1.2339,
+ "theoretical_loss": 3.44273212159447,
+ "tokens_seen": 2116943872
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002898134403209629,
+ "loss": 1.2212,
+ "theoretical_loss": 3.442723407021702,
+ "tokens_seen": 2117009408
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897973921765296,
+ "loss": 1.3382,
+ "theoretical_loss": 3.4427146927942402,
+ "tokens_seen": 2117074944
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897813440320963,
+ "loss": 1.1565,
+ "theoretical_loss": 3.442705978912061,
+ "tokens_seen": 2117140480
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000289765295887663,
+ "loss": 1.2541,
+ "theoretical_loss": 3.442697265375139,
+ "tokens_seen": 2117206016
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897492477432297,
+ "loss": 1.2564,
+ "theoretical_loss": 3.4426885521834505,
+ "tokens_seen": 2117271552
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897331995987964,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4426798393369706,
+ "tokens_seen": 2117337088
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897171514543631,
+ "loss": 1.2022,
+ "theoretical_loss": 3.4426711268356756,
+ "tokens_seen": 2117402624
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002897011033099298,
+ "loss": 1.1869,
+ "theoretical_loss": 3.442662414679541,
+ "tokens_seen": 2117468160
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002896850551654965,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4426537028685416,
+ "tokens_seen": 2117533696
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002896690070210632,
+ "loss": 1.2502,
+ "theoretical_loss": 3.442644991402654,
+ "tokens_seen": 2117599232
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002896529588766299,
+ "loss": 1.3387,
+ "theoretical_loss": 3.442636280281853,
+ "tokens_seen": 2117664768
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002896369107321966,
+ "loss": 1.2224,
+ "theoretical_loss": 3.442627569506116,
+ "tokens_seen": 2117730304
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002896208625877633,
+ "loss": 1.3095,
+ "theoretical_loss": 3.4426188590754165,
+ "tokens_seen": 2117795840
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028960481444333,
+ "loss": 1.3399,
+ "theoretical_loss": 3.442610148989731,
+ "tokens_seen": 2117861376
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895887662988967,
+ "loss": 1.25,
+ "theoretical_loss": 3.442601439249036,
+ "tokens_seen": 2117926912
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895727181544634,
+ "loss": 1.3456,
+ "theoretical_loss": 3.442592729853306,
+ "tokens_seen": 2117992448
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895566700100301,
+ "loss": 1.2459,
+ "theoretical_loss": 3.442584020802517,
+ "tokens_seen": 2118057984
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895406218655968,
+ "loss": 1.2221,
+ "theoretical_loss": 3.4425753120966447,
+ "tokens_seen": 2118123520
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895245737211635,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4425666037356653,
+ "tokens_seen": 2118189056
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002895085255767302,
+ "loss": 1.3126,
+ "theoretical_loss": 3.4425578957195535,
+ "tokens_seen": 2118254592
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002894924774322969,
+ "loss": 1.1686,
+ "theoretical_loss": 3.442549188048285,
+ "tokens_seen": 2118320128
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002894764292878636,
+ "loss": 1.3358,
+ "theoretical_loss": 3.4425404807218367,
+ "tokens_seen": 2118385664
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002894603811434303,
+ "loss": 1.327,
+ "theoretical_loss": 3.4425317737401837,
+ "tokens_seen": 2118451200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000289444332998997,
+ "loss": 1.274,
+ "theoretical_loss": 3.442523067103301,
+ "tokens_seen": 2118516736
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002894282848545637,
+ "loss": 1.1927,
+ "theoretical_loss": 3.442514360811165,
+ "tokens_seen": 2118582272
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002894122367101304,
+ "loss": 1.2467,
+ "theoretical_loss": 3.442505654863751,
+ "tokens_seen": 2118647808
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893961885656971,
+ "loss": 1.247,
+ "theoretical_loss": 3.4424969492610344,
+ "tokens_seen": 2118713344
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893801404212638,
+ "loss": 1.3265,
+ "theoretical_loss": 3.442488244002992,
+ "tokens_seen": 2118778880
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893640922768305,
+ "loss": 1.1696,
+ "theoretical_loss": 3.442479539089599,
+ "tokens_seen": 2118844416
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893480441323972,
+ "loss": 1.3036,
+ "theoretical_loss": 3.4424708345208304,
+ "tokens_seen": 2118909952
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893319959879639,
+ "loss": 1.2407,
+ "theoretical_loss": 3.4424621302966627,
+ "tokens_seen": 2118975488
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002893159478435306,
+ "loss": 1.2972,
+ "theoretical_loss": 3.442453426417071,
+ "tokens_seen": 2119041024
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892998996990973,
+ "loss": 1.1693,
+ "theoretical_loss": 3.4424447228820316,
+ "tokens_seen": 2119106560
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000289283851554664,
+ "loss": 1.2878,
+ "theoretical_loss": 3.4424360196915202,
+ "tokens_seen": 2119172096
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892678034102307,
+ "loss": 1.1474,
+ "theoretical_loss": 3.442427316845512,
+ "tokens_seen": 2119237632
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892517552657974,
+ "loss": 1.2193,
+ "theoretical_loss": 3.4424186143439828,
+ "tokens_seen": 2119303168
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892357071213641,
+ "loss": 1.2735,
+ "theoretical_loss": 3.4424099121869087,
+ "tokens_seen": 2119368704
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892196589769308,
+ "loss": 1.2168,
+ "theoretical_loss": 3.442401210374265,
+ "tokens_seen": 2119434240
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002892036108324975,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4423925089060274,
+ "tokens_seen": 2119499776
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002891875626880642,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4423838077821722,
+ "tokens_seen": 2119565312
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002891715145436309,
+ "loss": 1.239,
+ "theoretical_loss": 3.442375107002675,
+ "tokens_seen": 2119630848
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002891554663991976,
+ "loss": 1.348,
+ "theoretical_loss": 3.442366406567511,
+ "tokens_seen": 2119696384
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002891394182547643,
+ "loss": 1.2413,
+ "theoretical_loss": 3.442357706476656,
+ "tokens_seen": 2119761920
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000289123370110331,
+ "loss": 1.4197,
+ "theoretical_loss": 3.442349006730086,
+ "tokens_seen": 2119827456
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002891073219658977,
+ "loss": 1.2307,
+ "theoretical_loss": 3.4423403073277763,
+ "tokens_seen": 2119892992
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002890912738214644,
+ "loss": 1.3436,
+ "theoretical_loss": 3.4423316082697037,
+ "tokens_seen": 2119958528
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028907522567703106,
+ "loss": 1.2063,
+ "theoretical_loss": 3.442322909555843,
+ "tokens_seen": 2120024064
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 709563,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1953669786453247,
+ "objective/train/theoretical_loss": 3.44231421118617,
+ "objective/train/tokens_used": 2140549600,
+ "theoretical_loss": 3.44231421118617,
+ "tokens_seen": 2120089600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028905917753259777,
+ "loss": 1.21,
+ "theoretical_loss": 3.44231421118617,
+ "tokens_seen": 2120089600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028904312938816447,
+ "loss": 1.2073,
+ "theoretical_loss": 3.442305513160661,
+ "tokens_seen": 2120155136
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028902708124373117,
+ "loss": 1.2469,
+ "theoretical_loss": 3.442296815479291,
+ "tokens_seen": 2120220672
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028901103309929787,
+ "loss": 1.2533,
+ "theoretical_loss": 3.442288118142036,
+ "tokens_seen": 2120286208
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028899498495486457,
+ "loss": 1.2671,
+ "theoretical_loss": 3.442279421148872,
+ "tokens_seen": 2120351744
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028897893681043127,
+ "loss": 1.271,
+ "theoretical_loss": 3.4422707244997746,
+ "tokens_seen": 2120417280
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028896288866599797,
+ "loss": 1.2723,
+ "theoretical_loss": 3.44226202819472,
+ "tokens_seen": 2120482816
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028894684052156467,
+ "loss": 1.2294,
+ "theoretical_loss": 3.4422533322336824,
+ "tokens_seen": 2120548352
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028893079237713137,
+ "loss": 1.2041,
+ "theoretical_loss": 3.44224463661664,
+ "tokens_seen": 2120613888
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028891474423269807,
+ "loss": 1.2519,
+ "theoretical_loss": 3.442235941343566,
+ "tokens_seen": 2120679424
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028889869608826477,
+ "loss": 1.2088,
+ "theoretical_loss": 3.442227246414438,
+ "tokens_seen": 2120744960
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002888826479438315,
+ "loss": 1.3304,
+ "theoretical_loss": 3.4422185518292316,
+ "tokens_seen": 2120810496
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028886659979939823,
+ "loss": 1.3289,
+ "theoretical_loss": 3.4422098575879216,
+ "tokens_seen": 2120876032
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028885055165496493,
+ "loss": 1.3174,
+ "theoretical_loss": 3.4422011636904846,
+ "tokens_seen": 2120941568
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028883450351053163,
+ "loss": 1.1963,
+ "theoretical_loss": 3.442192470136896,
+ "tokens_seen": 2121007104
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028881845536609833,
+ "loss": 1.3458,
+ "theoretical_loss": 3.4421837769271315,
+ "tokens_seen": 2121072640
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028880240722166503,
+ "loss": 1.3453,
+ "theoretical_loss": 3.442175084061167,
+ "tokens_seen": 2121138176
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028878635907723173,
+ "loss": 1.2966,
+ "theoretical_loss": 3.4421663915389784,
+ "tokens_seen": 2121203712
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028877031093279843,
+ "loss": 1.3202,
+ "theoretical_loss": 3.442157699360542,
+ "tokens_seen": 2121269248
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028875426278836513,
+ "loss": 1.3194,
+ "theoretical_loss": 3.442149007525832,
+ "tokens_seen": 2121334784
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028873821464393183,
+ "loss": 1.3137,
+ "theoretical_loss": 3.442140316034826,
+ "tokens_seen": 2121400320
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028872216649949853,
+ "loss": 1.2858,
+ "theoretical_loss": 3.442131624887499,
+ "tokens_seen": 2121465856
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028870611835506523,
+ "loss": 1.2993,
+ "theoretical_loss": 3.442122934083826,
+ "tokens_seen": 2121531392
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028869007021063193,
+ "loss": 1.3396,
+ "theoretical_loss": 3.4421142436237844,
+ "tokens_seen": 2121596928
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028867402206619863,
+ "loss": 1.1164,
+ "theoretical_loss": 3.4421055535073486,
+ "tokens_seen": 2121662464
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028865797392176534,
+ "loss": 1.2378,
+ "theoretical_loss": 3.442096863734495,
+ "tokens_seen": 2121728000
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028864192577733204,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4420881743051996,
+ "tokens_seen": 2121793536
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028862587763289874,
+ "loss": 1.404,
+ "theoretical_loss": 3.442079485219438,
+ "tokens_seen": 2121859072
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028860982948846544,
+ "loss": 1.3609,
+ "theoretical_loss": 3.4420707964771857,
+ "tokens_seen": 2121924608
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028859378134403214,
+ "loss": 1.2224,
+ "theoretical_loss": 3.442062108078419,
+ "tokens_seen": 2121990144
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028857773319959884,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4420534200231137,
+ "tokens_seen": 2122055680
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028856168505516554,
+ "loss": 1.2877,
+ "theoretical_loss": 3.4420447323112455,
+ "tokens_seen": 2122121216
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028854563691073224,
+ "loss": 1.233,
+ "theoretical_loss": 3.44203604494279,
+ "tokens_seen": 2122186752
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028852958876629894,
+ "loss": 1.3715,
+ "theoretical_loss": 3.4420273579177225,
+ "tokens_seen": 2122252288
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028851354062186564,
+ "loss": 1.1467,
+ "theoretical_loss": 3.4420186712360206,
+ "tokens_seen": 2122317824
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028849749247743234,
+ "loss": 1.293,
+ "theoretical_loss": 3.442009984897658,
+ "tokens_seen": 2122383360
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028848144433299904,
+ "loss": 1.3675,
+ "theoretical_loss": 3.4420012989026123,
+ "tokens_seen": 2122448896
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028846539618856574,
+ "loss": 1.1896,
+ "theoretical_loss": 3.441992613250858,
+ "tokens_seen": 2122514432
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028844934804413244,
+ "loss": 1.3655,
+ "theoretical_loss": 3.441983927942372,
+ "tokens_seen": 2122579968
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002884332998996991,
+ "loss": 1.209,
+ "theoretical_loss": 3.4419752429771293,
+ "tokens_seen": 2122645504
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002884172517552658,
+ "loss": 1.2895,
+ "theoretical_loss": 3.441966558355106,
+ "tokens_seen": 2122711040
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002884012036108325,
+ "loss": 1.2985,
+ "theoretical_loss": 3.4419578740762784,
+ "tokens_seen": 2122776576
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002883851554663992,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4419491901406216,
+ "tokens_seen": 2122842112
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002883691073219659,
+ "loss": 1.2628,
+ "theoretical_loss": 3.4419405065481126,
+ "tokens_seen": 2122907648
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002883530591775326,
+ "loss": 1.2672,
+ "theoretical_loss": 3.4419318232987255,
+ "tokens_seen": 2122973184
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002883370110330993,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4419231403924373,
+ "tokens_seen": 2123038720
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000288320962888666,
+ "loss": 1.3343,
+ "theoretical_loss": 3.441914457829224,
+ "tokens_seen": 2123104256
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002883049147442327,
+ "loss": 1.3214,
+ "theoretical_loss": 3.441905775609061,
+ "tokens_seen": 2123169792
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882888665997994,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4418970937319244,
+ "tokens_seen": 2123235328
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882728184553661,
+ "loss": 1.3256,
+ "theoretical_loss": 3.44188841219779,
+ "tokens_seen": 2123300864
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 710783,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2148313522338867,
+ "objective/train/theoretical_loss": 3.441879731006633,
+ "objective/train/tokens_used": 2143826400,
+ "theoretical_loss": 3.441879731006633,
+ "tokens_seen": 2123366400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882567703109328,
+ "loss": 1.2569,
+ "theoretical_loss": 3.441879731006633,
+ "tokens_seen": 2123366400
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882407221664995,
+ "loss": 1.2502,
+ "theoretical_loss": 3.44187105015843,
+ "tokens_seen": 2123431936
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882246740220662,
+ "loss": 1.2611,
+ "theoretical_loss": 3.4418623696531574,
+ "tokens_seen": 2123497472
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002882086258776329,
+ "loss": 1.2391,
+ "theoretical_loss": 3.44185368949079,
+ "tokens_seen": 2123563008
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002881925777331996,
+ "loss": 1.2672,
+ "theoretical_loss": 3.441845009671304,
+ "tokens_seen": 2123628544
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002881765295887663,
+ "loss": 1.3013,
+ "theoretical_loss": 3.4418363301946755,
+ "tokens_seen": 2123694080
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000288160481444333,
+ "loss": 1.3159,
+ "theoretical_loss": 3.44182765106088,
+ "tokens_seen": 2123759616
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002881444332998997,
+ "loss": 1.2709,
+ "theoretical_loss": 3.441818972269894,
+ "tokens_seen": 2123825152
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002881283851554664,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4418102938216926,
+ "tokens_seen": 2123890688
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002881123370110331,
+ "loss": 1.3081,
+ "theoretical_loss": 3.441801615716252,
+ "tokens_seen": 2123956224
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880962888665998,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4417929379535486,
+ "tokens_seen": 2124021760
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880802407221665,
+ "loss": 1.3721,
+ "theoretical_loss": 3.4417842605335576,
+ "tokens_seen": 2124087296
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880641925777332,
+ "loss": 1.2431,
+ "theoretical_loss": 3.441775583456255,
+ "tokens_seen": 2124152832
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880481444332999,
+ "loss": 1.3118,
+ "theoretical_loss": 3.4417669067216172,
+ "tokens_seen": 2124218368
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880320962888666,
+ "loss": 1.2263,
+ "theoretical_loss": 3.4417582303296195,
+ "tokens_seen": 2124283904
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002880160481444333,
+ "loss": 1.2527,
+ "theoretical_loss": 3.441749554280238,
+ "tokens_seen": 2124349440
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000288,
+ "loss": 1.3048,
+ "theoretical_loss": 3.441740878573449,
+ "tokens_seen": 2124414976
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879839518555667,
+ "loss": 1.2588,
+ "theoretical_loss": 3.4417322032092272,
+ "tokens_seen": 2124480512
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879679037111334,
+ "loss": 1.2845,
+ "theoretical_loss": 3.4417235281875502,
+ "tokens_seen": 2124546048
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879518555667001,
+ "loss": 1.3406,
+ "theoretical_loss": 3.441714853508392,
+ "tokens_seen": 2124611584
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879358074222668,
+ "loss": 1.2738,
+ "theoretical_loss": 3.4417061791717307,
+ "tokens_seen": 2124677120
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879197592778335,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4416975051775407,
+ "tokens_seen": 2124742656
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002879037111334002,
+ "loss": 1.3574,
+ "theoretical_loss": 3.4416888315257985,
+ "tokens_seen": 2124808192
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002878876629889669,
+ "loss": 1.3336,
+ "theoretical_loss": 3.4416801582164793,
+ "tokens_seen": 2124873728
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002878716148445336,
+ "loss": 1.2521,
+ "theoretical_loss": 3.44167148524956,
+ "tokens_seen": 2124939264
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002878555667001003,
+ "loss": 1.3123,
+ "theoretical_loss": 3.4416628126250153,
+ "tokens_seen": 2125004800
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000287839518555667,
+ "loss": 1.2726,
+ "theoretical_loss": 3.4416541403428225,
+ "tokens_seen": 2125070336
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002878234704112337,
+ "loss": 1.2862,
+ "theoretical_loss": 3.4416454684029567,
+ "tokens_seen": 2125135872
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002878074222668004,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4416367968053945,
+ "tokens_seen": 2125201408
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877913741223671,
+ "loss": 1.337,
+ "theoretical_loss": 3.4416281255501113,
+ "tokens_seen": 2125266944
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877753259779338,
+ "loss": 1.1889,
+ "theoretical_loss": 3.441619454637083,
+ "tokens_seen": 2125332480
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877592778335005,
+ "loss": 1.2525,
+ "theoretical_loss": 3.441610784066285,
+ "tokens_seen": 2125398016
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877432296890672,
+ "loss": 1.3288,
+ "theoretical_loss": 3.4416021138376944,
+ "tokens_seen": 2125463552
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877271815446339,
+ "loss": 1.2596,
+ "theoretical_loss": 3.4415934439512865,
+ "tokens_seen": 2125529088
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002877111334002006,
+ "loss": 1.316,
+ "theoretical_loss": 3.441584774407038,
+ "tokens_seen": 2125594624
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002876950852557673,
+ "loss": 1.317,
+ "theoretical_loss": 3.4415761052049234,
+ "tokens_seen": 2125660160
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000287679037111334,
+ "loss": 1.2413,
+ "theoretical_loss": 3.44156743634492,
+ "tokens_seen": 2125725696
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002876629889669007,
+ "loss": 1.3342,
+ "theoretical_loss": 3.441558767827003,
+ "tokens_seen": 2125791232
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002876469408224674,
+ "loss": 1.3839,
+ "theoretical_loss": 3.4415500996511486,
+ "tokens_seen": 2125856768
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002876308926780341,
+ "loss": 1.1593,
+ "theoretical_loss": 3.4415414318173325,
+ "tokens_seen": 2125922304
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002876148445336008,
+ "loss": 1.2782,
+ "theoretical_loss": 3.4415327643255313,
+ "tokens_seen": 2125987840
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875987963891675,
+ "loss": 1.3625,
+ "theoretical_loss": 3.44152409717572,
+ "tokens_seen": 2126053376
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875827482447342,
+ "loss": 1.2358,
+ "theoretical_loss": 3.441515430367876,
+ "tokens_seen": 2126118912
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875667001003009,
+ "loss": 1.3056,
+ "theoretical_loss": 3.441506763901974,
+ "tokens_seen": 2126184448
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875506519558676,
+ "loss": 1.2511,
+ "theoretical_loss": 3.4414980977779903,
+ "tokens_seen": 2126249984
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875346038114343,
+ "loss": 1.266,
+ "theoretical_loss": 3.441489431995901,
+ "tokens_seen": 2126315520
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000287518555667001,
+ "loss": 1.308,
+ "theoretical_loss": 3.4414807665556815,
+ "tokens_seen": 2126381056
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002875025075225677,
+ "loss": 1.3626,
+ "theoretical_loss": 3.441472101457309,
+ "tokens_seen": 2126446592
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028748645937813443,
+ "loss": 1.3481,
+ "theoretical_loss": 3.4414634367007584,
+ "tokens_seen": 2126512128
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028747041123370113,
+ "loss": 1.2207,
+ "theoretical_loss": 3.441454772286006,
+ "tokens_seen": 2126577664
+ },
+ {
+ "epoch": 0.64,
+ "objective/train/docs_used": 711700,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1988120079040527,
+ "objective/train/theoretical_loss": 3.4414461082130283,
+ "objective/train/tokens_used": 2147103200,
+ "theoretical_loss": 3.4414461082130283,
+ "tokens_seen": 2126643200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028745436308926783,
+ "loss": 1.1536,
+ "theoretical_loss": 3.4414461082130283,
+ "tokens_seen": 2126643200
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028743831494483453,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4414374444818003,
+ "tokens_seen": 2126708736
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028742226680040123,
+ "loss": 1.1813,
+ "theoretical_loss": 3.4414287810922985,
+ "tokens_seen": 2126774272
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028740621865596793,
+ "loss": 1.3024,
+ "theoretical_loss": 3.4414201180444994,
+ "tokens_seen": 2126839808
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028739017051153463,
+ "loss": 1.2308,
+ "theoretical_loss": 3.441411455338378,
+ "tokens_seen": 2126905344
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028737412236710133,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4414027929739115,
+ "tokens_seen": 2126970880
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028735807422266803,
+ "loss": 1.2961,
+ "theoretical_loss": 3.4413941309510747,
+ "tokens_seen": 2127036416
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028734202607823473,
+ "loss": 1.2575,
+ "theoretical_loss": 3.441385469269844,
+ "tokens_seen": 2127101952
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028732597793380143,
+ "loss": 1.3445,
+ "theoretical_loss": 3.4413768079301956,
+ "tokens_seen": 2127167488
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028730992978936813,
+ "loss": 1.2895,
+ "theoretical_loss": 3.4413681469321054,
+ "tokens_seen": 2127233024
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028729388164493484,
+ "loss": 1.4079,
+ "theoretical_loss": 3.4413594862755494,
+ "tokens_seen": 2127298560
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028727783350050154,
+ "loss": 1.2045,
+ "theoretical_loss": 3.4413508259605035,
+ "tokens_seen": 2127364096
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028726178535606824,
+ "loss": 1.3002,
+ "theoretical_loss": 3.441342165986944,
+ "tokens_seen": 2127429632
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028724573721163494,
+ "loss": 1.3561,
+ "theoretical_loss": 3.441333506354847,
+ "tokens_seen": 2127495168
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028722968906720164,
+ "loss": 1.301,
+ "theoretical_loss": 3.441324847064188,
+ "tokens_seen": 2127560704
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028721364092276834,
+ "loss": 1.3778,
+ "theoretical_loss": 3.441316188114943,
+ "tokens_seen": 2127626240
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028719759277833504,
+ "loss": 1.2908,
+ "theoretical_loss": 3.441307529507089,
+ "tokens_seen": 2127691776
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028718154463390174,
+ "loss": 1.3052,
+ "theoretical_loss": 3.4412988712406007,
+ "tokens_seen": 2127757312
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028716549648946844,
+ "loss": 1.2593,
+ "theoretical_loss": 3.441290213315455,
+ "tokens_seen": 2127822848
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028714944834503514,
+ "loss": 1.1829,
+ "theoretical_loss": 3.441281555731628,
+ "tokens_seen": 2127888384
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028713340020060184,
+ "loss": 1.2046,
+ "theoretical_loss": 3.441272898489095,
+ "tokens_seen": 2127953920
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028711735205616854,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4412642415878327,
+ "tokens_seen": 2128019456
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028710130391173524,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4412555850278164,
+ "tokens_seen": 2128084992
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028708525576730194,
+ "loss": 1.2943,
+ "theoretical_loss": 3.441246928809023,
+ "tokens_seen": 2128150528
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028706920762286864,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4412382729314284,
+ "tokens_seen": 2128216064
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.00028705315947843534,
+ "loss": 1.384,
+ "theoretical_loss": 3.4412296173950083,
+ "tokens_seen": 2128281600
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.000287037111334002,
+ "loss": 1.3061,
+ "theoretical_loss": 3.441220962199739,
+ "tokens_seen": 2128347136
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002870210631895687,
+ "loss": 1.2688,
+ "theoretical_loss": 3.4412123073455962,
+ "tokens_seen": 2128412672
+ },
+ {
+ "epoch": 0.64,
+ "learning_rate": 0.0002870050150451354,
+ "loss": 1.4018,
+ "theoretical_loss": 3.4412036528325562,
+ "tokens_seen": 2128478208
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869889669007021,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4411949986605954,
+ "tokens_seen": 2128543744
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869729187562688,
+ "loss": 1.243,
+ "theoretical_loss": 3.441186344829689,
+ "tokens_seen": 2128609280
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869568706118355,
+ "loss": 1.2756,
+ "theoretical_loss": 3.4411776913398135,
+ "tokens_seen": 2128674816
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869408224674022,
+ "loss": 1.3485,
+ "theoretical_loss": 3.4411690381909454,
+ "tokens_seen": 2128740352
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869247743229689,
+ "loss": 1.2449,
+ "theoretical_loss": 3.44116038538306,
+ "tokens_seen": 2128805888
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002869087261785356,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4411517329161345,
+ "tokens_seen": 2128871424
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002868926780341023,
+ "loss": 1.3337,
+ "theoretical_loss": 3.4411430807901433,
+ "tokens_seen": 2128936960
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000286876629889669,
+ "loss": 1.2606,
+ "theoretical_loss": 3.441134429005064,
+ "tokens_seen": 2129002496
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002868605817452357,
+ "loss": 1.285,
+ "theoretical_loss": 3.4411257775608717,
+ "tokens_seen": 2129068032
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002868445336008024,
+ "loss": 1.3337,
+ "theoretical_loss": 3.441117126457543,
+ "tokens_seen": 2129133568
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002868284854563691,
+ "loss": 1.2708,
+ "theoretical_loss": 3.441108475695054,
+ "tokens_seen": 2129199104
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002868124373119358,
+ "loss": 1.2781,
+ "theoretical_loss": 3.4410998252733798,
+ "tokens_seen": 2129264640
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867963891675025,
+ "loss": 1.3124,
+ "theoretical_loss": 3.441091175192498,
+ "tokens_seen": 2129330176
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867803410230692,
+ "loss": 1.3762,
+ "theoretical_loss": 3.441082525452384,
+ "tokens_seen": 2129395712
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867642928786359,
+ "loss": 1.2174,
+ "theoretical_loss": 3.4410738760530135,
+ "tokens_seen": 2129461248
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867482447342026,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4410652269943633,
+ "tokens_seen": 2129526784
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867321965897693,
+ "loss": 1.3083,
+ "theoretical_loss": 3.4410565782764087,
+ "tokens_seen": 2129592320
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000286716148445336,
+ "loss": 1.2282,
+ "theoretical_loss": 3.441047929899127,
+ "tokens_seen": 2129657856
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002867001003009027,
+ "loss": 1.1837,
+ "theoretical_loss": 3.4410392818624924,
+ "tokens_seen": 2129723392
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866840521564694,
+ "loss": 1.218,
+ "theoretical_loss": 3.441030634166483,
+ "tokens_seen": 2129788928
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866680040120361,
+ "loss": 1.2565,
+ "theoretical_loss": 3.441021986811074,
+ "tokens_seen": 2129854464
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 712976,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.272249460220337,
+ "objective/train/theoretical_loss": 3.4410133397962412,
+ "objective/train/tokens_used": 2150380000,
+ "theoretical_loss": 3.4410133397962412,
+ "tokens_seen": 2129920000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866519558676028,
+ "loss": 1.319,
+ "theoretical_loss": 3.4410133397962412,
+ "tokens_seen": 2129920000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866359077231695,
+ "loss": 1.2676,
+ "theoretical_loss": 3.441004693121961,
+ "tokens_seen": 2129985536
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866198595787362,
+ "loss": 1.3269,
+ "theoretical_loss": 3.44099604678821,
+ "tokens_seen": 2130051072
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002866038114343029,
+ "loss": 1.298,
+ "theoretical_loss": 3.4409874007949632,
+ "tokens_seen": 2130116608
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002865877632898696,
+ "loss": 1.1871,
+ "theoretical_loss": 3.440978755142198,
+ "tokens_seen": 2130182144
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002865717151454363,
+ "loss": 1.145,
+ "theoretical_loss": 3.4409701098298893,
+ "tokens_seen": 2130247680
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000286555667001003,
+ "loss": 1.2723,
+ "theoretical_loss": 3.4409614648580145,
+ "tokens_seen": 2130313216
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002865396188565697,
+ "loss": 1.3892,
+ "theoretical_loss": 3.440952820226549,
+ "tokens_seen": 2130378752
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002865235707121364,
+ "loss": 1.2214,
+ "theoretical_loss": 3.440944175935469,
+ "tokens_seen": 2130444288
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002865075225677031,
+ "loss": 1.2733,
+ "theoretical_loss": 3.44093553198475,
+ "tokens_seen": 2130509824
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002864914744232698,
+ "loss": 1.3492,
+ "theoretical_loss": 3.440926888374369,
+ "tokens_seen": 2130575360
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028647542627883657,
+ "loss": 1.3449,
+ "theoretical_loss": 3.4409182451043017,
+ "tokens_seen": 2130640896
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028645937813440327,
+ "loss": 1.2009,
+ "theoretical_loss": 3.440909602174525,
+ "tokens_seen": 2130706432
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028644332998996997,
+ "loss": 1.2939,
+ "theoretical_loss": 3.4409009595850137,
+ "tokens_seen": 2130771968
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028642728184553667,
+ "loss": 1.2052,
+ "theoretical_loss": 3.440892317335745,
+ "tokens_seen": 2130837504
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028641123370110337,
+ "loss": 1.296,
+ "theoretical_loss": 3.4408836754266945,
+ "tokens_seen": 2130903040
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028639518555667,
+ "loss": 1.3046,
+ "theoretical_loss": 3.440875033857839,
+ "tokens_seen": 2130968576
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002863791374122367,
+ "loss": 1.3161,
+ "theoretical_loss": 3.440866392629154,
+ "tokens_seen": 2131034112
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002863630892678034,
+ "loss": 1.3044,
+ "theoretical_loss": 3.4408577517406154,
+ "tokens_seen": 2131099648
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002863470411233701,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4408491111922004,
+ "tokens_seen": 2131165184
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002863309929789368,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4408404709838845,
+ "tokens_seen": 2131230720
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002863149448345035,
+ "loss": 1.1804,
+ "theoretical_loss": 3.440831831115643,
+ "tokens_seen": 2131296256
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862988966900702,
+ "loss": 1.288,
+ "theoretical_loss": 3.4408231915874543,
+ "tokens_seen": 2131361792
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862828485456369,
+ "loss": 1.352,
+ "theoretical_loss": 3.4408145523992926,
+ "tokens_seen": 2131427328
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862668004012036,
+ "loss": 1.2042,
+ "theoretical_loss": 3.4408059135511344,
+ "tokens_seen": 2131492864
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862507522567703,
+ "loss": 1.2325,
+ "theoretical_loss": 3.4407972750429567,
+ "tokens_seen": 2131558400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000286234704112337,
+ "loss": 1.2514,
+ "theoretical_loss": 3.4407886368747347,
+ "tokens_seen": 2131623936
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862186559679037,
+ "loss": 1.2866,
+ "theoretical_loss": 3.4407799990464456,
+ "tokens_seen": 2131689472
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002862026078234704,
+ "loss": 1.2288,
+ "theoretical_loss": 3.440771361558064,
+ "tokens_seen": 2131755008
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002861865596790371,
+ "loss": 1.3365,
+ "theoretical_loss": 3.440762724409568,
+ "tokens_seen": 2131820544
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002861705115346038,
+ "loss": 1.1642,
+ "theoretical_loss": 3.4407540876009324,
+ "tokens_seen": 2131886080
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002861544633901705,
+ "loss": 1.3112,
+ "theoretical_loss": 3.440745451132134,
+ "tokens_seen": 2131951616
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002861384152457372,
+ "loss": 1.3974,
+ "theoretical_loss": 3.4407368150031483,
+ "tokens_seen": 2132017152
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028612236710130393,
+ "loss": 1.2782,
+ "theoretical_loss": 3.440728179213952,
+ "tokens_seen": 2132082688
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028610631895687063,
+ "loss": 1.2066,
+ "theoretical_loss": 3.440719543764522,
+ "tokens_seen": 2132148224
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028609027081243733,
+ "loss": 1.3006,
+ "theoretical_loss": 3.4407109086548333,
+ "tokens_seen": 2132213760
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028607422266800403,
+ "loss": 1.28,
+ "theoretical_loss": 3.4407022738848623,
+ "tokens_seen": 2132279296
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028605817452357073,
+ "loss": 1.1735,
+ "theoretical_loss": 3.4406936394545857,
+ "tokens_seen": 2132344832
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028604212637913743,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4406850053639793,
+ "tokens_seen": 2132410368
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028602607823470413,
+ "loss": 1.3143,
+ "theoretical_loss": 3.4406763716130193,
+ "tokens_seen": 2132475904
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028601003009027083,
+ "loss": 1.2306,
+ "theoretical_loss": 3.4406677382016824,
+ "tokens_seen": 2132541440
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028599398194583753,
+ "loss": 1.1986,
+ "theoretical_loss": 3.440659105129944,
+ "tokens_seen": 2132606976
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028597793380140423,
+ "loss": 1.3056,
+ "theoretical_loss": 3.440650472397781,
+ "tokens_seen": 2132672512
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028596188565697093,
+ "loss": 1.2364,
+ "theoretical_loss": 3.4406418400051693,
+ "tokens_seen": 2132738048
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028594583751253763,
+ "loss": 1.2781,
+ "theoretical_loss": 3.440633207952085,
+ "tokens_seen": 2132803584
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028592978936810433,
+ "loss": 1.3399,
+ "theoretical_loss": 3.440624576238505,
+ "tokens_seen": 2132869120
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028591374122367104,
+ "loss": 1.2428,
+ "theoretical_loss": 3.4406159448644047,
+ "tokens_seen": 2132934656
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028589769307923774,
+ "loss": 1.2658,
+ "theoretical_loss": 3.4406073138297604,
+ "tokens_seen": 2133000192
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028588164493480444,
+ "loss": 1.3416,
+ "theoretical_loss": 3.4405986831345485,
+ "tokens_seen": 2133065728
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028586559679037114,
+ "loss": 1.1882,
+ "theoretical_loss": 3.440590052778745,
+ "tokens_seen": 2133131264
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 714181,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3658976554870605,
+ "objective/train/theoretical_loss": 3.4405814227623273,
+ "objective/train/tokens_used": 2153656800,
+ "theoretical_loss": 3.4405814227623273,
+ "tokens_seen": 2133196800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028584954864593784,
+ "loss": 1.3199,
+ "theoretical_loss": 3.4405814227623273,
+ "tokens_seen": 2133196800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028583350050150454,
+ "loss": 1.2737,
+ "theoretical_loss": 3.44057279308527,
+ "tokens_seen": 2133262336
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028581745235707124,
+ "loss": 1.302,
+ "theoretical_loss": 3.44056416374755,
+ "tokens_seen": 2133327872
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028580140421263794,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4405555347491443,
+ "tokens_seen": 2133393408
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028578535606820464,
+ "loss": 1.217,
+ "theoretical_loss": 3.4405469060900273,
+ "tokens_seen": 2133458944
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028576930792377134,
+ "loss": 1.3069,
+ "theoretical_loss": 3.440538277770177,
+ "tokens_seen": 2133524480
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028575325977933804,
+ "loss": 1.3104,
+ "theoretical_loss": 3.440529649789569,
+ "tokens_seen": 2133590016
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028573721163490474,
+ "loss": 1.3214,
+ "theoretical_loss": 3.4405210221481797,
+ "tokens_seen": 2133655552
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028572116349047144,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4405123948459844,
+ "tokens_seen": 2133721088
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028570511534603814,
+ "loss": 1.3434,
+ "theoretical_loss": 3.4405037678829604,
+ "tokens_seen": 2133786624
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028568906720160484,
+ "loss": 1.2786,
+ "theoretical_loss": 3.440495141259084,
+ "tokens_seen": 2133852160
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028567301905717155,
+ "loss": 1.4118,
+ "theoretical_loss": 3.440486514974331,
+ "tokens_seen": 2133917696
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028565697091273825,
+ "loss": 1.1879,
+ "theoretical_loss": 3.440477889028678,
+ "tokens_seen": 2133983232
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028564092276830495,
+ "loss": 1.2928,
+ "theoretical_loss": 3.4404692634221004,
+ "tokens_seen": 2134048768
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002856248746238716,
+ "loss": 1.3331,
+ "theoretical_loss": 3.4404606381545753,
+ "tokens_seen": 2134114304
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002856088264794383,
+ "loss": 1.3225,
+ "theoretical_loss": 3.440452013226079,
+ "tokens_seen": 2134179840
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000285592778335005,
+ "loss": 1.2651,
+ "theoretical_loss": 3.440443388636587,
+ "tokens_seen": 2134245376
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002855767301905717,
+ "loss": 1.2758,
+ "theoretical_loss": 3.440434764386077,
+ "tokens_seen": 2134310912
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002855606820461384,
+ "loss": 1.3103,
+ "theoretical_loss": 3.4404261404745236,
+ "tokens_seen": 2134376448
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002855446339017051,
+ "loss": 1.3924,
+ "theoretical_loss": 3.440417516901904,
+ "tokens_seen": 2134441984
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002855285857572718,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4404088936681942,
+ "tokens_seen": 2134507520
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002855125376128385,
+ "loss": 1.1782,
+ "theoretical_loss": 3.4404002707733703,
+ "tokens_seen": 2134573056
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854964894684052,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4403916482174095,
+ "tokens_seen": 2134638592
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854804413239719,
+ "loss": 1.2355,
+ "theoretical_loss": 3.440383026000287,
+ "tokens_seen": 2134704128
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854643931795386,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4403744041219797,
+ "tokens_seen": 2134769664
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854483450351053,
+ "loss": 1.3601,
+ "theoretical_loss": 3.4403657825824636,
+ "tokens_seen": 2134835200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000285432296890672,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4403571613817148,
+ "tokens_seen": 2134900736
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854162487462387,
+ "loss": 1.1016,
+ "theoretical_loss": 3.44034854051971,
+ "tokens_seen": 2134966272
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002854002006018054,
+ "loss": 1.2066,
+ "theoretical_loss": 3.4403399199964255,
+ "tokens_seen": 2135031808
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853841524573721,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4403312998118376,
+ "tokens_seen": 2135097344
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853681043129388,
+ "loss": 1.2458,
+ "theoretical_loss": 3.4403226799659223,
+ "tokens_seen": 2135162880
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853520561685055,
+ "loss": 1.3211,
+ "theoretical_loss": 3.4403140604586557,
+ "tokens_seen": 2135228416
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853360080240722,
+ "loss": 1.2906,
+ "theoretical_loss": 3.4403054412900147,
+ "tokens_seen": 2135293952
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853199598796389,
+ "loss": 1.3148,
+ "theoretical_loss": 3.4402968224599757,
+ "tokens_seen": 2135359488
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002853039117352056,
+ "loss": 1.222,
+ "theoretical_loss": 3.4402882039685148,
+ "tokens_seen": 2135425024
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002852878635907723,
+ "loss": 1.327,
+ "theoretical_loss": 3.4402795858156074,
+ "tokens_seen": 2135490560
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028527181544633906,
+ "loss": 1.2713,
+ "theoretical_loss": 3.440270968001231,
+ "tokens_seen": 2135556096
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028525576730190576,
+ "loss": 1.1929,
+ "theoretical_loss": 3.4402623505253613,
+ "tokens_seen": 2135621632
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028523971915747246,
+ "loss": 1.2891,
+ "theoretical_loss": 3.440253733387975,
+ "tokens_seen": 2135687168
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028522367101303916,
+ "loss": 1.2808,
+ "theoretical_loss": 3.440245116589048,
+ "tokens_seen": 2135752704
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028520762286860586,
+ "loss": 1.2178,
+ "theoretical_loss": 3.4402365001285573,
+ "tokens_seen": 2135818240
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028519157472417256,
+ "loss": 1.2552,
+ "theoretical_loss": 3.440227884006479,
+ "tokens_seen": 2135883776
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028517552657973926,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4402192682227883,
+ "tokens_seen": 2135949312
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028515947843530597,
+ "loss": 1.2823,
+ "theoretical_loss": 3.4402106527774627,
+ "tokens_seen": 2136014848
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028514343029087267,
+ "loss": 1.2314,
+ "theoretical_loss": 3.4402020376704785,
+ "tokens_seen": 2136080384
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028512738214643937,
+ "loss": 1.3729,
+ "theoretical_loss": 3.4401934229018116,
+ "tokens_seen": 2136145920
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028511133400200607,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4401848084714386,
+ "tokens_seen": 2136211456
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028509528585757277,
+ "loss": 1.2693,
+ "theoretical_loss": 3.440176194379336,
+ "tokens_seen": 2136276992
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028507923771313947,
+ "loss": 1.2829,
+ "theoretical_loss": 3.44016758062548,
+ "tokens_seen": 2136342528
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028506318956870617,
+ "loss": 1.1675,
+ "theoretical_loss": 3.4401589672098463,
+ "tokens_seen": 2136408064
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 715089,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2422316074371338,
+ "objective/train/theoretical_loss": 3.4401503541324123,
+ "objective/train/tokens_used": 2156933600,
+ "theoretical_loss": 3.4401503541324123,
+ "tokens_seen": 2136473600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028504714142427287,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4401503541324123,
+ "tokens_seen": 2136473600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028503109327983957,
+ "loss": 1.2492,
+ "theoretical_loss": 3.4401417413931537,
+ "tokens_seen": 2136539136
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028501504513540627,
+ "loss": 1.2835,
+ "theoretical_loss": 3.4401331289920467,
+ "tokens_seen": 2136604672
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849989969909729,
+ "loss": 1.2665,
+ "theoretical_loss": 3.440124516929068,
+ "tokens_seen": 2136670208
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849829488465396,
+ "loss": 1.1957,
+ "theoretical_loss": 3.440115905204194,
+ "tokens_seen": 2136735744
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849669007021063,
+ "loss": 1.3285,
+ "theoretical_loss": 3.440107293817401,
+ "tokens_seen": 2136801280
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000284950852557673,
+ "loss": 1.3692,
+ "theoretical_loss": 3.4400986827686655,
+ "tokens_seen": 2136866816
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849348044132397,
+ "loss": 1.2463,
+ "theoretical_loss": 3.4400900720579632,
+ "tokens_seen": 2136932352
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849187562688064,
+ "loss": 1.2299,
+ "theoretical_loss": 3.4400814616852715,
+ "tokens_seen": 2136997888
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002849027081243731,
+ "loss": 1.1824,
+ "theoretical_loss": 3.440072851650566,
+ "tokens_seen": 2137063424
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848866599799398,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4400642419538228,
+ "tokens_seen": 2137128960
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848706118355065,
+ "loss": 1.2442,
+ "theoretical_loss": 3.440055632595019,
+ "tokens_seen": 2137194496
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848545636910732,
+ "loss": 1.3446,
+ "theoretical_loss": 3.440047023574131,
+ "tokens_seen": 2137260032
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848385155466399,
+ "loss": 1.2072,
+ "theoretical_loss": 3.440038414891135,
+ "tokens_seen": 2137325568
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848224674022066,
+ "loss": 1.3568,
+ "theoretical_loss": 3.440029806546007,
+ "tokens_seen": 2137391104
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002848064192577733,
+ "loss": 1.2776,
+ "theoretical_loss": 3.440021198538724,
+ "tokens_seen": 2137456640
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028479037111334,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4400125908692614,
+ "tokens_seen": 2137522176
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002847743229689067,
+ "loss": 1.2445,
+ "theoretical_loss": 3.440003983537596,
+ "tokens_seen": 2137587712
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002847582748244734,
+ "loss": 1.284,
+ "theoretical_loss": 3.4399953765437052,
+ "tokens_seen": 2137653248
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028474222668004013,
+ "loss": 1.2469,
+ "theoretical_loss": 3.4399867698875646,
+ "tokens_seen": 2137718784
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028472617853560683,
+ "loss": 1.2293,
+ "theoretical_loss": 3.43997816356915,
+ "tokens_seen": 2137784320
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028471013039117353,
+ "loss": 1.4287,
+ "theoretical_loss": 3.439969557588439,
+ "tokens_seen": 2137849856
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028469408224674023,
+ "loss": 1.2873,
+ "theoretical_loss": 3.439960951945407,
+ "tokens_seen": 2137915392
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028467803410230693,
+ "loss": 1.3406,
+ "theoretical_loss": 3.4399523466400312,
+ "tokens_seen": 2137980928
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028466198595787363,
+ "loss": 1.2347,
+ "theoretical_loss": 3.4399437416722876,
+ "tokens_seen": 2138046464
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028464593781344033,
+ "loss": 1.3204,
+ "theoretical_loss": 3.439935137042152,
+ "tokens_seen": 2138112000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028462988966900703,
+ "loss": 1.3039,
+ "theoretical_loss": 3.4399265327496016,
+ "tokens_seen": 2138177536
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028461384152457373,
+ "loss": 1.2692,
+ "theoretical_loss": 3.4399179287946127,
+ "tokens_seen": 2138243072
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028459779338014043,
+ "loss": 1.3484,
+ "theoretical_loss": 3.4399093251771617,
+ "tokens_seen": 2138308608
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028458174523570713,
+ "loss": 1.2679,
+ "theoretical_loss": 3.439900721897225,
+ "tokens_seen": 2138374144
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028456569709127383,
+ "loss": 1.3801,
+ "theoretical_loss": 3.439892118954779,
+ "tokens_seen": 2138439680
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028454964894684054,
+ "loss": 1.2999,
+ "theoretical_loss": 3.4398835163497994,
+ "tokens_seen": 2138505216
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028453360080240724,
+ "loss": 1.3563,
+ "theoretical_loss": 3.4398749140822638,
+ "tokens_seen": 2138570752
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028451755265797394,
+ "loss": 1.2508,
+ "theoretical_loss": 3.439866312152148,
+ "tokens_seen": 2138636288
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028450150451354064,
+ "loss": 1.2227,
+ "theoretical_loss": 3.4398577105594286,
+ "tokens_seen": 2138701824
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028448545636910734,
+ "loss": 1.2314,
+ "theoretical_loss": 3.439849109304082,
+ "tokens_seen": 2138767360
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028446940822467404,
+ "loss": 1.1626,
+ "theoretical_loss": 3.4398405083860846,
+ "tokens_seen": 2138832896
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028445336008024074,
+ "loss": 1.317,
+ "theoretical_loss": 3.439831907805413,
+ "tokens_seen": 2138898432
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028443731193580744,
+ "loss": 1.3239,
+ "theoretical_loss": 3.439823307562043,
+ "tokens_seen": 2138963968
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028442126379137414,
+ "loss": 1.2671,
+ "theoretical_loss": 3.4398147076559518,
+ "tokens_seen": 2139029504
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028440521564694084,
+ "loss": 1.2787,
+ "theoretical_loss": 3.439806108087115,
+ "tokens_seen": 2139095040
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028438916750250754,
+ "loss": 1.3516,
+ "theoretical_loss": 3.43979750885551,
+ "tokens_seen": 2139160576
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028437311935807424,
+ "loss": 1.2763,
+ "theoretical_loss": 3.439788909961113,
+ "tokens_seen": 2139226112
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028435707121364094,
+ "loss": 1.3482,
+ "theoretical_loss": 3.4397803114039003,
+ "tokens_seen": 2139291648
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028434102306920764,
+ "loss": 1.2589,
+ "theoretical_loss": 3.4397717131838483,
+ "tokens_seen": 2139357184
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028432497492477434,
+ "loss": 1.2303,
+ "theoretical_loss": 3.4397631153009334,
+ "tokens_seen": 2139422720
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028430892678034104,
+ "loss": 1.2349,
+ "theoretical_loss": 3.439754517755132,
+ "tokens_seen": 2139488256
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028429287863590775,
+ "loss": 1.3468,
+ "theoretical_loss": 3.4397459205464207,
+ "tokens_seen": 2139553792
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028427683049147445,
+ "loss": 1.3202,
+ "theoretical_loss": 3.439737323674776,
+ "tokens_seen": 2139619328
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028426078234704115,
+ "loss": 1.3578,
+ "theoretical_loss": 3.4397287271401744,
+ "tokens_seen": 2139684864
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 716299,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2635315656661987,
+ "objective/train/theoretical_loss": 3.439720130942592,
+ "objective/train/tokens_used": 2160210400,
+ "theoretical_loss": 3.439720130942592,
+ "tokens_seen": 2139750400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028424473420260785,
+ "loss": 1.3092,
+ "theoretical_loss": 3.439720130942592,
+ "tokens_seen": 2139750400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002842286860581745,
+ "loss": 1.2773,
+ "theoretical_loss": 3.4397115350820053,
+ "tokens_seen": 2139815936
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002842126379137412,
+ "loss": 1.3772,
+ "theoretical_loss": 3.4397029395583916,
+ "tokens_seen": 2139881472
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841965897693079,
+ "loss": 1.33,
+ "theoretical_loss": 3.439694344371727,
+ "tokens_seen": 2139947008
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841805416248746,
+ "loss": 1.2345,
+ "theoretical_loss": 3.439685749521987,
+ "tokens_seen": 2140012544
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841644934804413,
+ "loss": 1.2941,
+ "theoretical_loss": 3.439677155009149,
+ "tokens_seen": 2140078080
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000284148445336008,
+ "loss": 1.291,
+ "theoretical_loss": 3.43966856083319,
+ "tokens_seen": 2140143616
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841323971915747,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4396599669940846,
+ "tokens_seen": 2140209152
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841163490471414,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4396513734918113,
+ "tokens_seen": 2140274688
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002841003009027081,
+ "loss": 1.3992,
+ "theoretical_loss": 3.439642780326346,
+ "tokens_seen": 2140340224
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002840842527582748,
+ "loss": 1.2595,
+ "theoretical_loss": 3.439634187497664,
+ "tokens_seen": 2140405760
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028406820461384155,
+ "loss": 1.2743,
+ "theoretical_loss": 3.439625595005743,
+ "tokens_seen": 2140471296
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028405215646940825,
+ "loss": 1.27,
+ "theoretical_loss": 3.4396170028505595,
+ "tokens_seen": 2140536832
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028403610832497496,
+ "loss": 1.1822,
+ "theoretical_loss": 3.4396084110320895,
+ "tokens_seen": 2140602368
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028402006018054166,
+ "loss": 1.2351,
+ "theoretical_loss": 3.4395998195503097,
+ "tokens_seen": 2140667904
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028400401203610836,
+ "loss": 1.2489,
+ "theoretical_loss": 3.439591228405197,
+ "tokens_seen": 2140733440
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028398796389167506,
+ "loss": 1.1456,
+ "theoretical_loss": 3.439582637596727,
+ "tokens_seen": 2140798976
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028397191574724176,
+ "loss": 1.2808,
+ "theoretical_loss": 3.439574047124877,
+ "tokens_seen": 2140864512
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028395586760280846,
+ "loss": 1.298,
+ "theoretical_loss": 3.439565456989623,
+ "tokens_seen": 2140930048
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028393981945837516,
+ "loss": 1.2826,
+ "theoretical_loss": 3.4395568671909422,
+ "tokens_seen": 2140995584
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028392377131394186,
+ "loss": 1.2551,
+ "theoretical_loss": 3.43954827772881,
+ "tokens_seen": 2141061120
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028390772316950856,
+ "loss": 1.315,
+ "theoretical_loss": 3.439539688603204,
+ "tokens_seen": 2141126656
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028389167502507526,
+ "loss": 1.2538,
+ "theoretical_loss": 3.4395310998141,
+ "tokens_seen": 2141192192
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028387562688064196,
+ "loss": 1.2835,
+ "theoretical_loss": 3.439522511361475,
+ "tokens_seen": 2141257728
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028385957873620866,
+ "loss": 1.1975,
+ "theoretical_loss": 3.439513923245305,
+ "tokens_seen": 2141323264
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028384353059177536,
+ "loss": 1.2396,
+ "theoretical_loss": 3.4395053354655674,
+ "tokens_seen": 2141388800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028382748244734206,
+ "loss": 1.2159,
+ "theoretical_loss": 3.4394967480222376,
+ "tokens_seen": 2141454336
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028381143430290876,
+ "loss": 1.3735,
+ "theoretical_loss": 3.4394881609152925,
+ "tokens_seen": 2141519872
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028379538615847547,
+ "loss": 1.4056,
+ "theoretical_loss": 3.4394795741447095,
+ "tokens_seen": 2141585408
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028377933801404217,
+ "loss": 1.3849,
+ "theoretical_loss": 3.4394709877104637,
+ "tokens_seen": 2141650944
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028376328986960887,
+ "loss": 1.2988,
+ "theoretical_loss": 3.4394624016125332,
+ "tokens_seen": 2141716480
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028374724172517557,
+ "loss": 1.3509,
+ "theoretical_loss": 3.439453815850893,
+ "tokens_seen": 2141782016
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028373119358074227,
+ "loss": 1.2107,
+ "theoretical_loss": 3.4394452304255205,
+ "tokens_seen": 2141847552
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028371514543630897,
+ "loss": 1.2127,
+ "theoretical_loss": 3.4394366453363925,
+ "tokens_seen": 2141913088
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028369909729187567,
+ "loss": 1.3388,
+ "theoretical_loss": 3.439428060583485,
+ "tokens_seen": 2141978624
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028368304914744237,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4394194761667745,
+ "tokens_seen": 2142044160
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028366700100300907,
+ "loss": 1.2409,
+ "theoretical_loss": 3.4394108920862374,
+ "tokens_seen": 2142109696
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028365095285857577,
+ "loss": 1.3456,
+ "theoretical_loss": 3.439402308341851,
+ "tokens_seen": 2142175232
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028363490471414247,
+ "loss": 1.3051,
+ "theoretical_loss": 3.4393937249335913,
+ "tokens_seen": 2142240768
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028361885656970917,
+ "loss": 1.253,
+ "theoretical_loss": 3.439385141861435,
+ "tokens_seen": 2142306304
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002836028084252758,
+ "loss": 1.2485,
+ "theoretical_loss": 3.4393765591253587,
+ "tokens_seen": 2142371840
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002835867602808425,
+ "loss": 1.2345,
+ "theoretical_loss": 3.4393679767253387,
+ "tokens_seen": 2142437376
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002835707121364092,
+ "loss": 1.3034,
+ "theoretical_loss": 3.439359394661352,
+ "tokens_seen": 2142502912
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002835546639919759,
+ "loss": 1.2671,
+ "theoretical_loss": 3.439350812933375,
+ "tokens_seen": 2142568448
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002835386158475426,
+ "loss": 1.3994,
+ "theoretical_loss": 3.439342231541384,
+ "tokens_seen": 2142633984
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002835225677031093,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4393336504853558,
+ "tokens_seen": 2142699520
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000283506519558676,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4393250697652666,
+ "tokens_seen": 2142765056
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834904714142427,
+ "loss": 1.3181,
+ "theoretical_loss": 3.4393164893810937,
+ "tokens_seen": 2142830592
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834744232698094,
+ "loss": 1.3779,
+ "theoretical_loss": 3.439307909332814,
+ "tokens_seen": 2142896128
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834583751253761,
+ "loss": 1.2088,
+ "theoretical_loss": 3.439299329620402,
+ "tokens_seen": 2142961664
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 717319,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1742408275604248,
+ "objective/train/theoretical_loss": 3.4392907502438366,
+ "objective/train/tokens_used": 2163487200,
+ "theoretical_loss": 3.4392907502438366,
+ "tokens_seen": 2143027200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834423269809428,
+ "loss": 1.1179,
+ "theoretical_loss": 3.4392907502438366,
+ "tokens_seen": 2143027200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834262788365095,
+ "loss": 1.3458,
+ "theoretical_loss": 3.439282171203093,
+ "tokens_seen": 2143092736
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002834102306920762,
+ "loss": 1.2117,
+ "theoretical_loss": 3.4392735924981483,
+ "tokens_seen": 2143158272
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002833941825476429,
+ "loss": 1.2574,
+ "theoretical_loss": 3.439265014128979,
+ "tokens_seen": 2143223808
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028337813440320963,
+ "loss": 1.2617,
+ "theoretical_loss": 3.4392564360955618,
+ "tokens_seen": 2143289344
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028336208625877633,
+ "loss": 1.3169,
+ "theoretical_loss": 3.439247858397873,
+ "tokens_seen": 2143354880
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028334603811434303,
+ "loss": 1.1802,
+ "theoretical_loss": 3.43923928103589,
+ "tokens_seen": 2143420416
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028332998996990973,
+ "loss": 1.2077,
+ "theoretical_loss": 3.4392307040095877,
+ "tokens_seen": 2143485952
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028331394182547643,
+ "loss": 1.236,
+ "theoretical_loss": 3.4392221273189447,
+ "tokens_seen": 2143551488
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028329789368104313,
+ "loss": 1.2567,
+ "theoretical_loss": 3.4392135509639363,
+ "tokens_seen": 2143617024
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028328184553660983,
+ "loss": 1.2374,
+ "theoretical_loss": 3.4392049749445395,
+ "tokens_seen": 2143682560
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028326579739217653,
+ "loss": 1.2954,
+ "theoretical_loss": 3.439196399260731,
+ "tokens_seen": 2143748096
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028324974924774323,
+ "loss": 1.2155,
+ "theoretical_loss": 3.4391878239124876,
+ "tokens_seen": 2143813632
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028323370110330993,
+ "loss": 1.2455,
+ "theoretical_loss": 3.4391792488997854,
+ "tokens_seen": 2143879168
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028321765295887663,
+ "loss": 1.2733,
+ "theoretical_loss": 3.439170674222601,
+ "tokens_seen": 2143944704
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028320160481444333,
+ "loss": 1.2501,
+ "theoretical_loss": 3.439162099880911,
+ "tokens_seen": 2144010240
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028318555667001004,
+ "loss": 1.2821,
+ "theoretical_loss": 3.4391535258746933,
+ "tokens_seen": 2144075776
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028316950852557674,
+ "loss": 1.1951,
+ "theoretical_loss": 3.4391449522039226,
+ "tokens_seen": 2144141312
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028315346038114344,
+ "loss": 1.2892,
+ "theoretical_loss": 3.439136378868577,
+ "tokens_seen": 2144206848
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028313741223671014,
+ "loss": 1.2367,
+ "theoretical_loss": 3.4391278058686323,
+ "tokens_seen": 2144272384
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028312136409227684,
+ "loss": 1.2295,
+ "theoretical_loss": 3.4391192332040648,
+ "tokens_seen": 2144337920
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028310531594784354,
+ "loss": 1.2583,
+ "theoretical_loss": 3.4391106608748525,
+ "tokens_seen": 2144403456
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028308926780341024,
+ "loss": 1.1736,
+ "theoretical_loss": 3.439102088880971,
+ "tokens_seen": 2144468992
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028307321965897694,
+ "loss": 1.2622,
+ "theoretical_loss": 3.439093517222397,
+ "tokens_seen": 2144534528
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028305717151454364,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4390849458991077,
+ "tokens_seen": 2144600064
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028304112337011034,
+ "loss": 1.3321,
+ "theoretical_loss": 3.4390763749110786,
+ "tokens_seen": 2144665600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028302507522567704,
+ "loss": 1.2596,
+ "theoretical_loss": 3.439067804258288,
+ "tokens_seen": 2144731136
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028300902708124374,
+ "loss": 1.2641,
+ "theoretical_loss": 3.4390592339407107,
+ "tokens_seen": 2144796672
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028299297893681044,
+ "loss": 1.1929,
+ "theoretical_loss": 3.439050663958325,
+ "tokens_seen": 2144862208
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028297693079237714,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4390420943111066,
+ "tokens_seen": 2144927744
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028296088264794384,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4390335249990325,
+ "tokens_seen": 2144993280
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028294483450351054,
+ "loss": 1.2738,
+ "theoretical_loss": 3.439024956022079,
+ "tokens_seen": 2145058816
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028292878635907725,
+ "loss": 1.1918,
+ "theoretical_loss": 3.439016387380223,
+ "tokens_seen": 2145124352
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028291273821464395,
+ "loss": 1.3033,
+ "theoretical_loss": 3.439007819073441,
+ "tokens_seen": 2145189888
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028289669007021065,
+ "loss": 1.1335,
+ "theoretical_loss": 3.4389992511017105,
+ "tokens_seen": 2145255424
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028288064192577735,
+ "loss": 1.3457,
+ "theoretical_loss": 3.438990683465007,
+ "tokens_seen": 2145320960
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028286459378134405,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4389821161633076,
+ "tokens_seen": 2145386496
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028284854563691075,
+ "loss": 1.1651,
+ "theoretical_loss": 3.4389735491965894,
+ "tokens_seen": 2145452032
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028283249749247745,
+ "loss": 1.2404,
+ "theoretical_loss": 3.438964982564828,
+ "tokens_seen": 2145517568
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028281644934804415,
+ "loss": 1.2693,
+ "theoretical_loss": 3.4389564162680015,
+ "tokens_seen": 2145583104
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028280040120361085,
+ "loss": 1.3149,
+ "theoretical_loss": 3.4389478503060853,
+ "tokens_seen": 2145648640
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028278435305917755,
+ "loss": 1.2982,
+ "theoretical_loss": 3.4389392846790567,
+ "tokens_seen": 2145714176
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028276830491474425,
+ "loss": 1.2044,
+ "theoretical_loss": 3.4389307193868923,
+ "tokens_seen": 2145779712
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028275225677031095,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4389221544295694,
+ "tokens_seen": 2145845248
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028273620862587765,
+ "loss": 1.1876,
+ "theoretical_loss": 3.438913589807063,
+ "tokens_seen": 2145910784
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028272016048144435,
+ "loss": 1.2871,
+ "theoretical_loss": 3.438905025519351,
+ "tokens_seen": 2145976320
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028270411233701105,
+ "loss": 1.303,
+ "theoretical_loss": 3.4388964615664106,
+ "tokens_seen": 2146041856
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028268806419257775,
+ "loss": 1.2778,
+ "theoretical_loss": 3.4388878979482174,
+ "tokens_seen": 2146107392
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028267201604814446,
+ "loss": 1.1871,
+ "theoretical_loss": 3.438879334664749,
+ "tokens_seen": 2146172928
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028265596790371116,
+ "loss": 1.2415,
+ "theoretical_loss": 3.438870771715981,
+ "tokens_seen": 2146238464
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 718519,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2924401760101318,
+ "objective/train/theoretical_loss": 3.4388622091018908,
+ "objective/train/tokens_used": 2166764000,
+ "theoretical_loss": 3.4388622091018908,
+ "tokens_seen": 2146304000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028263991975927786,
+ "loss": 1.282,
+ "theoretical_loss": 3.4388622091018908,
+ "tokens_seen": 2146304000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028262387161484456,
+ "loss": 1.3103,
+ "theoretical_loss": 3.438853646822455,
+ "tokens_seen": 2146369536
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028260782347041126,
+ "loss": 1.304,
+ "theoretical_loss": 3.4388450848776504,
+ "tokens_seen": 2146435072
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028259177532597796,
+ "loss": 1.2821,
+ "theoretical_loss": 3.4388365232674536,
+ "tokens_seen": 2146500608
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028257572718154466,
+ "loss": 1.2281,
+ "theoretical_loss": 3.4388279619918416,
+ "tokens_seen": 2146566144
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028255967903711136,
+ "loss": 1.3116,
+ "theoretical_loss": 3.4388194010507904,
+ "tokens_seen": 2146631680
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028254363089267806,
+ "loss": 1.181,
+ "theoretical_loss": 3.4388108404442774,
+ "tokens_seen": 2146697216
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028252758274824476,
+ "loss": 1.2712,
+ "theoretical_loss": 3.438802280172279,
+ "tokens_seen": 2146762752
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028251153460381146,
+ "loss": 1.1706,
+ "theoretical_loss": 3.438793720234772,
+ "tokens_seen": 2146828288
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028249548645937816,
+ "loss": 1.2739,
+ "theoretical_loss": 3.438785160631733,
+ "tokens_seen": 2146893824
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028247943831494486,
+ "loss": 1.3052,
+ "theoretical_loss": 3.4387766013631387,
+ "tokens_seen": 2146959360
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028246339017051156,
+ "loss": 1.1615,
+ "theoretical_loss": 3.4387680424289666,
+ "tokens_seen": 2147024896
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028244734202607826,
+ "loss": 1.1769,
+ "theoretical_loss": 3.438759483829192,
+ "tokens_seen": 2147090432
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028243129388164496,
+ "loss": 1.2976,
+ "theoretical_loss": 3.438750925563793,
+ "tokens_seen": 2147155968
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028241524573721167,
+ "loss": 1.2085,
+ "theoretical_loss": 3.438742367632745,
+ "tokens_seen": 2147221504
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028239919759277837,
+ "loss": 1.389,
+ "theoretical_loss": 3.438733810036026,
+ "tokens_seen": 2147287040
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028238314944834507,
+ "loss": 1.317,
+ "theoretical_loss": 3.4387252527736125,
+ "tokens_seen": 2147352576
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028236710130391177,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4387166958454802,
+ "tokens_seen": 2147418112
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028235105315947847,
+ "loss": 1.2478,
+ "theoretical_loss": 3.438708139251607,
+ "tokens_seen": 2147483648
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028233500501504517,
+ "loss": 1.2929,
+ "theoretical_loss": 3.438699582991969,
+ "tokens_seen": 2147549184
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028231895687061187,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4386910270665436,
+ "tokens_seen": 2147614720
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028230290872617857,
+ "loss": 1.2908,
+ "theoretical_loss": 3.438682471475307,
+ "tokens_seen": 2147680256
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028228686058174527,
+ "loss": 1.2017,
+ "theoretical_loss": 3.4386739162182356,
+ "tokens_seen": 2147745792
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028227081243731197,
+ "loss": 1.1989,
+ "theoretical_loss": 3.438665361295307,
+ "tokens_seen": 2147811328
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028225476429287867,
+ "loss": 1.3864,
+ "theoretical_loss": 3.438656806706497,
+ "tokens_seen": 2147876864
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028223871614844537,
+ "loss": 1.2446,
+ "theoretical_loss": 3.4386482524517836,
+ "tokens_seen": 2147942400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002822226680040121,
+ "loss": 1.333,
+ "theoretical_loss": 3.4386396985311425,
+ "tokens_seen": 2148007936
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002822066198595788,
+ "loss": 1.2523,
+ "theoretical_loss": 3.4386311449445515,
+ "tokens_seen": 2148073472
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821905717151454,
+ "loss": 1.2525,
+ "theoretical_loss": 3.4386225916919857,
+ "tokens_seen": 2148139008
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821745235707121,
+ "loss": 1.2341,
+ "theoretical_loss": 3.438614038773424,
+ "tokens_seen": 2148204544
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821584754262788,
+ "loss": 1.2057,
+ "theoretical_loss": 3.438605486188841,
+ "tokens_seen": 2148270080
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821424272818455,
+ "loss": 1.4116,
+ "theoretical_loss": 3.438596933938215,
+ "tokens_seen": 2148335616
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821263791374122,
+ "loss": 1.2248,
+ "theoretical_loss": 3.438588382021522,
+ "tokens_seen": 2148401152
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002821103309929789,
+ "loss": 1.2971,
+ "theoretical_loss": 3.43857983043874,
+ "tokens_seen": 2148466688
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002820942828485456,
+ "loss": 1.2306,
+ "theoretical_loss": 3.4385712791898437,
+ "tokens_seen": 2148532224
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002820782347041123,
+ "loss": 1.2755,
+ "theoretical_loss": 3.4385627282748112,
+ "tokens_seen": 2148597760
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000282062186559679,
+ "loss": 1.1841,
+ "theoretical_loss": 3.4385541776936197,
+ "tokens_seen": 2148663296
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002820461384152457,
+ "loss": 1.304,
+ "theoretical_loss": 3.4385456274462447,
+ "tokens_seen": 2148728832
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002820300902708124,
+ "loss": 1.2747,
+ "theoretical_loss": 3.438537077532664,
+ "tokens_seen": 2148794368
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028201404212637913,
+ "loss": 1.1649,
+ "theoretical_loss": 3.4385285279528546,
+ "tokens_seen": 2148859904
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028199799398194583,
+ "loss": 1.2288,
+ "theoretical_loss": 3.438519978706792,
+ "tokens_seen": 2148925440
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028198194583751253,
+ "loss": 1.3736,
+ "theoretical_loss": 3.4385114297944543,
+ "tokens_seen": 2148990976
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028196589769307923,
+ "loss": 1.2906,
+ "theoretical_loss": 3.4385028812158174,
+ "tokens_seen": 2149056512
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028194984954864593,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4384943329708584,
+ "tokens_seen": 2149122048
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028193380140421263,
+ "loss": 1.2492,
+ "theoretical_loss": 3.4384857850595543,
+ "tokens_seen": 2149187584
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028191775325977933,
+ "loss": 1.2658,
+ "theoretical_loss": 3.4384772374818815,
+ "tokens_seen": 2149253120
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028190170511534603,
+ "loss": 1.2749,
+ "theoretical_loss": 3.4384686902378174,
+ "tokens_seen": 2149318656
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028188565697091273,
+ "loss": 1.2117,
+ "theoretical_loss": 3.438460143327338,
+ "tokens_seen": 2149384192
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028186960882647943,
+ "loss": 1.2582,
+ "theoretical_loss": 3.438451596750421,
+ "tokens_seen": 2149449728
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028185356068204613,
+ "loss": 1.2535,
+ "theoretical_loss": 3.4384430505070425,
+ "tokens_seen": 2149515264
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 719453,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.367647409439087,
+ "objective/train/theoretical_loss": 3.4384345045971796,
+ "objective/train/tokens_used": 2170040800,
+ "theoretical_loss": 3.4384345045971796,
+ "tokens_seen": 2149580800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028183751253761283,
+ "loss": 1.255,
+ "theoretical_loss": 3.4384345045971796,
+ "tokens_seen": 2149580800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028182146439317953,
+ "loss": 1.1786,
+ "theoretical_loss": 3.4384259590208095,
+ "tokens_seen": 2149646336
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028180541624874624,
+ "loss": 1.2142,
+ "theoretical_loss": 3.438417413777908,
+ "tokens_seen": 2149711872
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028178936810431294,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4384088688684535,
+ "tokens_seen": 2149777408
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028177331995987964,
+ "loss": 1.2848,
+ "theoretical_loss": 3.438400324292421,
+ "tokens_seen": 2149842944
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028175727181544634,
+ "loss": 1.3754,
+ "theoretical_loss": 3.4383917800497885,
+ "tokens_seen": 2149908480
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028174122367101304,
+ "loss": 1.2372,
+ "theoretical_loss": 3.438383236140533,
+ "tokens_seen": 2149974016
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028172517552657974,
+ "loss": 1.2769,
+ "theoretical_loss": 3.43837469256463,
+ "tokens_seen": 2150039552
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028170912738214644,
+ "loss": 1.2896,
+ "theoretical_loss": 3.4383661493220576,
+ "tokens_seen": 2150105088
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028169307923771314,
+ "loss": 1.2648,
+ "theoretical_loss": 3.4383576064127923,
+ "tokens_seen": 2150170624
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002816770310932799,
+ "loss": 1.2132,
+ "theoretical_loss": 3.438349063836811,
+ "tokens_seen": 2150236160
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002816609829488466,
+ "loss": 1.2764,
+ "theoretical_loss": 3.43834052159409,
+ "tokens_seen": 2150301696
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002816449348044133,
+ "loss": 1.2283,
+ "theoretical_loss": 3.438331979684607,
+ "tokens_seen": 2150367232
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028162888665998,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4383234381083376,
+ "tokens_seen": 2150432768
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002816128385155467,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4383148968652604,
+ "tokens_seen": 2150498304
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002815967903711134,
+ "loss": 1.2463,
+ "theoretical_loss": 3.438306355955351,
+ "tokens_seen": 2150563840
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002815807422266801,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4382978153785864,
+ "tokens_seen": 2150629376
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028156469408224674,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4382892751349434,
+ "tokens_seen": 2150694912
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028154864593781345,
+ "loss": 1.3914,
+ "theoretical_loss": 3.4382807352243994,
+ "tokens_seen": 2150760448
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028153259779338015,
+ "loss": 1.211,
+ "theoretical_loss": 3.438272195646931,
+ "tokens_seen": 2150825984
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028151654964894685,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4382636564025146,
+ "tokens_seen": 2150891520
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028150050150451355,
+ "loss": 1.3161,
+ "theoretical_loss": 3.4382551174911278,
+ "tokens_seen": 2150957056
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028148445336008025,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4382465789127474,
+ "tokens_seen": 2151022592
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028146840521564695,
+ "loss": 1.3453,
+ "theoretical_loss": 3.4382380406673496,
+ "tokens_seen": 2151088128
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028145235707121365,
+ "loss": 1.238,
+ "theoretical_loss": 3.4382295027549112,
+ "tokens_seen": 2151153664
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028143630892678035,
+ "loss": 1.3722,
+ "theoretical_loss": 3.43822096517541,
+ "tokens_seen": 2151219200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028142026078234705,
+ "loss": 1.2632,
+ "theoretical_loss": 3.438212427928822,
+ "tokens_seen": 2151284736
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028140421263791375,
+ "loss": 1.1628,
+ "theoretical_loss": 3.4382038910151254,
+ "tokens_seen": 2151350272
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028138816449348045,
+ "loss": 1.3094,
+ "theoretical_loss": 3.4381953544342956,
+ "tokens_seen": 2151415808
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028137211634904715,
+ "loss": 1.3061,
+ "theoretical_loss": 3.43818681818631,
+ "tokens_seen": 2151481344
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028135606820461385,
+ "loss": 1.2126,
+ "theoretical_loss": 3.438178282271145,
+ "tokens_seen": 2151546880
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028134002006018055,
+ "loss": 1.2818,
+ "theoretical_loss": 3.4381697466887786,
+ "tokens_seen": 2151612416
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028132397191574725,
+ "loss": 1.2908,
+ "theoretical_loss": 3.438161211439187,
+ "tokens_seen": 2151677952
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028130792377131396,
+ "loss": 1.2637,
+ "theoretical_loss": 3.438152676522347,
+ "tokens_seen": 2151743488
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028129187562688066,
+ "loss": 1.197,
+ "theoretical_loss": 3.438144141938236,
+ "tokens_seen": 2151809024
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028127582748244736,
+ "loss": 1.2516,
+ "theoretical_loss": 3.4381356076868306,
+ "tokens_seen": 2151874560
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028125977933801406,
+ "loss": 1.1802,
+ "theoretical_loss": 3.4381270737681073,
+ "tokens_seen": 2151940096
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028124373119358076,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4381185401820438,
+ "tokens_seen": 2152005632
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028122768304914746,
+ "loss": 1.3302,
+ "theoretical_loss": 3.438110006928616,
+ "tokens_seen": 2152071168
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028121163490471416,
+ "loss": 1.3128,
+ "theoretical_loss": 3.438101474007802,
+ "tokens_seen": 2152136704
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028119558676028086,
+ "loss": 1.2822,
+ "theoretical_loss": 3.4380929414195776,
+ "tokens_seen": 2152202240
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028117953861584756,
+ "loss": 1.2157,
+ "theoretical_loss": 3.4380844091639204,
+ "tokens_seen": 2152267776
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028116349047141426,
+ "loss": 1.2927,
+ "theoretical_loss": 3.438075877240807,
+ "tokens_seen": 2152333312
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028114744232698096,
+ "loss": 1.2401,
+ "theoretical_loss": 3.4380673456502144,
+ "tokens_seen": 2152398848
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028113139418254766,
+ "loss": 1.1975,
+ "theoretical_loss": 3.43805881439212,
+ "tokens_seen": 2152464384
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028111534603811436,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4380502834664997,
+ "tokens_seen": 2152529920
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028109929789368106,
+ "loss": 1.2765,
+ "theoretical_loss": 3.438041752873331,
+ "tokens_seen": 2152595456
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028108324974924776,
+ "loss": 1.2454,
+ "theoretical_loss": 3.4380332226125914,
+ "tokens_seen": 2152660992
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028106720160481446,
+ "loss": 1.2165,
+ "theoretical_loss": 3.4380246926842566,
+ "tokens_seen": 2152726528
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028105115346038117,
+ "loss": 1.2602,
+ "theoretical_loss": 3.4380161630883044,
+ "tokens_seen": 2152792064
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 720672,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3936753273010254,
+ "objective/train/theoretical_loss": 3.4380076338247116,
+ "objective/train/tokens_used": 2173317600,
+ "theoretical_loss": 3.4380076338247116,
+ "tokens_seen": 2152857600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028103510531594787,
+ "loss": 1.3479,
+ "theoretical_loss": 3.4380076338247116,
+ "tokens_seen": 2152857600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028101905717151457,
+ "loss": 1.3889,
+ "theoretical_loss": 3.4379991048934544,
+ "tokens_seen": 2152923136
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028100300902708127,
+ "loss": 1.3055,
+ "theoretical_loss": 3.437990576294511,
+ "tokens_seen": 2152988672
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028098696088264797,
+ "loss": 1.294,
+ "theoretical_loss": 3.4379820480278576,
+ "tokens_seen": 2153054208
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028097091273821467,
+ "loss": 1.2262,
+ "theoretical_loss": 3.437973520093471,
+ "tokens_seen": 2153119744
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028095486459378137,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4379649924913283,
+ "tokens_seen": 2153185280
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028093881644934807,
+ "loss": 1.2358,
+ "theoretical_loss": 3.437956465221407,
+ "tokens_seen": 2153250816
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028092276830491477,
+ "loss": 1.4183,
+ "theoretical_loss": 3.437947938283683,
+ "tokens_seen": 2153316352
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028090672016048147,
+ "loss": 1.3137,
+ "theoretical_loss": 3.4379394116781343,
+ "tokens_seen": 2153381888
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028089067201604817,
+ "loss": 1.2573,
+ "theoretical_loss": 3.437930885404737,
+ "tokens_seen": 2153447424
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028087462387161487,
+ "loss": 1.2296,
+ "theoretical_loss": 3.4379223594634687,
+ "tokens_seen": 2153512960
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028085857572718157,
+ "loss": 1.173,
+ "theoretical_loss": 3.4379138338543056,
+ "tokens_seen": 2153578496
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002808425275827483,
+ "loss": 1.2997,
+ "theoretical_loss": 3.4379053085772258,
+ "tokens_seen": 2153644032
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000280826479438315,
+ "loss": 1.3307,
+ "theoretical_loss": 3.437896783632205,
+ "tokens_seen": 2153709568
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002808104312938817,
+ "loss": 1.4054,
+ "theoretical_loss": 3.4378882590192212,
+ "tokens_seen": 2153775104
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002807943831494483,
+ "loss": 1.2966,
+ "theoretical_loss": 3.4378797347382504,
+ "tokens_seen": 2153840640
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000280778335005015,
+ "loss": 1.215,
+ "theoretical_loss": 3.437871210789271,
+ "tokens_seen": 2153906176
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002807622868605817,
+ "loss": 1.2402,
+ "theoretical_loss": 3.4378626871722577,
+ "tokens_seen": 2153971712
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002807462387161484,
+ "loss": 1.2053,
+ "theoretical_loss": 3.43785416388719,
+ "tokens_seen": 2154037248
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002807301905717151,
+ "loss": 1.2436,
+ "theoretical_loss": 3.437845640934043,
+ "tokens_seen": 2154102784
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002807141424272818,
+ "loss": 1.2136,
+ "theoretical_loss": 3.4378371183127947,
+ "tokens_seen": 2154168320
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002806980942828485,
+ "loss": 1.2321,
+ "theoretical_loss": 3.4378285960234214,
+ "tokens_seen": 2154233856
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002806820461384152,
+ "loss": 1.1644,
+ "theoretical_loss": 3.437820074065901,
+ "tokens_seen": 2154299392
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002806659979939819,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4378115524402095,
+ "tokens_seen": 2154364928
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002806499498495486,
+ "loss": 1.3015,
+ "theoretical_loss": 3.4378030311463244,
+ "tokens_seen": 2154430464
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028063390170511533,
+ "loss": 1.3441,
+ "theoretical_loss": 3.4377945101842227,
+ "tokens_seen": 2154496000
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028061785356068203,
+ "loss": 1.1304,
+ "theoretical_loss": 3.4377859895538814,
+ "tokens_seen": 2154561536
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028060180541624873,
+ "loss": 1.3071,
+ "theoretical_loss": 3.437777469255277,
+ "tokens_seen": 2154627072
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028058575727181543,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4377689492883867,
+ "tokens_seen": 2154692608
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028056970912738213,
+ "loss": 1.2034,
+ "theoretical_loss": 3.4377604296531885,
+ "tokens_seen": 2154758144
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028055366098294883,
+ "loss": 1.2403,
+ "theoretical_loss": 3.437751910349658,
+ "tokens_seen": 2154823680
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028053761283851553,
+ "loss": 1.1638,
+ "theoretical_loss": 3.4377433913777726,
+ "tokens_seen": 2154889216
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028052156469408223,
+ "loss": 1.2091,
+ "theoretical_loss": 3.4377348727375097,
+ "tokens_seen": 2154954752
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028050551654964893,
+ "loss": 1.343,
+ "theoretical_loss": 3.437726354428846,
+ "tokens_seen": 2155020288
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028048946840521563,
+ "loss": 1.2072,
+ "theoretical_loss": 3.4377178364517587,
+ "tokens_seen": 2155085824
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002804734202607824,
+ "loss": 1.3629,
+ "theoretical_loss": 3.437709318806225,
+ "tokens_seen": 2155151360
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002804573721163491,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4377008014922206,
+ "tokens_seen": 2155216896
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002804413239719158,
+ "loss": 1.2546,
+ "theoretical_loss": 3.4376922845097244,
+ "tokens_seen": 2155282432
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002804252758274825,
+ "loss": 1.271,
+ "theoretical_loss": 3.4376837678587124,
+ "tokens_seen": 2155347968
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002804092276830492,
+ "loss": 1.2886,
+ "theoretical_loss": 3.4376752515391615,
+ "tokens_seen": 2155413504
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002803931795386159,
+ "loss": 1.1938,
+ "theoretical_loss": 3.437666735551049,
+ "tokens_seen": 2155479040
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002803771313941826,
+ "loss": 1.2558,
+ "theoretical_loss": 3.4376582198943524,
+ "tokens_seen": 2155544576
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002803610832497493,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4376497045690475,
+ "tokens_seen": 2155610112
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000280345035105316,
+ "loss": 1.2987,
+ "theoretical_loss": 3.437641189575112,
+ "tokens_seen": 2155675648
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002803289869608827,
+ "loss": 1.2198,
+ "theoretical_loss": 3.437632674912524,
+ "tokens_seen": 2155741184
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002803129388164494,
+ "loss": 1.2366,
+ "theoretical_loss": 3.437624160581259,
+ "tokens_seen": 2155806720
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802968906720161,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4376156465812944,
+ "tokens_seen": 2155872256
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802808425275828,
+ "loss": 1.3122,
+ "theoretical_loss": 3.437607132912608,
+ "tokens_seen": 2155937792
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802647943831495,
+ "loss": 1.2743,
+ "theoretical_loss": 3.4375986195751755,
+ "tokens_seen": 2156003328
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802487462387162,
+ "loss": 1.2334,
+ "theoretical_loss": 3.4375901065689747,
+ "tokens_seen": 2156068864
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 721882,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2278430461883545,
+ "objective/train/theoretical_loss": 3.4375815938939827,
+ "objective/train/tokens_used": 2176594400,
+ "theoretical_loss": 3.4375815938939827,
+ "tokens_seen": 2156134400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802326980942829,
+ "loss": 1.2663,
+ "theoretical_loss": 3.4375815938939827,
+ "tokens_seen": 2156134400
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802166499498496,
+ "loss": 1.1849,
+ "theoretical_loss": 3.437573081550177,
+ "tokens_seen": 2156199936
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002802006018054163,
+ "loss": 1.3121,
+ "theoretical_loss": 3.4375645695375336,
+ "tokens_seen": 2156265472
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000280184553660983,
+ "loss": 1.2944,
+ "theoretical_loss": 3.43755605785603,
+ "tokens_seen": 2156331008
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002801685055165497,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4375475465056438,
+ "tokens_seen": 2156396544
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028015245737211635,
+ "loss": 1.3913,
+ "theoretical_loss": 3.4375390354863513,
+ "tokens_seen": 2156462080
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028013640922768305,
+ "loss": 1.1688,
+ "theoretical_loss": 3.43753052479813,
+ "tokens_seen": 2156527616
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028012036108324975,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4375220144409564,
+ "tokens_seen": 2156593152
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028010431293881645,
+ "loss": 1.1509,
+ "theoretical_loss": 3.4375135044148086,
+ "tokens_seen": 2156658688
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028008826479438315,
+ "loss": 1.3136,
+ "theoretical_loss": 3.4375049947196628,
+ "tokens_seen": 2156724224
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028007221664994985,
+ "loss": 1.289,
+ "theoretical_loss": 3.437496485355496,
+ "tokens_seen": 2156789760
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028005616850551655,
+ "loss": 1.2396,
+ "theoretical_loss": 3.437487976322286,
+ "tokens_seen": 2156855296
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028004012036108325,
+ "loss": 1.1943,
+ "theoretical_loss": 3.437479467620009,
+ "tokens_seen": 2156920832
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028002407221664995,
+ "loss": 1.3249,
+ "theoretical_loss": 3.437470959248643,
+ "tokens_seen": 2156986368
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00028000802407221665,
+ "loss": 1.1917,
+ "theoretical_loss": 3.437462451208164,
+ "tokens_seen": 2157051904
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027999197592778335,
+ "loss": 1.3302,
+ "theoretical_loss": 3.4374539434985505,
+ "tokens_seen": 2157117440
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027997592778335005,
+ "loss": 1.1716,
+ "theoretical_loss": 3.437445436119778,
+ "tokens_seen": 2157182976
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027995987963891675,
+ "loss": 1.1639,
+ "theoretical_loss": 3.4374369290718247,
+ "tokens_seen": 2157248512
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027994383149448345,
+ "loss": 1.327,
+ "theoretical_loss": 3.4374284223546674,
+ "tokens_seen": 2157314048
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027992778335005016,
+ "loss": 1.3163,
+ "theoretical_loss": 3.437419915968283,
+ "tokens_seen": 2157379584
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027991173520561686,
+ "loss": 1.309,
+ "theoretical_loss": 3.4374114099126487,
+ "tokens_seen": 2157445120
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027989568706118356,
+ "loss": 1.3796,
+ "theoretical_loss": 3.437402904187742,
+ "tokens_seen": 2157510656
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027987963891675026,
+ "loss": 1.287,
+ "theoretical_loss": 3.437394398793539,
+ "tokens_seen": 2157576192
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027986359077231696,
+ "loss": 1.239,
+ "theoretical_loss": 3.4373858937300175,
+ "tokens_seen": 2157641728
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027984754262788366,
+ "loss": 1.2294,
+ "theoretical_loss": 3.4373773889971546,
+ "tokens_seen": 2157707264
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027983149448345036,
+ "loss": 1.2523,
+ "theoretical_loss": 3.4373688845949273,
+ "tokens_seen": 2157772800
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027981544633901706,
+ "loss": 1.2528,
+ "theoretical_loss": 3.437360380523313,
+ "tokens_seen": 2157838336
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027979939819458376,
+ "loss": 1.2704,
+ "theoretical_loss": 3.437351876782288,
+ "tokens_seen": 2157903872
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027978335005015046,
+ "loss": 1.1192,
+ "theoretical_loss": 3.43734337337183,
+ "tokens_seen": 2157969408
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027976730190571716,
+ "loss": 1.3242,
+ "theoretical_loss": 3.4373348702919166,
+ "tokens_seen": 2158034944
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027975125376128386,
+ "loss": 1.3475,
+ "theoretical_loss": 3.4373263675425236,
+ "tokens_seen": 2158100480
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027973520561685056,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4373178651236294,
+ "tokens_seen": 2158166016
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027971915747241726,
+ "loss": 1.2826,
+ "theoretical_loss": 3.4373093630352103,
+ "tokens_seen": 2158231552
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027970310932798396,
+ "loss": 1.2845,
+ "theoretical_loss": 3.4373008612772438,
+ "tokens_seen": 2158297088
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027968706118355066,
+ "loss": 1.3971,
+ "theoretical_loss": 3.4372923598497067,
+ "tokens_seen": 2158362624
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027967101303911737,
+ "loss": 1.3146,
+ "theoretical_loss": 3.4372838587525765,
+ "tokens_seen": 2158428160
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027965496489468407,
+ "loss": 1.3108,
+ "theoretical_loss": 3.4372753579858304,
+ "tokens_seen": 2158493696
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027963891675025077,
+ "loss": 1.1798,
+ "theoretical_loss": 3.437266857549445,
+ "tokens_seen": 2158559232
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027962286860581747,
+ "loss": 1.1867,
+ "theoretical_loss": 3.437258357443398,
+ "tokens_seen": 2158624768
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027960682046138417,
+ "loss": 1.3677,
+ "theoretical_loss": 3.437249857667666,
+ "tokens_seen": 2158690304
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027959077231695087,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4372413582222263,
+ "tokens_seen": 2158755840
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027957472417251757,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4372328591070564,
+ "tokens_seen": 2158821376
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027955867602808427,
+ "loss": 1.333,
+ "theoretical_loss": 3.4372243603221335,
+ "tokens_seen": 2158886912
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027954262788365097,
+ "loss": 1.2125,
+ "theoretical_loss": 3.437215861867434,
+ "tokens_seen": 2158952448
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027952657973921767,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4372073637429352,
+ "tokens_seen": 2159017984
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027951053159478437,
+ "loss": 1.3248,
+ "theoretical_loss": 3.437198865948615,
+ "tokens_seen": 2159083520
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.00027949448345035107,
+ "loss": 1.2228,
+ "theoretical_loss": 3.4371903684844503,
+ "tokens_seen": 2159149056
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002794784353059178,
+ "loss": 1.2415,
+ "theoretical_loss": 3.4371818713504174,
+ "tokens_seen": 2159214592
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002794623871614845,
+ "loss": 1.2723,
+ "theoretical_loss": 3.4371733745464947,
+ "tokens_seen": 2159280128
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002794463390170512,
+ "loss": 1.2951,
+ "theoretical_loss": 3.4371648780726582,
+ "tokens_seen": 2159345664
+ },
+ {
+ "epoch": 0.65,
+ "objective/train/docs_used": 722801,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2955522537231445,
+ "objective/train/theoretical_loss": 3.4371563819288857,
+ "objective/train/tokens_used": 2179871200,
+ "theoretical_loss": 3.4371563819288857,
+ "tokens_seen": 2159411200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002794302908726179,
+ "loss": 1.324,
+ "theoretical_loss": 3.4371563819288857,
+ "tokens_seen": 2159411200
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002794142427281846,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4371478861151545,
+ "tokens_seen": 2159476736
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793981945837513,
+ "loss": 1.2453,
+ "theoretical_loss": 3.437139390631441,
+ "tokens_seen": 2159542272
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793821464393179,
+ "loss": 1.2313,
+ "theoretical_loss": 3.4371308954777233,
+ "tokens_seen": 2159607808
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793660982948846,
+ "loss": 1.3156,
+ "theoretical_loss": 3.437122400653978,
+ "tokens_seen": 2159673344
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793500501504513,
+ "loss": 1.268,
+ "theoretical_loss": 3.4371139061601825,
+ "tokens_seen": 2159738880
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000279334002006018,
+ "loss": 1.2093,
+ "theoretical_loss": 3.4371054119963143,
+ "tokens_seen": 2159804416
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793179538615847,
+ "loss": 1.2553,
+ "theoretical_loss": 3.4370969181623496,
+ "tokens_seen": 2159869952
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002793019057171514,
+ "loss": 1.1994,
+ "theoretical_loss": 3.4370884246582665,
+ "tokens_seen": 2159935488
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002792858575727181,
+ "loss": 1.2408,
+ "theoretical_loss": 3.4370799314840417,
+ "tokens_seen": 2160001024
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002792698094282849,
+ "loss": 1.3032,
+ "theoretical_loss": 3.437071438639652,
+ "tokens_seen": 2160066560
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002792537612838516,
+ "loss": 1.3169,
+ "theoretical_loss": 3.437062946125076,
+ "tokens_seen": 2160132096
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002792377131394183,
+ "loss": 1.3403,
+ "theoretical_loss": 3.4370544539402896,
+ "tokens_seen": 2160197632
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000279221664994985,
+ "loss": 1.2272,
+ "theoretical_loss": 3.4370459620852705,
+ "tokens_seen": 2160263168
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002792056168505517,
+ "loss": 1.2813,
+ "theoretical_loss": 3.4370374705599955,
+ "tokens_seen": 2160328704
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791895687061184,
+ "loss": 1.2493,
+ "theoretical_loss": 3.437028979364442,
+ "tokens_seen": 2160394240
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791735205616851,
+ "loss": 1.1897,
+ "theoretical_loss": 3.4370204884985873,
+ "tokens_seen": 2160459776
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791574724172518,
+ "loss": 1.3055,
+ "theoretical_loss": 3.4370119979624087,
+ "tokens_seen": 2160525312
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791414242728185,
+ "loss": 1.3155,
+ "theoretical_loss": 3.4370035077558834,
+ "tokens_seen": 2160590848
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791253761283852,
+ "loss": 1.2721,
+ "theoretical_loss": 3.436995017878988,
+ "tokens_seen": 2160656384
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002791093279839519,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4369865283317003,
+ "tokens_seen": 2160721920
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002790932798395186,
+ "loss": 1.3446,
+ "theoretical_loss": 3.4369780391139977,
+ "tokens_seen": 2160787456
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002790772316950853,
+ "loss": 1.3699,
+ "theoretical_loss": 3.436969550225857,
+ "tokens_seen": 2160852992
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.000279061183550652,
+ "loss": 1.2372,
+ "theoretical_loss": 3.436961061667255,
+ "tokens_seen": 2160918528
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002790451354062187,
+ "loss": 1.3306,
+ "theoretical_loss": 3.4369525734381696,
+ "tokens_seen": 2160984064
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002790290872617854,
+ "loss": 1.2613,
+ "theoretical_loss": 3.436944085538578,
+ "tokens_seen": 2161049600
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002790130391173521,
+ "loss": 1.2156,
+ "theoretical_loss": 3.436935597968457,
+ "tokens_seen": 2161115136
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002789969909729188,
+ "loss": 1.2649,
+ "theoretical_loss": 3.436927110727784,
+ "tokens_seen": 2161180672
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002789809428284855,
+ "loss": 1.2021,
+ "theoretical_loss": 3.436918623816537,
+ "tokens_seen": 2161246208
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002789648946840522,
+ "loss": 1.2299,
+ "theoretical_loss": 3.436910137234692,
+ "tokens_seen": 2161311744
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002789488465396189,
+ "loss": 1.3193,
+ "theoretical_loss": 3.4369016509822266,
+ "tokens_seen": 2161377280
+ },
+ {
+ "epoch": 0.65,
+ "learning_rate": 0.0002789327983951856,
+ "loss": 1.3036,
+ "theoretical_loss": 3.4368931650591183,
+ "tokens_seen": 2161442816
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002789167502507523,
+ "loss": 1.2368,
+ "theoretical_loss": 3.436884679465344,
+ "tokens_seen": 2161508352
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000278900702106319,
+ "loss": 1.2621,
+ "theoretical_loss": 3.4368761942008814,
+ "tokens_seen": 2161573888
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788846539618857,
+ "loss": 1.1959,
+ "theoretical_loss": 3.436867709265707,
+ "tokens_seen": 2161639424
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788686058174524,
+ "loss": 1.3286,
+ "theoretical_loss": 3.436859224659799,
+ "tokens_seen": 2161704960
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788525576730191,
+ "loss": 1.1778,
+ "theoretical_loss": 3.436850740383134,
+ "tokens_seen": 2161770496
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788365095285858,
+ "loss": 1.3236,
+ "theoretical_loss": 3.436842256435689,
+ "tokens_seen": 2161836032
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788204613841525,
+ "loss": 1.2618,
+ "theoretical_loss": 3.4368337728174425,
+ "tokens_seen": 2161901568
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002788044132397192,
+ "loss": 1.2125,
+ "theoretical_loss": 3.43682528952837,
+ "tokens_seen": 2161967104
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002787883650952859,
+ "loss": 1.3596,
+ "theoretical_loss": 3.43681680656845,
+ "tokens_seen": 2162032640
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002787723169508526,
+ "loss": 1.1955,
+ "theoretical_loss": 3.4368083239376594,
+ "tokens_seen": 2162098176
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027875626880641925,
+ "loss": 1.1608,
+ "theoretical_loss": 3.4367998416359753,
+ "tokens_seen": 2162163712
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027874022066198595,
+ "loss": 1.2924,
+ "theoretical_loss": 3.436791359663375,
+ "tokens_seen": 2162229248
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027872417251755265,
+ "loss": 1.0921,
+ "theoretical_loss": 3.4367828780198364,
+ "tokens_seen": 2162294784
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027870812437311935,
+ "loss": 1.2422,
+ "theoretical_loss": 3.4367743967053355,
+ "tokens_seen": 2162360320
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027869207622868605,
+ "loss": 1.2413,
+ "theoretical_loss": 3.4367659157198505,
+ "tokens_seen": 2162425856
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027867602808425275,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4367574350633587,
+ "tokens_seen": 2162491392
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027865997993981945,
+ "loss": 1.3192,
+ "theoretical_loss": 3.4367489547358367,
+ "tokens_seen": 2162556928
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027864393179538615,
+ "loss": 1.4066,
+ "theoretical_loss": 3.4367404747372623,
+ "tokens_seen": 2162622464
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 724069,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.196091651916504,
+ "objective/train/theoretical_loss": 3.4367319950676127,
+ "objective/train/tokens_used": 2183148000,
+ "theoretical_loss": 3.4367319950676127,
+ "tokens_seen": 2162688000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027862788365095285,
+ "loss": 1.2863,
+ "theoretical_loss": 3.4367319950676127,
+ "tokens_seen": 2162688000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027861183550651955,
+ "loss": 1.249,
+ "theoretical_loss": 3.4367235157268654,
+ "tokens_seen": 2162753536
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027859578736208625,
+ "loss": 1.2445,
+ "theoretical_loss": 3.436715036714997,
+ "tokens_seen": 2162819072
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027857973921765295,
+ "loss": 1.2465,
+ "theoretical_loss": 3.436706558031985,
+ "tokens_seen": 2162884608
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027856369107321966,
+ "loss": 1.2393,
+ "theoretical_loss": 3.4366980796778073,
+ "tokens_seen": 2162950144
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027854764292878636,
+ "loss": 1.3412,
+ "theoretical_loss": 3.4366896016524406,
+ "tokens_seen": 2163015680
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027853159478435306,
+ "loss": 1.1941,
+ "theoretical_loss": 3.4366811239558626,
+ "tokens_seen": 2163081216
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027851554663991976,
+ "loss": 1.3069,
+ "theoretical_loss": 3.43667264658805,
+ "tokens_seen": 2163146752
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027849949849548646,
+ "loss": 1.3146,
+ "theoretical_loss": 3.4366641695489806,
+ "tokens_seen": 2163212288
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027848345035105316,
+ "loss": 1.2928,
+ "theoretical_loss": 3.436655692838631,
+ "tokens_seen": 2163277824
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027846740220661986,
+ "loss": 1.2436,
+ "theoretical_loss": 3.4366472164569792,
+ "tokens_seen": 2163343360
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027845135406218656,
+ "loss": 1.3359,
+ "theoretical_loss": 3.436638740404003,
+ "tokens_seen": 2163408896
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027843530591775326,
+ "loss": 1.2725,
+ "theoretical_loss": 3.4366302646796782,
+ "tokens_seen": 2163474432
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027841925777331996,
+ "loss": 1.2524,
+ "theoretical_loss": 3.436621789283983,
+ "tokens_seen": 2163539968
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027840320962888666,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4366133142168946,
+ "tokens_seen": 2163605504
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027838716148445336,
+ "loss": 1.2345,
+ "theoretical_loss": 3.436604839478391,
+ "tokens_seen": 2163671040
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027837111334002006,
+ "loss": 1.1999,
+ "theoretical_loss": 3.4365963650684477,
+ "tokens_seen": 2163736576
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027835506519558676,
+ "loss": 1.369,
+ "theoretical_loss": 3.4365878909870435,
+ "tokens_seen": 2163802112
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027833901705115346,
+ "loss": 1.3435,
+ "theoretical_loss": 3.4365794172341557,
+ "tokens_seen": 2163867648
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027832296890672016,
+ "loss": 1.2682,
+ "theoretical_loss": 3.436570943809761,
+ "tokens_seen": 2163933184
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027830692076228687,
+ "loss": 1.1939,
+ "theoretical_loss": 3.436562470713837,
+ "tokens_seen": 2163998720
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027829087261785357,
+ "loss": 1.2439,
+ "theoretical_loss": 3.436553997946361,
+ "tokens_seen": 2164064256
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027827482447342027,
+ "loss": 1.2257,
+ "theoretical_loss": 3.4365455255073103,
+ "tokens_seen": 2164129792
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027825877632898697,
+ "loss": 1.353,
+ "theoretical_loss": 3.436537053396662,
+ "tokens_seen": 2164195328
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027824272818455367,
+ "loss": 1.2564,
+ "theoretical_loss": 3.436528581614394,
+ "tokens_seen": 2164260864
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027822668004012037,
+ "loss": 1.28,
+ "theoretical_loss": 3.436520110160483,
+ "tokens_seen": 2164326400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027821063189568707,
+ "loss": 1.3541,
+ "theoretical_loss": 3.436511639034907,
+ "tokens_seen": 2164391936
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027819458375125377,
+ "loss": 1.2283,
+ "theoretical_loss": 3.436503168237643,
+ "tokens_seen": 2164457472
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027817853560682047,
+ "loss": 1.2818,
+ "theoretical_loss": 3.436494697768668,
+ "tokens_seen": 2164523008
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027816248746238717,
+ "loss": 1.3455,
+ "theoretical_loss": 3.4364862276279595,
+ "tokens_seen": 2164588544
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027814643931795387,
+ "loss": 1.3924,
+ "theoretical_loss": 3.4364777578154952,
+ "tokens_seen": 2164654080
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027813039117352057,
+ "loss": 1.2284,
+ "theoretical_loss": 3.436469288331252,
+ "tokens_seen": 2164719616
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002781143430290873,
+ "loss": 1.3133,
+ "theoretical_loss": 3.436460819175208,
+ "tokens_seen": 2164785152
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000278098294884654,
+ "loss": 1.2281,
+ "theoretical_loss": 3.4364523503473396,
+ "tokens_seen": 2164850688
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780822467402207,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4364438818476244,
+ "tokens_seen": 2164916224
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780661985957874,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4364354136760404,
+ "tokens_seen": 2164981760
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780501504513541,
+ "loss": 1.2576,
+ "theoretical_loss": 3.436426945832564,
+ "tokens_seen": 2165047296
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780341023069208,
+ "loss": 1.1744,
+ "theoretical_loss": 3.436418478317173,
+ "tokens_seen": 2165112832
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780180541624875,
+ "loss": 1.2468,
+ "theoretical_loss": 3.436410011129845,
+ "tokens_seen": 2165178368
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002780020060180542,
+ "loss": 1.2308,
+ "theoretical_loss": 3.436401544270557,
+ "tokens_seen": 2165243904
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002779859578736209,
+ "loss": 1.3324,
+ "theoretical_loss": 3.436393077739287,
+ "tokens_seen": 2165309440
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002779699097291876,
+ "loss": 1.1759,
+ "theoretical_loss": 3.436384611536011,
+ "tokens_seen": 2165374976
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002779538615847543,
+ "loss": 1.3107,
+ "theoretical_loss": 3.4363761456607076,
+ "tokens_seen": 2165440512
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000277937813440321,
+ "loss": 1.192,
+ "theoretical_loss": 3.436367680113354,
+ "tokens_seen": 2165506048
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002779217652958877,
+ "loss": 1.3118,
+ "theoretical_loss": 3.436359214893927,
+ "tokens_seen": 2165571584
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002779057171514544,
+ "loss": 1.187,
+ "theoretical_loss": 3.4363507500024046,
+ "tokens_seen": 2165637120
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778896690070211,
+ "loss": 1.3252,
+ "theoretical_loss": 3.436342285438764,
+ "tokens_seen": 2165702656
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778736208625878,
+ "loss": 1.2995,
+ "theoretical_loss": 3.436333821202982,
+ "tokens_seen": 2165768192
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778575727181545,
+ "loss": 1.3257,
+ "theoretical_loss": 3.4363253572950367,
+ "tokens_seen": 2165833728
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778415245737212,
+ "loss": 1.2101,
+ "theoretical_loss": 3.4363168937149053,
+ "tokens_seen": 2165899264
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 725023,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2539417743682861,
+ "objective/train/theoretical_loss": 3.4363084304625655,
+ "objective/train/tokens_used": 2186424800,
+ "theoretical_loss": 3.4363084304625655,
+ "tokens_seen": 2165964800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778254764292879,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4363084304625655,
+ "tokens_seen": 2165964800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002778094282848546,
+ "loss": 1.3366,
+ "theoretical_loss": 3.4362999675379937,
+ "tokens_seen": 2166030336
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002777933801404213,
+ "loss": 1.2662,
+ "theoretical_loss": 3.4362915049411686,
+ "tokens_seen": 2166095872
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000277777331995988,
+ "loss": 1.3058,
+ "theoretical_loss": 3.4362830426720663,
+ "tokens_seen": 2166161408
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002777612838515547,
+ "loss": 1.3021,
+ "theoretical_loss": 3.4362745807306654,
+ "tokens_seen": 2166226944
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002777452357071214,
+ "loss": 1.3195,
+ "theoretical_loss": 3.4362661191169424,
+ "tokens_seen": 2166292480
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002777291875626881,
+ "loss": 1.2236,
+ "theoretical_loss": 3.436257657830875,
+ "tokens_seen": 2166358016
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002777131394182548,
+ "loss": 1.2025,
+ "theoretical_loss": 3.4362491968724402,
+ "tokens_seen": 2166423552
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776970912738215,
+ "loss": 1.3413,
+ "theoretical_loss": 3.4362407362416163,
+ "tokens_seen": 2166489088
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776810431293882,
+ "loss": 1.3679,
+ "theoretical_loss": 3.43623227593838,
+ "tokens_seen": 2166554624
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776649949849549,
+ "loss": 1.3038,
+ "theoretical_loss": 3.436223815962709,
+ "tokens_seen": 2166620160
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776489468405216,
+ "loss": 1.2452,
+ "theoretical_loss": 3.4362153563145807,
+ "tokens_seen": 2166685696
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776328986960883,
+ "loss": 1.2922,
+ "theoretical_loss": 3.4362068969939727,
+ "tokens_seen": 2166751232
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000277616850551655,
+ "loss": 1.2617,
+ "theoretical_loss": 3.4361984380008614,
+ "tokens_seen": 2166816768
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002776008024072217,
+ "loss": 1.2217,
+ "theoretical_loss": 3.436189979335226,
+ "tokens_seen": 2166882304
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775847542627884,
+ "loss": 1.2308,
+ "theoretical_loss": 3.436181520997042,
+ "tokens_seen": 2166947840
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775687061183551,
+ "loss": 1.2311,
+ "theoretical_loss": 3.436173062986288,
+ "tokens_seen": 2167013376
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775526579739218,
+ "loss": 1.2988,
+ "theoretical_loss": 3.436164605302941,
+ "tokens_seen": 2167078912
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775366098294885,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4361561479469787,
+ "tokens_seen": 2167144448
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775205616850552,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4361476909183786,
+ "tokens_seen": 2167209984
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002775045135406219,
+ "loss": 1.2641,
+ "theoretical_loss": 3.4361392342171175,
+ "tokens_seen": 2167275520
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002774884653961886,
+ "loss": 1.1933,
+ "theoretical_loss": 3.4361307778431733,
+ "tokens_seen": 2167341056
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002774724172517553,
+ "loss": 1.1872,
+ "theoretical_loss": 3.4361223217965238,
+ "tokens_seen": 2167406592
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000277456369107322,
+ "loss": 1.1886,
+ "theoretical_loss": 3.4361138660771458,
+ "tokens_seen": 2167472128
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002774403209628887,
+ "loss": 1.3763,
+ "theoretical_loss": 3.4361054106850166,
+ "tokens_seen": 2167537664
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002774242728184554,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4360969556201146,
+ "tokens_seen": 2167603200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002774082246740221,
+ "loss": 1.2399,
+ "theoretical_loss": 3.4360885008824162,
+ "tokens_seen": 2167668736
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002773921765295888,
+ "loss": 1.2881,
+ "theoretical_loss": 3.4360800464718992,
+ "tokens_seen": 2167734272
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002773761283851555,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4360715923885414,
+ "tokens_seen": 2167799808
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027736008024072215,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4360631386323197,
+ "tokens_seen": 2167865344
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027734403209628885,
+ "loss": 1.2507,
+ "theoretical_loss": 3.4360546852032123,
+ "tokens_seen": 2167930880
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027732798395185555,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4360462321011958,
+ "tokens_seen": 2167996416
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027731193580742225,
+ "loss": 1.2662,
+ "theoretical_loss": 3.4360377793262487,
+ "tokens_seen": 2168061952
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027729588766298895,
+ "loss": 1.2878,
+ "theoretical_loss": 3.436029326878347,
+ "tokens_seen": 2168127488
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027727983951855565,
+ "loss": 1.2268,
+ "theoretical_loss": 3.436020874757469,
+ "tokens_seen": 2168193024
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027726379137412235,
+ "loss": 1.2632,
+ "theoretical_loss": 3.436012422963593,
+ "tokens_seen": 2168258560
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027724774322968905,
+ "loss": 1.2978,
+ "theoretical_loss": 3.4360039714966946,
+ "tokens_seen": 2168324096
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027723169508525575,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4359955203567525,
+ "tokens_seen": 2168389632
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027721564694082245,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4359870695437444,
+ "tokens_seen": 2168455168
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027719959879638915,
+ "loss": 1.2189,
+ "theoretical_loss": 3.435978619057647,
+ "tokens_seen": 2168520704
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027718355065195586,
+ "loss": 1.2232,
+ "theoretical_loss": 3.435970168898438,
+ "tokens_seen": 2168586240
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027716750250752256,
+ "loss": 1.3604,
+ "theoretical_loss": 3.4359617190660954,
+ "tokens_seen": 2168651776
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027715145436308926,
+ "loss": 1.2792,
+ "theoretical_loss": 3.4359532695605957,
+ "tokens_seen": 2168717312
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027713540621865596,
+ "loss": 1.2758,
+ "theoretical_loss": 3.435944820381917,
+ "tokens_seen": 2168782848
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027711935807422266,
+ "loss": 1.246,
+ "theoretical_loss": 3.435936371530037,
+ "tokens_seen": 2168848384
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027710330992978936,
+ "loss": 1.3601,
+ "theoretical_loss": 3.4359279230049324,
+ "tokens_seen": 2168913920
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027708726178535606,
+ "loss": 1.355,
+ "theoretical_loss": 3.4359194748065818,
+ "tokens_seen": 2168979456
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027707121364092276,
+ "loss": 1.308,
+ "theoretical_loss": 3.4359110269349618,
+ "tokens_seen": 2169044992
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027705516549648946,
+ "loss": 1.3152,
+ "theoretical_loss": 3.43590257939005,
+ "tokens_seen": 2169110528
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027703911735205616,
+ "loss": 1.259,
+ "theoretical_loss": 3.435894132171824,
+ "tokens_seen": 2169176064
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 726246,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2564411163330078,
+ "objective/train/theoretical_loss": 3.4358856852802617,
+ "objective/train/tokens_used": 2189701600,
+ "theoretical_loss": 3.4358856852802617,
+ "tokens_seen": 2169241600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027702306920762286,
+ "loss": 1.2804,
+ "theoretical_loss": 3.4358856852802617,
+ "tokens_seen": 2169241600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027700702106318956,
+ "loss": 1.2098,
+ "theoretical_loss": 3.43587723871534,
+ "tokens_seen": 2169307136
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027699097291875626,
+ "loss": 1.2285,
+ "theoretical_loss": 3.435868792477037,
+ "tokens_seen": 2169372672
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027697492477432296,
+ "loss": 1.1938,
+ "theoretical_loss": 3.4358603465653292,
+ "tokens_seen": 2169438208
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027695887662988966,
+ "loss": 1.2302,
+ "theoretical_loss": 3.4358519009801953,
+ "tokens_seen": 2169503744
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027694282848545637,
+ "loss": 1.25,
+ "theoretical_loss": 3.435843455721612,
+ "tokens_seen": 2169569280
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027692678034102307,
+ "loss": 1.2779,
+ "theoretical_loss": 3.435835010789557,
+ "tokens_seen": 2169634816
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027691073219658977,
+ "loss": 1.2488,
+ "theoretical_loss": 3.4358265661840086,
+ "tokens_seen": 2169700352
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027689468405215647,
+ "loss": 1.2754,
+ "theoretical_loss": 3.435818121904943,
+ "tokens_seen": 2169765888
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027687863590772317,
+ "loss": 1.329,
+ "theoretical_loss": 3.435809677952338,
+ "tokens_seen": 2169831424
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002768625877632899,
+ "loss": 1.237,
+ "theoretical_loss": 3.4358012343261723,
+ "tokens_seen": 2169896960
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002768465396188566,
+ "loss": 1.2092,
+ "theoretical_loss": 3.435792791026422,
+ "tokens_seen": 2169962496
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002768304914744233,
+ "loss": 1.3439,
+ "theoretical_loss": 3.4357843480530654,
+ "tokens_seen": 2170028032
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027681444332999,
+ "loss": 1.2607,
+ "theoretical_loss": 3.4357759054060795,
+ "tokens_seen": 2170093568
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767983951855567,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4357674630854422,
+ "tokens_seen": 2170159104
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767823470411234,
+ "loss": 1.191,
+ "theoretical_loss": 3.4357590210911315,
+ "tokens_seen": 2170224640
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767662988966901,
+ "loss": 1.2514,
+ "theoretical_loss": 3.435750579423124,
+ "tokens_seen": 2170290176
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767502507522568,
+ "loss": 1.1695,
+ "theoretical_loss": 3.4357421380813977,
+ "tokens_seen": 2170355712
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027673420260782353,
+ "loss": 1.3135,
+ "theoretical_loss": 3.4357336970659302,
+ "tokens_seen": 2170421248
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767181544633902,
+ "loss": 1.2709,
+ "theoretical_loss": 3.4357252563766987,
+ "tokens_seen": 2170486784
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002767021063189569,
+ "loss": 1.2385,
+ "theoretical_loss": 3.435716816013681,
+ "tokens_seen": 2170552320
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002766860581745236,
+ "loss": 1.2866,
+ "theoretical_loss": 3.4357083759768545,
+ "tokens_seen": 2170617856
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002766700100300903,
+ "loss": 1.3206,
+ "theoretical_loss": 3.435699936266197,
+ "tokens_seen": 2170683392
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000276653961885657,
+ "loss": 1.1749,
+ "theoretical_loss": 3.435691496881686,
+ "tokens_seen": 2170748928
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002766379137412237,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4356830578232986,
+ "tokens_seen": 2170814464
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002766218655967904,
+ "loss": 1.3343,
+ "theoretical_loss": 3.4356746190910132,
+ "tokens_seen": 2170880000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002766058174523571,
+ "loss": 1.251,
+ "theoretical_loss": 3.4356661806848066,
+ "tokens_seen": 2170945536
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765897693079238,
+ "loss": 1.301,
+ "theoretical_loss": 3.4356577426046564,
+ "tokens_seen": 2171011072
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765737211634905,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4356493048505405,
+ "tokens_seen": 2171076608
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765576730190572,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4356408674224364,
+ "tokens_seen": 2171142144
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765416248746239,
+ "loss": 1.2797,
+ "theoretical_loss": 3.4356324303203216,
+ "tokens_seen": 2171207680
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765255767301906,
+ "loss": 1.2676,
+ "theoretical_loss": 3.4356239935441737,
+ "tokens_seen": 2171273216
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002765095285857573,
+ "loss": 1.3007,
+ "theoretical_loss": 3.4356155570939704,
+ "tokens_seen": 2171338752
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000276493480441324,
+ "loss": 1.1645,
+ "theoretical_loss": 3.4356071209696886,
+ "tokens_seen": 2171404288
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002764774322968907,
+ "loss": 1.3095,
+ "theoretical_loss": 3.435598685171307,
+ "tokens_seen": 2171469824
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002764613841524574,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4355902496988016,
+ "tokens_seen": 2171535360
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002764453360080241,
+ "loss": 1.1354,
+ "theoretical_loss": 3.435581814552152,
+ "tokens_seen": 2171600896
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002764292878635908,
+ "loss": 1.3093,
+ "theoretical_loss": 3.435573379731334,
+ "tokens_seen": 2171666432
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002764132397191575,
+ "loss": 1.2135,
+ "theoretical_loss": 3.435564945236326,
+ "tokens_seen": 2171731968
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763971915747242,
+ "loss": 1.2994,
+ "theoretical_loss": 3.4355565110671056,
+ "tokens_seen": 2171797504
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763811434302909,
+ "loss": 1.3061,
+ "theoretical_loss": 3.4355480772236504,
+ "tokens_seen": 2171863040
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763650952858576,
+ "loss": 1.3247,
+ "theoretical_loss": 3.435539643705938,
+ "tokens_seen": 2171928576
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763490471414243,
+ "loss": 1.2691,
+ "theoretical_loss": 3.435531210513945,
+ "tokens_seen": 2171994112
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000276332998996991,
+ "loss": 1.3157,
+ "theoretical_loss": 3.4355227776476505,
+ "tokens_seen": 2172059648
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763169508525577,
+ "loss": 1.1872,
+ "theoretical_loss": 3.4355143451070314,
+ "tokens_seen": 2172125184
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002763009027081244,
+ "loss": 1.2944,
+ "theoretical_loss": 3.435505912892065,
+ "tokens_seen": 2172190720
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762848545636911,
+ "loss": 1.3045,
+ "theoretical_loss": 3.435497481002729,
+ "tokens_seen": 2172256256
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762688064192578,
+ "loss": 1.3385,
+ "theoretical_loss": 3.435489049439002,
+ "tokens_seen": 2172321792
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762527582748245,
+ "loss": 1.2899,
+ "theoretical_loss": 3.4354806182008604,
+ "tokens_seen": 2172387328
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762367101303912,
+ "loss": 1.1809,
+ "theoretical_loss": 3.435472187288282,
+ "tokens_seen": 2172452864
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 727225,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3450744152069092,
+ "objective/train/theoretical_loss": 3.4354637567012447,
+ "objective/train/tokens_used": 2192978400,
+ "theoretical_loss": 3.4354637567012447,
+ "tokens_seen": 2172518400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762206619859579,
+ "loss": 1.3029,
+ "theoretical_loss": 3.4354637567012447,
+ "tokens_seen": 2172518400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002762046138415246,
+ "loss": 1.2384,
+ "theoretical_loss": 3.4354553264397265,
+ "tokens_seen": 2172583936
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002761885656970913,
+ "loss": 1.2204,
+ "theoretical_loss": 3.4354468965037044,
+ "tokens_seen": 2172649472
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000276172517552658,
+ "loss": 1.222,
+ "theoretical_loss": 3.4354384668931557,
+ "tokens_seen": 2172715008
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002761564694082247,
+ "loss": 1.2255,
+ "theoretical_loss": 3.435430037608059,
+ "tokens_seen": 2172780544
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002761404212637914,
+ "loss": 1.287,
+ "theoretical_loss": 3.435421608648391,
+ "tokens_seen": 2172846080
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002761243731193581,
+ "loss": 1.2286,
+ "theoretical_loss": 3.4354131800141303,
+ "tokens_seen": 2172911616
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002761083249749248,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4354047517052537,
+ "tokens_seen": 2172977152
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002760922768304915,
+ "loss": 1.2868,
+ "theoretical_loss": 3.4353963237217386,
+ "tokens_seen": 2173042688
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002760762286860582,
+ "loss": 1.2537,
+ "theoretical_loss": 3.4353878960635638,
+ "tokens_seen": 2173108224
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002760601805416249,
+ "loss": 1.3856,
+ "theoretical_loss": 3.435379468730706,
+ "tokens_seen": 2173173760
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002760441323971916,
+ "loss": 1.3105,
+ "theoretical_loss": 3.4353710417231427,
+ "tokens_seen": 2173239296
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002760280842527583,
+ "loss": 1.3522,
+ "theoretical_loss": 3.4353626150408525,
+ "tokens_seen": 2173304832
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000276012036108325,
+ "loss": 1.2926,
+ "theoretical_loss": 3.435354188683812,
+ "tokens_seen": 2173370368
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002759959879638917,
+ "loss": 1.2617,
+ "theoretical_loss": 3.435345762651999,
+ "tokens_seen": 2173435904
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002759799398194584,
+ "loss": 1.2295,
+ "theoretical_loss": 3.435337336945392,
+ "tokens_seen": 2173501440
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002759638916750251,
+ "loss": 1.364,
+ "theoretical_loss": 3.4353289115639685,
+ "tokens_seen": 2173566976
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027594784353059175,
+ "loss": 1.2711,
+ "theoretical_loss": 3.435320486507705,
+ "tokens_seen": 2173632512
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027593179538615845,
+ "loss": 1.2468,
+ "theoretical_loss": 3.43531206177658,
+ "tokens_seen": 2173698048
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027591574724172515,
+ "loss": 1.1475,
+ "theoretical_loss": 3.4353036373705708,
+ "tokens_seen": 2173763584
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027589969909729185,
+ "loss": 1.2761,
+ "theoretical_loss": 3.4352952132896553,
+ "tokens_seen": 2173829120
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027588365095285855,
+ "loss": 1.2536,
+ "theoretical_loss": 3.4352867895338113,
+ "tokens_seen": 2173894656
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027586760280842525,
+ "loss": 1.3076,
+ "theoretical_loss": 3.4352783661030166,
+ "tokens_seen": 2173960192
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027585155466399195,
+ "loss": 1.233,
+ "theoretical_loss": 3.435269942997248,
+ "tokens_seen": 2174025728
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027583550651955865,
+ "loss": 1.2202,
+ "theoretical_loss": 3.435261520216484,
+ "tokens_seen": 2174091264
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027581945837512536,
+ "loss": 1.3136,
+ "theoretical_loss": 3.4352530977607016,
+ "tokens_seen": 2174156800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027580341023069206,
+ "loss": 1.2573,
+ "theoretical_loss": 3.435244675629879,
+ "tokens_seen": 2174222336
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027578736208625876,
+ "loss": 1.1988,
+ "theoretical_loss": 3.435236253823994,
+ "tokens_seen": 2174287872
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027577131394182546,
+ "loss": 1.2127,
+ "theoretical_loss": 3.4352278323430236,
+ "tokens_seen": 2174353408
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027575526579739216,
+ "loss": 1.2602,
+ "theoretical_loss": 3.435219411186946,
+ "tokens_seen": 2174418944
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027573921765295886,
+ "loss": 1.2297,
+ "theoretical_loss": 3.4352109903557384,
+ "tokens_seen": 2174484480
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027572316950852556,
+ "loss": 1.3152,
+ "theoretical_loss": 3.435202569849379,
+ "tokens_seen": 2174550016
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027570712136409226,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4351941496678453,
+ "tokens_seen": 2174615552
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027569107321965896,
+ "loss": 1.2269,
+ "theoretical_loss": 3.4351857298111153,
+ "tokens_seen": 2174681088
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002756750250752257,
+ "loss": 1.3053,
+ "theoretical_loss": 3.435177310279166,
+ "tokens_seen": 2174746624
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002756589769307924,
+ "loss": 1.2675,
+ "theoretical_loss": 3.4351688910719753,
+ "tokens_seen": 2174812160
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002756429287863591,
+ "loss": 1.227,
+ "theoretical_loss": 3.435160472189521,
+ "tokens_seen": 2174877696
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002756268806419258,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4351520536317808,
+ "tokens_seen": 2174943232
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002756108324974925,
+ "loss": 1.3607,
+ "theoretical_loss": 3.4351436353987324,
+ "tokens_seen": 2175008768
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002755947843530592,
+ "loss": 1.2301,
+ "theoretical_loss": 3.4351352174903536,
+ "tokens_seen": 2175074304
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002755787362086259,
+ "loss": 1.2755,
+ "theoretical_loss": 3.4351267999066217,
+ "tokens_seen": 2175139840
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002755626880641926,
+ "loss": 1.2091,
+ "theoretical_loss": 3.4351183826475147,
+ "tokens_seen": 2175205376
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002755466399197593,
+ "loss": 1.2878,
+ "theoretical_loss": 3.4351099657130106,
+ "tokens_seen": 2175270912
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000275530591775326,
+ "loss": 1.2171,
+ "theoretical_loss": 3.4351015491030865,
+ "tokens_seen": 2175336448
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002755145436308927,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4350931328177206,
+ "tokens_seen": 2175401984
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754984954864594,
+ "loss": 1.2454,
+ "theoretical_loss": 3.43508471685689,
+ "tokens_seen": 2175467520
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754824473420261,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4350763012205734,
+ "tokens_seen": 2175533056
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754663991975928,
+ "loss": 1.29,
+ "theoretical_loss": 3.4350678859087473,
+ "tokens_seen": 2175598592
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754503510531595,
+ "loss": 1.186,
+ "theoretical_loss": 3.43505947092139,
+ "tokens_seen": 2175664128
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754343029087262,
+ "loss": 1.3757,
+ "theoretical_loss": 3.4350510562584797,
+ "tokens_seen": 2175729664
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 728507,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2073462009429932,
+ "objective/train/theoretical_loss": 3.4350426419199938,
+ "objective/train/tokens_used": 2196255200,
+ "theoretical_loss": 3.4350426419199938,
+ "tokens_seen": 2175795200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754182547642929,
+ "loss": 1.2139,
+ "theoretical_loss": 3.4350426419199938,
+ "tokens_seen": 2175795200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002754022066198596,
+ "loss": 1.2933,
+ "theoretical_loss": 3.435034227905909,
+ "tokens_seen": 2175860736
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002753861584754263,
+ "loss": 1.3251,
+ "theoretical_loss": 3.4350258142162047,
+ "tokens_seen": 2175926272
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027537011033099303,
+ "loss": 1.3179,
+ "theoretical_loss": 3.4350174008508576,
+ "tokens_seen": 2175991808
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027535406218655973,
+ "loss": 1.3092,
+ "theoretical_loss": 3.4350089878098453,
+ "tokens_seen": 2176057344
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027533801404212643,
+ "loss": 1.2077,
+ "theoretical_loss": 3.4350005750931465,
+ "tokens_seen": 2176122880
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002753219658976931,
+ "loss": 1.3125,
+ "theoretical_loss": 3.434992162700738,
+ "tokens_seen": 2176188416
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002753059177532598,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4349837506325978,
+ "tokens_seen": 2176253952
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002752898696088265,
+ "loss": 1.3384,
+ "theoretical_loss": 3.434975338888704,
+ "tokens_seen": 2176319488
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002752738214643932,
+ "loss": 1.2628,
+ "theoretical_loss": 3.4349669274690338,
+ "tokens_seen": 2176385024
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002752577733199599,
+ "loss": 1.1707,
+ "theoretical_loss": 3.434958516373565,
+ "tokens_seen": 2176450560
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002752417251755266,
+ "loss": 1.1902,
+ "theoretical_loss": 3.4349501056022755,
+ "tokens_seen": 2176516096
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002752256770310933,
+ "loss": 1.2094,
+ "theoretical_loss": 3.4349416951551435,
+ "tokens_seen": 2176581632
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027520962888666,
+ "loss": 1.2816,
+ "theoretical_loss": 3.434933285032146,
+ "tokens_seen": 2176647168
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751935807422267,
+ "loss": 1.3582,
+ "theoretical_loss": 3.4349248752332606,
+ "tokens_seen": 2176712704
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751775325977934,
+ "loss": 1.3919,
+ "theoretical_loss": 3.434916465758466,
+ "tokens_seen": 2176778240
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751614844533601,
+ "loss": 1.235,
+ "theoretical_loss": 3.4349080566077395,
+ "tokens_seen": 2176843776
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751454363089268,
+ "loss": 1.2004,
+ "theoretical_loss": 3.4348996477810587,
+ "tokens_seen": 2176909312
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751293881644935,
+ "loss": 1.2158,
+ "theoretical_loss": 3.4348912392784015,
+ "tokens_seen": 2176974848
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002751133400200602,
+ "loss": 1.1515,
+ "theoretical_loss": 3.4348828310997455,
+ "tokens_seen": 2177040384
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750972918756269,
+ "loss": 1.1633,
+ "theoretical_loss": 3.434874423245069,
+ "tokens_seen": 2177105920
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750812437311936,
+ "loss": 1.3084,
+ "theoretical_loss": 3.4348660157143485,
+ "tokens_seen": 2177171456
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750651955867603,
+ "loss": 1.3353,
+ "theoretical_loss": 3.4348576085075635,
+ "tokens_seen": 2177236992
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000275049147442327,
+ "loss": 1.4075,
+ "theoretical_loss": 3.4348492016246905,
+ "tokens_seen": 2177302528
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750330992978937,
+ "loss": 1.2179,
+ "theoretical_loss": 3.434840795065708,
+ "tokens_seen": 2177368064
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750170511534604,
+ "loss": 1.2448,
+ "theoretical_loss": 3.4348323888305927,
+ "tokens_seen": 2177433600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002750010030090271,
+ "loss": 1.299,
+ "theoretical_loss": 3.4348239829193234,
+ "tokens_seen": 2177499136
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749849548645938,
+ "loss": 1.3144,
+ "theoretical_loss": 3.434815577331878,
+ "tokens_seen": 2177564672
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749689067201605,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4348071720682336,
+ "tokens_seen": 2177630208
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749528585757272,
+ "loss": 1.1974,
+ "theoretical_loss": 3.434798767128368,
+ "tokens_seen": 2177695744
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749368104312939,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4347903625122598,
+ "tokens_seen": 2177761280
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749207622868606,
+ "loss": 1.1783,
+ "theoretical_loss": 3.4347819582198857,
+ "tokens_seen": 2177826816
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002749047141424273,
+ "loss": 1.2499,
+ "theoretical_loss": 3.4347735542512243,
+ "tokens_seen": 2177892352
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274888665997994,
+ "loss": 1.2854,
+ "theoretical_loss": 3.434765150606253,
+ "tokens_seen": 2177957888
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002748726178535607,
+ "loss": 1.2936,
+ "theoretical_loss": 3.4347567472849496,
+ "tokens_seen": 2178023424
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002748565697091274,
+ "loss": 1.264,
+ "theoretical_loss": 3.434748344287292,
+ "tokens_seen": 2178088960
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002748405215646941,
+ "loss": 1.3377,
+ "theoretical_loss": 3.434739941613258,
+ "tokens_seen": 2178154496
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002748244734202608,
+ "loss": 1.2703,
+ "theoretical_loss": 3.4347315392628257,
+ "tokens_seen": 2178220032
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002748084252758275,
+ "loss": 1.2726,
+ "theoretical_loss": 3.4347231372359723,
+ "tokens_seen": 2178285568
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002747923771313942,
+ "loss": 1.3109,
+ "theoretical_loss": 3.4347147355326757,
+ "tokens_seen": 2178351104
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002747763289869609,
+ "loss": 1.3036,
+ "theoretical_loss": 3.434706334152914,
+ "tokens_seen": 2178416640
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002747602808425276,
+ "loss": 1.2241,
+ "theoretical_loss": 3.434697933096665,
+ "tokens_seen": 2178482176
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002747442326980943,
+ "loss": 1.2488,
+ "theoretical_loss": 3.4346895323639064,
+ "tokens_seen": 2178547712
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274728184553661,
+ "loss": 1.286,
+ "theoretical_loss": 3.4346811319546164,
+ "tokens_seen": 2178613248
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002747121364092277,
+ "loss": 1.3469,
+ "theoretical_loss": 3.434672731868772,
+ "tokens_seen": 2178678784
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746960882647944,
+ "loss": 1.2434,
+ "theoretical_loss": 3.434664332106352,
+ "tokens_seen": 2178744320
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746800401203611,
+ "loss": 1.2891,
+ "theoretical_loss": 3.434655932667333,
+ "tokens_seen": 2178809856
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746639919759278,
+ "loss": 1.1401,
+ "theoretical_loss": 3.4346475335516935,
+ "tokens_seen": 2178875392
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746479438314945,
+ "loss": 1.3492,
+ "theoretical_loss": 3.434639134759412,
+ "tokens_seen": 2178940928
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746318956870612,
+ "loss": 1.2154,
+ "theoretical_loss": 3.434630736290465,
+ "tokens_seen": 2179006464
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 729371,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2518582344055176,
+ "objective/train/theoretical_loss": 3.4346223381448313,
+ "objective/train/tokens_used": 2199532000,
+ "theoretical_loss": 3.4346223381448313,
+ "tokens_seen": 2179072000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002746158475426279,
+ "loss": 1.2806,
+ "theoretical_loss": 3.4346223381448313,
+ "tokens_seen": 2179072000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002745997993981946,
+ "loss": 1.2542,
+ "theoretical_loss": 3.4346139403224885,
+ "tokens_seen": 2179137536
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002745837512537613,
+ "loss": 1.2155,
+ "theoretical_loss": 3.434605542823414,
+ "tokens_seen": 2179203072
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274567703109328,
+ "loss": 1.2492,
+ "theoretical_loss": 3.434597145647586,
+ "tokens_seen": 2179268608
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027455165496489465,
+ "loss": 1.3191,
+ "theoretical_loss": 3.4345887487949827,
+ "tokens_seen": 2179334144
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027453560682046135,
+ "loss": 1.2907,
+ "theoretical_loss": 3.4345803522655816,
+ "tokens_seen": 2179399680
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027451955867602805,
+ "loss": 1.2163,
+ "theoretical_loss": 3.43457195605936,
+ "tokens_seen": 2179465216
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027450351053159475,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4345635601762967,
+ "tokens_seen": 2179530752
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027448746238716145,
+ "loss": 1.2769,
+ "theoretical_loss": 3.434555164616369,
+ "tokens_seen": 2179596288
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002744714142427282,
+ "loss": 1.178,
+ "theoretical_loss": 3.4345467693795544,
+ "tokens_seen": 2179661824
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002744553660982949,
+ "loss": 1.2632,
+ "theoretical_loss": 3.4345383744658315,
+ "tokens_seen": 2179727360
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002744393179538616,
+ "loss": 1.2443,
+ "theoretical_loss": 3.434529979875178,
+ "tokens_seen": 2179792896
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002744232698094283,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4345215856075715,
+ "tokens_seen": 2179858432
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274407221664995,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4345131916629894,
+ "tokens_seen": 2179923968
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743911735205617,
+ "loss": 1.252,
+ "theoretical_loss": 3.4345047980414107,
+ "tokens_seen": 2179989504
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743751253761284,
+ "loss": 1.2517,
+ "theoretical_loss": 3.434496404742813,
+ "tokens_seen": 2180055040
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743590772316951,
+ "loss": 1.267,
+ "theoretical_loss": 3.434488011767173,
+ "tokens_seen": 2180120576
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743430290872618,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4344796191144695,
+ "tokens_seen": 2180186112
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743269809428285,
+ "loss": 1.2405,
+ "theoretical_loss": 3.434471226784681,
+ "tokens_seen": 2180251648
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002743109327983952,
+ "loss": 1.2004,
+ "theoretical_loss": 3.4344628347777837,
+ "tokens_seen": 2180317184
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002742948846539619,
+ "loss": 1.2749,
+ "theoretical_loss": 3.434454443093757,
+ "tokens_seen": 2180382720
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002742788365095286,
+ "loss": 1.3867,
+ "theoretical_loss": 3.434446051732578,
+ "tokens_seen": 2180448256
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002742627883650953,
+ "loss": 1.2356,
+ "theoretical_loss": 3.434437660694225,
+ "tokens_seen": 2180513792
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274246740220662,
+ "loss": 1.2854,
+ "theoretical_loss": 3.4344292699786756,
+ "tokens_seen": 2180579328
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002742306920762287,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4344208795859075,
+ "tokens_seen": 2180644864
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002742146439317954,
+ "loss": 1.3423,
+ "theoretical_loss": 3.4344124895158985,
+ "tokens_seen": 2180710400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741985957873621,
+ "loss": 1.2093,
+ "theoretical_loss": 3.4344040997686274,
+ "tokens_seen": 2180775936
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741825476429288,
+ "loss": 1.2617,
+ "theoretical_loss": 3.434395710344071,
+ "tokens_seen": 2180841472
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741664994984955,
+ "loss": 1.2901,
+ "theoretical_loss": 3.434387321242208,
+ "tokens_seen": 2180907008
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741504513540622,
+ "loss": 1.2759,
+ "theoretical_loss": 3.434378932463016,
+ "tokens_seen": 2180972544
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741344032096289,
+ "loss": 1.1744,
+ "theoretical_loss": 3.4343705440064722,
+ "tokens_seen": 2181038080
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741183550651956,
+ "loss": 1.3334,
+ "theoretical_loss": 3.434362155872556,
+ "tokens_seen": 2181103616
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002741023069207623,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4343537680612437,
+ "tokens_seen": 2181169152
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000274086258776329,
+ "loss": 1.2653,
+ "theoretical_loss": 3.4343453805725144,
+ "tokens_seen": 2181234688
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002740702106318957,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4343369934063457,
+ "tokens_seen": 2181300224
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002740541624874624,
+ "loss": 1.1942,
+ "theoretical_loss": 3.4343286065627145,
+ "tokens_seen": 2181365760
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002740381143430291,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4343202200416005,
+ "tokens_seen": 2181431296
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002740220661985958,
+ "loss": 1.234,
+ "theoretical_loss": 3.43431183384298,
+ "tokens_seen": 2181496832
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002740060180541625,
+ "loss": 1.2002,
+ "theoretical_loss": 3.434303447966832,
+ "tokens_seen": 2181562368
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027398996990972923,
+ "loss": 1.1793,
+ "theoretical_loss": 3.434295062413134,
+ "tokens_seen": 2181627904
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027397392176529593,
+ "loss": 1.2862,
+ "theoretical_loss": 3.4342866771818636,
+ "tokens_seen": 2181693440
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027395787362086263,
+ "loss": 1.208,
+ "theoretical_loss": 3.434278292272999,
+ "tokens_seen": 2181758976
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027394182547642933,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4342699076865184,
+ "tokens_seen": 2181824512
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000273925777331996,
+ "loss": 1.2709,
+ "theoretical_loss": 3.434261523422399,
+ "tokens_seen": 2181890048
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002739097291875627,
+ "loss": 1.3568,
+ "theoretical_loss": 3.43425313948062,
+ "tokens_seen": 2181955584
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738936810431294,
+ "loss": 1.2199,
+ "theoretical_loss": 3.4342447558611577,
+ "tokens_seen": 2182021120
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738776328986961,
+ "loss": 1.239,
+ "theoretical_loss": 3.4342363725639915,
+ "tokens_seen": 2182086656
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738615847542628,
+ "loss": 1.1954,
+ "theoretical_loss": 3.434227989589098,
+ "tokens_seen": 2182152192
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738455366098295,
+ "loss": 1.4862,
+ "theoretical_loss": 3.4342196069364563,
+ "tokens_seen": 2182217728
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738294884653962,
+ "loss": 1.2685,
+ "theoretical_loss": 3.4342112246060434,
+ "tokens_seen": 2182283264
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 730643,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3123812675476074,
+ "objective/train/theoretical_loss": 3.434202842597838,
+ "objective/train/tokens_used": 2202808800,
+ "theoretical_loss": 3.434202842597838,
+ "tokens_seen": 2182348800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002738134403209629,
+ "loss": 1.2078,
+ "theoretical_loss": 3.434202842597838,
+ "tokens_seen": 2182348800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737973921765296,
+ "loss": 1.2615,
+ "theoretical_loss": 3.434194460911818,
+ "tokens_seen": 2182414336
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737813440320963,
+ "loss": 1.2898,
+ "theoretical_loss": 3.4341860795479606,
+ "tokens_seen": 2182479872
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000273765295887663,
+ "loss": 1.1619,
+ "theoretical_loss": 3.434177698506244,
+ "tokens_seen": 2182545408
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737492477432297,
+ "loss": 1.3331,
+ "theoretical_loss": 3.434169317786647,
+ "tokens_seen": 2182610944
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737331995987964,
+ "loss": 1.1833,
+ "theoretical_loss": 3.4341609373891466,
+ "tokens_seen": 2182676480
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737171514543631,
+ "loss": 1.3169,
+ "theoretical_loss": 3.434152557313721,
+ "tokens_seen": 2182742016
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002737011033099298,
+ "loss": 1.2581,
+ "theoretical_loss": 3.434144177560348,
+ "tokens_seen": 2182807552
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002736850551654965,
+ "loss": 1.1956,
+ "theoretical_loss": 3.434135798129006,
+ "tokens_seen": 2182873088
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002736690070210632,
+ "loss": 1.1916,
+ "theoretical_loss": 3.434127419019673,
+ "tokens_seen": 2182938624
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002736529588766299,
+ "loss": 1.3026,
+ "theoretical_loss": 3.4341190402323263,
+ "tokens_seen": 2183004160
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002736369107321966,
+ "loss": 1.3305,
+ "theoretical_loss": 3.434110661766944,
+ "tokens_seen": 2183069696
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002736208625877633,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4341022836235044,
+ "tokens_seen": 2183135232
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027360481444333,
+ "loss": 1.3034,
+ "theoretical_loss": 3.4340939058019857,
+ "tokens_seen": 2183200768
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735887662988967,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4340855283023655,
+ "tokens_seen": 2183266304
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735727181544634,
+ "loss": 1.2445,
+ "theoretical_loss": 3.4340771511246215,
+ "tokens_seen": 2183331840
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735566700100301,
+ "loss": 1.2815,
+ "theoretical_loss": 3.434068774268732,
+ "tokens_seen": 2183397376
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735406218655968,
+ "loss": 1.2949,
+ "theoretical_loss": 3.434060397734675,
+ "tokens_seen": 2183462912
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735245737211635,
+ "loss": 1.2032,
+ "theoretical_loss": 3.4340520215224286,
+ "tokens_seen": 2183528448
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002735085255767302,
+ "loss": 1.257,
+ "theoretical_loss": 3.4340436456319705,
+ "tokens_seen": 2183593984
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002734924774322969,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4340352700632786,
+ "tokens_seen": 2183659520
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002734764292878636,
+ "loss": 1.2131,
+ "theoretical_loss": 3.434026894816331,
+ "tokens_seen": 2183725056
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002734603811434303,
+ "loss": 1.2897,
+ "theoretical_loss": 3.434018519891106,
+ "tokens_seen": 2183790592
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000273444332998997,
+ "loss": 1.2334,
+ "theoretical_loss": 3.4340101452875813,
+ "tokens_seen": 2183856128
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002734282848545637,
+ "loss": 1.1846,
+ "theoretical_loss": 3.434001771005735,
+ "tokens_seen": 2183921664
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002734122367101304,
+ "loss": 1.2991,
+ "theoretical_loss": 3.433993397045545,
+ "tokens_seen": 2183987200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733961885656971,
+ "loss": 1.2285,
+ "theoretical_loss": 3.433985023406989,
+ "tokens_seen": 2184052736
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733801404212638,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4339766500900453,
+ "tokens_seen": 2184118272
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733640922768305,
+ "loss": 1.2202,
+ "theoretical_loss": 3.433968277094692,
+ "tokens_seen": 2184183808
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733480441323972,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4339599044209073,
+ "tokens_seen": 2184249344
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733319959879639,
+ "loss": 1.3189,
+ "theoretical_loss": 3.4339515320686687,
+ "tokens_seen": 2184314880
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002733159478435306,
+ "loss": 1.232,
+ "theoretical_loss": 3.433943160037954,
+ "tokens_seen": 2184380416
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732998996990973,
+ "loss": 1.3561,
+ "theoretical_loss": 3.4339347883287425,
+ "tokens_seen": 2184445952
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000273283851554664,
+ "loss": 1.2056,
+ "theoretical_loss": 3.4339264169410106,
+ "tokens_seen": 2184511488
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732678034102307,
+ "loss": 1.2955,
+ "theoretical_loss": 3.433918045874737,
+ "tokens_seen": 2184577024
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732517552657974,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4339096751299003,
+ "tokens_seen": 2184642560
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732357071213641,
+ "loss": 1.2858,
+ "theoretical_loss": 3.4339013047064775,
+ "tokens_seen": 2184708096
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732196589769308,
+ "loss": 1.2592,
+ "theoretical_loss": 3.4338929346044473,
+ "tokens_seen": 2184773632
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002732036108324975,
+ "loss": 1.2947,
+ "theoretical_loss": 3.4338845648237877,
+ "tokens_seen": 2184839168
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002731875626880642,
+ "loss": 1.2275,
+ "theoretical_loss": 3.433876195364476,
+ "tokens_seen": 2184904704
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002731715145436309,
+ "loss": 1.302,
+ "theoretical_loss": 3.433867826226491,
+ "tokens_seen": 2184970240
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002731554663991976,
+ "loss": 1.223,
+ "theoretical_loss": 3.4338594574098105,
+ "tokens_seen": 2185035776
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002731394182547643,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4338510889144125,
+ "tokens_seen": 2185101312
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000273123370110331,
+ "loss": 1.2727,
+ "theoretical_loss": 3.433842720740275,
+ "tokens_seen": 2185166848
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002731073219658977,
+ "loss": 1.2995,
+ "theoretical_loss": 3.4338343528873763,
+ "tokens_seen": 2185232384
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730912738214644,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4338259853556936,
+ "tokens_seen": 2185297920
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730752256770311,
+ "loss": 1.3197,
+ "theoretical_loss": 3.433817618145206,
+ "tokens_seen": 2185363456
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730591775325978,
+ "loss": 1.305,
+ "theoretical_loss": 3.433809251255891,
+ "tokens_seen": 2185428992
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730431293881645,
+ "loss": 1.3012,
+ "theoretical_loss": 3.4338008846877264,
+ "tokens_seen": 2185494528
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730270812437312,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4337925184406908,
+ "tokens_seen": 2185560064
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 731623,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2505277395248413,
+ "objective/train/theoretical_loss": 3.433784152514762,
+ "objective/train/tokens_used": 2206085600,
+ "theoretical_loss": 3.433784152514762,
+ "tokens_seen": 2185625600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002730110330992979,
+ "loss": 1.2646,
+ "theoretical_loss": 3.433784152514762,
+ "tokens_seen": 2185625600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002729949849548646,
+ "loss": 1.2002,
+ "theoretical_loss": 3.433775786909918,
+ "tokens_seen": 2185691136
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002729789368104313,
+ "loss": 1.2039,
+ "theoretical_loss": 3.433767421626137,
+ "tokens_seen": 2185756672
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272962888665998,
+ "loss": 1.3329,
+ "theoretical_loss": 3.433759056663397,
+ "tokens_seen": 2185822208
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002729468405215647,
+ "loss": 1.2831,
+ "theoretical_loss": 3.433750692021676,
+ "tokens_seen": 2185887744
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002729307923771314,
+ "loss": 1.2542,
+ "theoretical_loss": 3.433742327700952,
+ "tokens_seen": 2185953280
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002729147442326981,
+ "loss": 1.2321,
+ "theoretical_loss": 3.433733963701203,
+ "tokens_seen": 2186018816
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728986960882648,
+ "loss": 1.265,
+ "theoretical_loss": 3.4337256000224072,
+ "tokens_seen": 2186084352
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728826479438315,
+ "loss": 1.3418,
+ "theoretical_loss": 3.4337172366645428,
+ "tokens_seen": 2186149888
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728665997993982,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4337088736275874,
+ "tokens_seen": 2186215424
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728505516549649,
+ "loss": 1.2032,
+ "theoretical_loss": 3.4337005109115197,
+ "tokens_seen": 2186280960
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728345035105316,
+ "loss": 1.2839,
+ "theoretical_loss": 3.433692148516317,
+ "tokens_seen": 2186346496
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002728184553660983,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4336837864419585,
+ "tokens_seen": 2186412032
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272802407221665,
+ "loss": 1.3451,
+ "theoretical_loss": 3.4336754246884214,
+ "tokens_seen": 2186477568
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727863590772317,
+ "loss": 1.3112,
+ "theoretical_loss": 3.4336670632556836,
+ "tokens_seen": 2186543104
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727703109327984,
+ "loss": 1.3068,
+ "theoretical_loss": 3.433658702143724,
+ "tokens_seen": 2186608640
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727542627883651,
+ "loss": 1.3228,
+ "theoretical_loss": 3.4336503413525197,
+ "tokens_seen": 2186674176
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727382146439318,
+ "loss": 1.251,
+ "theoretical_loss": 3.4336419808820495,
+ "tokens_seen": 2186739712
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727221664994985,
+ "loss": 1.3134,
+ "theoretical_loss": 3.4336336207322917,
+ "tokens_seen": 2186805248
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002727061183550652,
+ "loss": 1.2953,
+ "theoretical_loss": 3.4336252609032236,
+ "tokens_seen": 2186870784
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002726900702106319,
+ "loss": 1.2165,
+ "theoretical_loss": 3.4336169013948235,
+ "tokens_seen": 2186936320
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002726740220661986,
+ "loss": 1.3344,
+ "theoretical_loss": 3.43360854220707,
+ "tokens_seen": 2187001856
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002726579739217653,
+ "loss": 1.2352,
+ "theoretical_loss": 3.4336001833399408,
+ "tokens_seen": 2187067392
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272641925777332,
+ "loss": 1.2273,
+ "theoretical_loss": 3.433591824793414,
+ "tokens_seen": 2187132928
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027262587763289873,
+ "loss": 1.2975,
+ "theoretical_loss": 3.4335834665674674,
+ "tokens_seen": 2187198464
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027260982948846543,
+ "loss": 1.2102,
+ "theoretical_loss": 3.4335751086620796,
+ "tokens_seen": 2187264000
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027259378134403213,
+ "loss": 1.2986,
+ "theoretical_loss": 3.4335667510772288,
+ "tokens_seen": 2187329536
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027257773319959883,
+ "loss": 1.3181,
+ "theoretical_loss": 3.4335583938128926,
+ "tokens_seen": 2187395072
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027256168505516553,
+ "loss": 1.2769,
+ "theoretical_loss": 3.4335500368690495,
+ "tokens_seen": 2187460608
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027254563691073223,
+ "loss": 1.2327,
+ "theoretical_loss": 3.4335416802456775,
+ "tokens_seen": 2187526144
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027252958876629893,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4335333239427546,
+ "tokens_seen": 2187591680
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002725135406218656,
+ "loss": 1.2468,
+ "theoretical_loss": 3.433524967960259,
+ "tokens_seen": 2187657216
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724974924774323,
+ "loss": 1.3969,
+ "theoretical_loss": 3.4335166122981686,
+ "tokens_seen": 2187722752
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272481444332999,
+ "loss": 1.3756,
+ "theoretical_loss": 3.433508256956462,
+ "tokens_seen": 2187788288
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724653961885657,
+ "loss": 1.3174,
+ "theoretical_loss": 3.433499901935117,
+ "tokens_seen": 2187853824
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724493480441324,
+ "loss": 1.3112,
+ "theoretical_loss": 3.4334915472341114,
+ "tokens_seen": 2187919360
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724332998996991,
+ "loss": 1.1969,
+ "theoretical_loss": 3.433483192853424,
+ "tokens_seen": 2187984896
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724172517552658,
+ "loss": 1.2444,
+ "theoretical_loss": 3.4334748387930327,
+ "tokens_seen": 2188050432
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002724012036108325,
+ "loss": 1.3377,
+ "theoretical_loss": 3.4334664850529153,
+ "tokens_seen": 2188115968
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002723851554663992,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4334581316330506,
+ "tokens_seen": 2188181504
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002723691073219659,
+ "loss": 1.2653,
+ "theoretical_loss": 3.433449778533416,
+ "tokens_seen": 2188247040
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002723530591775326,
+ "loss": 1.3012,
+ "theoretical_loss": 3.4334414257539896,
+ "tokens_seen": 2188312576
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002723370110330993,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4334330732947502,
+ "tokens_seen": 2188378112
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272320962888666,
+ "loss": 1.2738,
+ "theoretical_loss": 3.433424721155675,
+ "tokens_seen": 2188443648
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002723049147442327,
+ "loss": 1.1709,
+ "theoretical_loss": 3.4334163693367437,
+ "tokens_seen": 2188509184
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722888665997994,
+ "loss": 1.2487,
+ "theoretical_loss": 3.433408017837933,
+ "tokens_seen": 2188574720
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722728184553661,
+ "loss": 1.1693,
+ "theoretical_loss": 3.4333996666592217,
+ "tokens_seen": 2188640256
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722567703109328,
+ "loss": 1.2151,
+ "theoretical_loss": 3.4333913158005873,
+ "tokens_seen": 2188705792
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722407221664995,
+ "loss": 1.2476,
+ "theoretical_loss": 3.433382965262009,
+ "tokens_seen": 2188771328
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722246740220662,
+ "loss": 1.2152,
+ "theoretical_loss": 3.433374615043464,
+ "tokens_seen": 2188836864
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 732904,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.46730375289917,
+ "objective/train/theoretical_loss": 3.433366265144931,
+ "objective/train/tokens_used": 2209362400,
+ "theoretical_loss": 3.433366265144931,
+ "tokens_seen": 2188902400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002722086258776329,
+ "loss": 1.4159,
+ "theoretical_loss": 3.433366265144931,
+ "tokens_seen": 2188902400
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002721925777331996,
+ "loss": 1.2832,
+ "theoretical_loss": 3.433357915566388,
+ "tokens_seen": 2188967936
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002721765295887663,
+ "loss": 1.2477,
+ "theoretical_loss": 3.433349566307813,
+ "tokens_seen": 2189033472
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000272160481444333,
+ "loss": 1.3136,
+ "theoretical_loss": 3.4333412173691844,
+ "tokens_seen": 2189099008
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002721444332998997,
+ "loss": 1.2506,
+ "theoretical_loss": 3.43333286875048,
+ "tokens_seen": 2189164544
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002721283851554664,
+ "loss": 1.1821,
+ "theoretical_loss": 3.433324520451679,
+ "tokens_seen": 2189230080
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002721123370110331,
+ "loss": 1.3606,
+ "theoretical_loss": 3.433316172472758,
+ "tokens_seen": 2189295616
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002720962888665998,
+ "loss": 1.2758,
+ "theoretical_loss": 3.433307824813696,
+ "tokens_seen": 2189361152
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002720802407221665,
+ "loss": 1.274,
+ "theoretical_loss": 3.4332994774744714,
+ "tokens_seen": 2189426688
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027206419257773325,
+ "loss": 1.3246,
+ "theoretical_loss": 3.433291130455062,
+ "tokens_seen": 2189492224
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027204814443329995,
+ "loss": 1.2384,
+ "theoretical_loss": 3.433282783755446,
+ "tokens_seen": 2189557760
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027203209628886665,
+ "loss": 1.473,
+ "theoretical_loss": 3.4332744373756015,
+ "tokens_seen": 2189623296
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027201604814443335,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4332660913155073,
+ "tokens_seen": 2189688832
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027200000000000005,
+ "loss": 1.2508,
+ "theoretical_loss": 3.433257745575141,
+ "tokens_seen": 2189754368
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027198395185556675,
+ "loss": 1.2115,
+ "theoretical_loss": 3.4332494001544807,
+ "tokens_seen": 2189819904
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027196790371113345,
+ "loss": 1.3055,
+ "theoretical_loss": 3.4332410550535046,
+ "tokens_seen": 2189885440
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027195185556670015,
+ "loss": 1.1699,
+ "theoretical_loss": 3.4332327102721916,
+ "tokens_seen": 2189950976
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027193580742226685,
+ "loss": 1.2941,
+ "theoretical_loss": 3.433224365810519,
+ "tokens_seen": 2190016512
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027191975927783356,
+ "loss": 1.283,
+ "theoretical_loss": 3.433216021668465,
+ "tokens_seen": 2190082048
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027190371113340026,
+ "loss": 1.2748,
+ "theoretical_loss": 3.433207677846009,
+ "tokens_seen": 2190147584
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002718876629889669,
+ "loss": 1.302,
+ "theoretical_loss": 3.4331993343431275,
+ "tokens_seen": 2190213120
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002718716148445336,
+ "loss": 1.2686,
+ "theoretical_loss": 3.4331909911598,
+ "tokens_seen": 2190278656
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002718555667001003,
+ "loss": 1.2531,
+ "theoretical_loss": 3.4331826482960044,
+ "tokens_seen": 2190344192
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000271839518555667,
+ "loss": 1.2229,
+ "theoretical_loss": 3.4331743057517183,
+ "tokens_seen": 2190409728
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002718234704112337,
+ "loss": 1.2915,
+ "theoretical_loss": 3.4331659635269207,
+ "tokens_seen": 2190475264
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002718074222668004,
+ "loss": 1.2542,
+ "theoretical_loss": 3.4331576216215893,
+ "tokens_seen": 2190540800
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717913741223671,
+ "loss": 1.3559,
+ "theoretical_loss": 3.4331492800357024,
+ "tokens_seen": 2190606336
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717753259779338,
+ "loss": 1.28,
+ "theoretical_loss": 3.4331409387692386,
+ "tokens_seen": 2190671872
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717592778335005,
+ "loss": 1.1738,
+ "theoretical_loss": 3.4331325978221754,
+ "tokens_seen": 2190737408
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717432296890672,
+ "loss": 1.1663,
+ "theoretical_loss": 3.4331242571944913,
+ "tokens_seen": 2190802944
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717271815446339,
+ "loss": 1.2851,
+ "theoretical_loss": 3.433115916886165,
+ "tokens_seen": 2190868480
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002717111334002006,
+ "loss": 1.2457,
+ "theoretical_loss": 3.433107576897174,
+ "tokens_seen": 2190934016
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002716950852557673,
+ "loss": 1.2872,
+ "theoretical_loss": 3.433099237227497,
+ "tokens_seen": 2190999552
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000271679037111334,
+ "loss": 1.1734,
+ "theoretical_loss": 3.433090897877112,
+ "tokens_seen": 2191065088
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002716629889669007,
+ "loss": 1.2922,
+ "theoretical_loss": 3.4330825588459977,
+ "tokens_seen": 2191130624
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002716469408224674,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4330742201341318,
+ "tokens_seen": 2191196160
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002716308926780341,
+ "loss": 1.2942,
+ "theoretical_loss": 3.4330658817414923,
+ "tokens_seen": 2191261696
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002716148445336008,
+ "loss": 1.2626,
+ "theoretical_loss": 3.433057543668058,
+ "tokens_seen": 2191327232
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715987963891675,
+ "loss": 1.399,
+ "theoretical_loss": 3.4330492059138074,
+ "tokens_seen": 2191392768
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715827482447342,
+ "loss": 1.353,
+ "theoretical_loss": 3.4330408684787175,
+ "tokens_seen": 2191458304
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715667001003009,
+ "loss": 1.3003,
+ "theoretical_loss": 3.4330325313627674,
+ "tokens_seen": 2191523840
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715506519558676,
+ "loss": 1.1713,
+ "theoretical_loss": 3.4330241945659354,
+ "tokens_seen": 2191589376
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715346038114343,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4330158580881998,
+ "tokens_seen": 2191654912
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000271518555667001,
+ "loss": 1.2558,
+ "theoretical_loss": 3.4330075219295386,
+ "tokens_seen": 2191720448
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002715025075225677,
+ "loss": 1.2202,
+ "theoretical_loss": 3.43299918608993,
+ "tokens_seen": 2191785984
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714864593781344,
+ "loss": 1.2727,
+ "theoretical_loss": 3.432990850569352,
+ "tokens_seen": 2191851520
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714704112337011,
+ "loss": 1.1913,
+ "theoretical_loss": 3.432982515367784,
+ "tokens_seen": 2191917056
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714543630892678,
+ "loss": 1.2485,
+ "theoretical_loss": 3.4329741804852025,
+ "tokens_seen": 2191982592
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714383149448345,
+ "loss": 1.1969,
+ "theoretical_loss": 3.4329658459215873,
+ "tokens_seen": 2192048128
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714222668004012,
+ "loss": 1.163,
+ "theoretical_loss": 3.432957511676916,
+ "tokens_seen": 2192113664
+ },
+ {
+ "epoch": 0.66,
+ "objective/train/docs_used": 733863,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2483806610107422,
+ "objective/train/theoretical_loss": 3.4329491777511665,
+ "objective/train/tokens_used": 2212639200,
+ "theoretical_loss": 3.4329491777511665,
+ "tokens_seen": 2192179200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002714062186559679,
+ "loss": 1.2159,
+ "theoretical_loss": 3.4329491777511665,
+ "tokens_seen": 2192179200
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002713901705115346,
+ "loss": 1.341,
+ "theoretical_loss": 3.432940844144318,
+ "tokens_seen": 2192244736
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002713741223671013,
+ "loss": 1.2892,
+ "theoretical_loss": 3.432932510856348,
+ "tokens_seen": 2192310272
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000271358074222668,
+ "loss": 1.2212,
+ "theoretical_loss": 3.432924177887235,
+ "tokens_seen": 2192375808
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002713420260782347,
+ "loss": 1.3107,
+ "theoretical_loss": 3.4329158452369573,
+ "tokens_seen": 2192441344
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002713259779338014,
+ "loss": 1.2408,
+ "theoretical_loss": 3.432907512905493,
+ "tokens_seen": 2192506880
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002713099297893681,
+ "loss": 1.2682,
+ "theoretical_loss": 3.4328991808928206,
+ "tokens_seen": 2192572416
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002712938816449348,
+ "loss": 1.2602,
+ "theoretical_loss": 3.4328908491989183,
+ "tokens_seen": 2192637952
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002712778335005015,
+ "loss": 1.2583,
+ "theoretical_loss": 3.4328825178237645,
+ "tokens_seen": 2192703488
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002712617853560682,
+ "loss": 1.3101,
+ "theoretical_loss": 3.432874186767337,
+ "tokens_seen": 2192769024
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027124573721163493,
+ "loss": 1.2182,
+ "theoretical_loss": 3.432865856029615,
+ "tokens_seen": 2192834560
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027122968906720163,
+ "loss": 1.2433,
+ "theoretical_loss": 3.4328575256105758,
+ "tokens_seen": 2192900096
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027121364092276833,
+ "loss": 1.2139,
+ "theoretical_loss": 3.4328491955101983,
+ "tokens_seen": 2192965632
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027119759277833503,
+ "loss": 1.2454,
+ "theoretical_loss": 3.43284086572846,
+ "tokens_seen": 2193031168
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027118154463390173,
+ "loss": 1.2587,
+ "theoretical_loss": 3.4328325362653405,
+ "tokens_seen": 2193096704
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027116549648946843,
+ "loss": 1.2008,
+ "theoretical_loss": 3.4328242071208175,
+ "tokens_seen": 2193162240
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027114944834503513,
+ "loss": 1.305,
+ "theoretical_loss": 3.4328158782948686,
+ "tokens_seen": 2193227776
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027113340020060183,
+ "loss": 1.2119,
+ "theoretical_loss": 3.432807549787473,
+ "tokens_seen": 2193293312
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002711173520561685,
+ "loss": 1.1968,
+ "theoretical_loss": 3.432799221598608,
+ "tokens_seen": 2193358848
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002711013039117352,
+ "loss": 1.2479,
+ "theoretical_loss": 3.4327908937282534,
+ "tokens_seen": 2193424384
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002710852557673019,
+ "loss": 1.2537,
+ "theoretical_loss": 3.432782566176386,
+ "tokens_seen": 2193489920
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002710692076228686,
+ "loss": 1.2576,
+ "theoretical_loss": 3.432774238942985,
+ "tokens_seen": 2193555456
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002710531594784353,
+ "loss": 1.242,
+ "theoretical_loss": 3.432765912028029,
+ "tokens_seen": 2193620992
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000271037111334002,
+ "loss": 1.2886,
+ "theoretical_loss": 3.4327575854314953,
+ "tokens_seen": 2193686528
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002710210631895687,
+ "loss": 1.1737,
+ "theoretical_loss": 3.4327492591533626,
+ "tokens_seen": 2193752064
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002710050150451354,
+ "loss": 1.2621,
+ "theoretical_loss": 3.4327409331936094,
+ "tokens_seen": 2193817600
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709889669007021,
+ "loss": 1.337,
+ "theoretical_loss": 3.432732607552214,
+ "tokens_seen": 2193883136
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709729187562688,
+ "loss": 1.3282,
+ "theoretical_loss": 3.4327242822291546,
+ "tokens_seen": 2193948672
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709568706118355,
+ "loss": 1.197,
+ "theoretical_loss": 3.43271595722441,
+ "tokens_seen": 2194014208
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709408224674022,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4327076325379573,
+ "tokens_seen": 2194079744
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709247743229689,
+ "loss": 1.3134,
+ "theoretical_loss": 3.432699308169776,
+ "tokens_seen": 2194145280
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002709087261785356,
+ "loss": 1.1669,
+ "theoretical_loss": 3.432690984119844,
+ "tokens_seen": 2194210816
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.0002708926780341023,
+ "loss": 1.3069,
+ "theoretical_loss": 3.4326826603881395,
+ "tokens_seen": 2194276352
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.000270876629889669,
+ "loss": 1.2845,
+ "theoretical_loss": 3.432674336974641,
+ "tokens_seen": 2194341888
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027086058174523574,
+ "loss": 1.2932,
+ "theoretical_loss": 3.4326660138793272,
+ "tokens_seen": 2194407424
+ },
+ {
+ "epoch": 0.66,
+ "learning_rate": 0.00027084453360080244,
+ "loss": 1.2751,
+ "theoretical_loss": 3.432657691102176,
+ "tokens_seen": 2194472960
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027082848545636914,
+ "loss": 1.2929,
+ "theoretical_loss": 3.4326493686431654,
+ "tokens_seen": 2194538496
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027081243731193584,
+ "loss": 1.135,
+ "theoretical_loss": 3.4326410465022743,
+ "tokens_seen": 2194604032
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027079638916750255,
+ "loss": 1.1969,
+ "theoretical_loss": 3.4326327246794808,
+ "tokens_seen": 2194669568
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027078034102306925,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4326244031747635,
+ "tokens_seen": 2194735104
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027076429287863595,
+ "loss": 1.2099,
+ "theoretical_loss": 3.4326160819881006,
+ "tokens_seen": 2194800640
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027074824473420265,
+ "loss": 1.1847,
+ "theoretical_loss": 3.43260776111947,
+ "tokens_seen": 2194866176
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027073219658976935,
+ "loss": 1.212,
+ "theoretical_loss": 3.432599440568851,
+ "tokens_seen": 2194931712
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027071614844533605,
+ "loss": 1.3114,
+ "theoretical_loss": 3.432591120336221,
+ "tokens_seen": 2194997248
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027070010030090275,
+ "loss": 1.2093,
+ "theoretical_loss": 3.432582800421559,
+ "tokens_seen": 2195062784
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027068405215646945,
+ "loss": 1.2646,
+ "theoretical_loss": 3.432574480824843,
+ "tokens_seen": 2195128320
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027066800401203615,
+ "loss": 1.2579,
+ "theoretical_loss": 3.4325661615460517,
+ "tokens_seen": 2195193856
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027065195586760285,
+ "loss": 1.2372,
+ "theoretical_loss": 3.4325578425851626,
+ "tokens_seen": 2195259392
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027063590772316955,
+ "loss": 1.2807,
+ "theoretical_loss": 3.4325495239421553,
+ "tokens_seen": 2195324928
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027061985957873625,
+ "loss": 1.1864,
+ "theoretical_loss": 3.4325412056170075,
+ "tokens_seen": 2195390464
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 735101,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.258005976676941,
+ "objective/train/theoretical_loss": 3.4325328876096974,
+ "objective/train/tokens_used": 2215916000,
+ "theoretical_loss": 3.4325328876096974,
+ "tokens_seen": 2195456000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027060381143430295,
+ "loss": 1.2914,
+ "theoretical_loss": 3.4325328876096974,
+ "tokens_seen": 2195456000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027058776328986965,
+ "loss": 1.145,
+ "theoretical_loss": 3.432524569920204,
+ "tokens_seen": 2195521536
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027057171514543635,
+ "loss": 1.3145,
+ "theoretical_loss": 3.432516252548505,
+ "tokens_seen": 2195587072
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027055566700100305,
+ "loss": 1.3557,
+ "theoretical_loss": 3.432507935494579,
+ "tokens_seen": 2195652608
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027053961885656976,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4324996187584045,
+ "tokens_seen": 2195718144
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027052357071213646,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4324913023399595,
+ "tokens_seen": 2195783680
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027050752256770316,
+ "loss": 1.2614,
+ "theoretical_loss": 3.432482986239223,
+ "tokens_seen": 2195849216
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027049147442326986,
+ "loss": 1.2463,
+ "theoretical_loss": 3.432474670456173,
+ "tokens_seen": 2195914752
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002704754262788365,
+ "loss": 1.2255,
+ "theoretical_loss": 3.4324663549907877,
+ "tokens_seen": 2195980288
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002704593781344032,
+ "loss": 1.1144,
+ "theoretical_loss": 3.432458039843046,
+ "tokens_seen": 2196045824
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002704433299899699,
+ "loss": 1.1742,
+ "theoretical_loss": 3.4324497250129262,
+ "tokens_seen": 2196111360
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002704272818455366,
+ "loss": 1.2132,
+ "theoretical_loss": 3.432441410500406,
+ "tokens_seen": 2196176896
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002704112337011033,
+ "loss": 1.2292,
+ "theoretical_loss": 3.4324330963054646,
+ "tokens_seen": 2196242432
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00027039518555667,
+ "loss": 1.3344,
+ "theoretical_loss": 3.43242478242808,
+ "tokens_seen": 2196307968
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002703791374122367,
+ "loss": 1.2638,
+ "theoretical_loss": 3.432416468868231,
+ "tokens_seen": 2196373504
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002703630892678034,
+ "loss": 1.3223,
+ "theoretical_loss": 3.432408155625895,
+ "tokens_seen": 2196439040
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002703470411233701,
+ "loss": 1.178,
+ "theoretical_loss": 3.4323998427010514,
+ "tokens_seen": 2196504576
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002703309929789368,
+ "loss": 1.2271,
+ "theoretical_loss": 3.4323915300936783,
+ "tokens_seen": 2196570112
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002703149448345035,
+ "loss": 1.2382,
+ "theoretical_loss": 3.4323832178037543,
+ "tokens_seen": 2196635648
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702988966900702,
+ "loss": 1.2946,
+ "theoretical_loss": 3.4323749058312574,
+ "tokens_seen": 2196701184
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702828485456369,
+ "loss": 1.1953,
+ "theoretical_loss": 3.432366594176166,
+ "tokens_seen": 2196766720
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702668004012036,
+ "loss": 1.2693,
+ "theoretical_loss": 3.432358282838459,
+ "tokens_seen": 2196832256
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702507522567703,
+ "loss": 1.2531,
+ "theoretical_loss": 3.4323499718181147,
+ "tokens_seen": 2196897792
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000270234704112337,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4323416611151107,
+ "tokens_seen": 2196963328
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702186559679037,
+ "loss": 1.2197,
+ "theoretical_loss": 3.4323333507294267,
+ "tokens_seen": 2197028864
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002702026078234704,
+ "loss": 1.306,
+ "theoretical_loss": 3.4323250406610404,
+ "tokens_seen": 2197094400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701865596790371,
+ "loss": 1.3408,
+ "theoretical_loss": 3.43231673090993,
+ "tokens_seen": 2197159936
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701705115346038,
+ "loss": 1.3094,
+ "theoretical_loss": 3.4323084214760744,
+ "tokens_seen": 2197225472
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701544633901705,
+ "loss": 1.2987,
+ "theoretical_loss": 3.4323001123594516,
+ "tokens_seen": 2197291008
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701384152457372,
+ "loss": 1.2499,
+ "theoretical_loss": 3.4322918035600405,
+ "tokens_seen": 2197356544
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701223671013039,
+ "loss": 1.2786,
+ "theoretical_loss": 3.432283495077819,
+ "tokens_seen": 2197422080
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002701063189568706,
+ "loss": 1.326,
+ "theoretical_loss": 3.4322751869127663,
+ "tokens_seen": 2197487616
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002700902708124373,
+ "loss": 1.2045,
+ "theoretical_loss": 3.4322668790648603,
+ "tokens_seen": 2197553152
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000270074222668004,
+ "loss": 1.2295,
+ "theoretical_loss": 3.432258571534079,
+ "tokens_seen": 2197618688
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002700581745235707,
+ "loss": 1.2475,
+ "theoretical_loss": 3.432250264320402,
+ "tokens_seen": 2197684224
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002700421263791374,
+ "loss": 1.239,
+ "theoretical_loss": 3.4322419574238068,
+ "tokens_seen": 2197749760
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002700260782347041,
+ "loss": 1.2133,
+ "theoretical_loss": 3.432233650844272,
+ "tokens_seen": 2197815296
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002700100300902708,
+ "loss": 1.1795,
+ "theoretical_loss": 3.4322253445817763,
+ "tokens_seen": 2197880832
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002699939819458375,
+ "loss": 1.2336,
+ "theoretical_loss": 3.432217038636298,
+ "tokens_seen": 2197946368
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002699779338014042,
+ "loss": 1.2313,
+ "theoretical_loss": 3.4322087330078155,
+ "tokens_seen": 2198011904
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002699618856569709,
+ "loss": 1.2663,
+ "theoretical_loss": 3.432200427696307,
+ "tokens_seen": 2198077440
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002699458375125376,
+ "loss": 1.1997,
+ "theoretical_loss": 3.432192122701752,
+ "tokens_seen": 2198142976
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002699297893681043,
+ "loss": 1.2451,
+ "theoretical_loss": 3.4321838180241278,
+ "tokens_seen": 2198208512
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000269913741223671,
+ "loss": 1.1874,
+ "theoretical_loss": 3.432175513663413,
+ "tokens_seen": 2198274048
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002698976930792377,
+ "loss": 1.2854,
+ "theoretical_loss": 3.432167209619587,
+ "tokens_seen": 2198339584
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026988164493480443,
+ "loss": 1.2318,
+ "theoretical_loss": 3.4321589058926274,
+ "tokens_seen": 2198405120
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026986559679037113,
+ "loss": 1.2368,
+ "theoretical_loss": 3.432150602482513,
+ "tokens_seen": 2198470656
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026984954864593783,
+ "loss": 1.2765,
+ "theoretical_loss": 3.4321422993892217,
+ "tokens_seen": 2198536192
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026983350050150453,
+ "loss": 1.0955,
+ "theoretical_loss": 3.432133996612732,
+ "tokens_seen": 2198601728
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026981745235707123,
+ "loss": 1.2752,
+ "theoretical_loss": 3.432125694153024,
+ "tokens_seen": 2198667264
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 736290,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.264587163925171,
+ "objective/train/theoretical_loss": 3.432117392010074,
+ "objective/train/tokens_used": 2219192800,
+ "theoretical_loss": 3.432117392010074,
+ "tokens_seen": 2198732800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026980140421263793,
+ "loss": 1.1972,
+ "theoretical_loss": 3.432117392010074,
+ "tokens_seen": 2198732800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026978535606820463,
+ "loss": 1.2159,
+ "theoretical_loss": 3.432109090183862,
+ "tokens_seen": 2198798336
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026976930792377133,
+ "loss": 1.3173,
+ "theoretical_loss": 3.432100788674365,
+ "tokens_seen": 2198863872
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026975325977933803,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4320924874815635,
+ "tokens_seen": 2198929408
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026973721163490473,
+ "loss": 1.2741,
+ "theoretical_loss": 3.432084186605434,
+ "tokens_seen": 2198994944
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026972116349047143,
+ "loss": 1.2819,
+ "theoretical_loss": 3.4320758860459564,
+ "tokens_seen": 2199060480
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002697051153460381,
+ "loss": 1.3089,
+ "theoretical_loss": 3.4320675858031082,
+ "tokens_seen": 2199126016
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002696890672016048,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4320592858768686,
+ "tokens_seen": 2199191552
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002696730190571715,
+ "loss": 1.2121,
+ "theoretical_loss": 3.4320509862672157,
+ "tokens_seen": 2199257088
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026965697091273824,
+ "loss": 1.2555,
+ "theoretical_loss": 3.432042686974128,
+ "tokens_seen": 2199322624
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026964092276830494,
+ "loss": 1.2942,
+ "theoretical_loss": 3.432034387997584,
+ "tokens_seen": 2199388160
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026962487462387164,
+ "loss": 1.312,
+ "theoretical_loss": 3.4320260893375623,
+ "tokens_seen": 2199453696
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026960882647943834,
+ "loss": 1.3241,
+ "theoretical_loss": 3.4320177909940415,
+ "tokens_seen": 2199519232
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026959277833500504,
+ "loss": 1.249,
+ "theoretical_loss": 3.432009492967,
+ "tokens_seen": 2199584768
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026957673019057174,
+ "loss": 1.3359,
+ "theoretical_loss": 3.432001195256416,
+ "tokens_seen": 2199650304
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026956068204613844,
+ "loss": 1.2252,
+ "theoretical_loss": 3.4319928978622687,
+ "tokens_seen": 2199715840
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026954463390170514,
+ "loss": 1.2686,
+ "theoretical_loss": 3.431984600784536,
+ "tokens_seen": 2199781376
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026952858575727184,
+ "loss": 1.2417,
+ "theoretical_loss": 3.4319763040231965,
+ "tokens_seen": 2199846912
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026951253761283854,
+ "loss": 1.3448,
+ "theoretical_loss": 3.431968007578229,
+ "tokens_seen": 2199912448
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026949648946840524,
+ "loss": 1.2031,
+ "theoretical_loss": 3.431959711449611,
+ "tokens_seen": 2199977984
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026948044132397194,
+ "loss": 1.1484,
+ "theoretical_loss": 3.431951415637323,
+ "tokens_seen": 2200043520
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026946439317953864,
+ "loss": 1.3712,
+ "theoretical_loss": 3.4319431201413417,
+ "tokens_seen": 2200109056
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026944834503510534,
+ "loss": 1.2543,
+ "theoretical_loss": 3.431934824961646,
+ "tokens_seen": 2200174592
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026943229689067205,
+ "loss": 1.3211,
+ "theoretical_loss": 3.4319265300982154,
+ "tokens_seen": 2200240128
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026941624874623875,
+ "loss": 1.2639,
+ "theoretical_loss": 3.431918235551027,
+ "tokens_seen": 2200305664
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026940020060180545,
+ "loss": 1.3417,
+ "theoretical_loss": 3.4319099413200607,
+ "tokens_seen": 2200371200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026938415245737215,
+ "loss": 1.3156,
+ "theoretical_loss": 3.4319016474052937,
+ "tokens_seen": 2200436736
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026936810431293885,
+ "loss": 1.3307,
+ "theoretical_loss": 3.431893353806706,
+ "tokens_seen": 2200502272
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026935205616850555,
+ "loss": 1.0877,
+ "theoretical_loss": 3.4318850605242743,
+ "tokens_seen": 2200567808
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026933600802407225,
+ "loss": 1.2511,
+ "theoretical_loss": 3.431876767557979,
+ "tokens_seen": 2200633344
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026931995987963895,
+ "loss": 1.2661,
+ "theoretical_loss": 3.4318684749077972,
+ "tokens_seen": 2200698880
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026930391173520565,
+ "loss": 1.2161,
+ "theoretical_loss": 3.431860182573708,
+ "tokens_seen": 2200764416
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026928786359077235,
+ "loss": 1.1842,
+ "theoretical_loss": 3.4318518905556905,
+ "tokens_seen": 2200829952
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026927181544633905,
+ "loss": 1.2475,
+ "theoretical_loss": 3.431843598853722,
+ "tokens_seen": 2200895488
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026925576730190575,
+ "loss": 1.1703,
+ "theoretical_loss": 3.4318353074677823,
+ "tokens_seen": 2200961024
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026923971915747245,
+ "loss": 1.1451,
+ "theoretical_loss": 3.431827016397849,
+ "tokens_seen": 2201026560
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026922367101303915,
+ "loss": 1.2075,
+ "theoretical_loss": 3.4318187256439012,
+ "tokens_seen": 2201092096
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026920762286860585,
+ "loss": 1.2883,
+ "theoretical_loss": 3.431810435205917,
+ "tokens_seen": 2201157632
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026919157472417255,
+ "loss": 1.2326,
+ "theoretical_loss": 3.431802145083876,
+ "tokens_seen": 2201223168
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026917552657973926,
+ "loss": 1.2004,
+ "theoretical_loss": 3.431793855277755,
+ "tokens_seen": 2201288704
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026915947843530596,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4317855657875342,
+ "tokens_seen": 2201354240
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026914343029087266,
+ "loss": 1.3181,
+ "theoretical_loss": 3.4317772766131913,
+ "tokens_seen": 2201419776
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026912738214643936,
+ "loss": 1.2636,
+ "theoretical_loss": 3.431768987754705,
+ "tokens_seen": 2201485312
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026911133400200606,
+ "loss": 1.2769,
+ "theoretical_loss": 3.431760699212054,
+ "tokens_seen": 2201550848
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026909528585757276,
+ "loss": 1.3557,
+ "theoretical_loss": 3.431752410985217,
+ "tokens_seen": 2201616384
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002690792377131394,
+ "loss": 1.2522,
+ "theoretical_loss": 3.4317441230741723,
+ "tokens_seen": 2201681920
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002690631895687061,
+ "loss": 1.2836,
+ "theoretical_loss": 3.4317358354788983,
+ "tokens_seen": 2201747456
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002690471414242728,
+ "loss": 1.2729,
+ "theoretical_loss": 3.431727548199374,
+ "tokens_seen": 2201812992
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002690310932798395,
+ "loss": 1.199,
+ "theoretical_loss": 3.4317192612355774,
+ "tokens_seen": 2201878528
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002690150451354062,
+ "loss": 1.3288,
+ "theoretical_loss": 3.431710974587488,
+ "tokens_seen": 2201944064
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 737257,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1944656372070312,
+ "objective/train/theoretical_loss": 3.4317026882550827,
+ "objective/train/tokens_used": 2222469600,
+ "theoretical_loss": 3.4317026882550827,
+ "tokens_seen": 2202009600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689989969909729,
+ "loss": 1.2246,
+ "theoretical_loss": 3.4317026882550827,
+ "tokens_seen": 2202009600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689829488465396,
+ "loss": 1.2792,
+ "theoretical_loss": 3.4316944022383424,
+ "tokens_seen": 2202075136
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689669007021063,
+ "loss": 1.1584,
+ "theoretical_loss": 3.431686116537244,
+ "tokens_seen": 2202140672
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000268950852557673,
+ "loss": 1.274,
+ "theoretical_loss": 3.4316778311517666,
+ "tokens_seen": 2202206208
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689348044132397,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4316695460818885,
+ "tokens_seen": 2202271744
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689187562688064,
+ "loss": 1.2094,
+ "theoretical_loss": 3.431661261327589,
+ "tokens_seen": 2202337280
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002689027081243731,
+ "loss": 1.219,
+ "theoretical_loss": 3.431652976888846,
+ "tokens_seen": 2202402816
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688866599799398,
+ "loss": 1.2498,
+ "theoretical_loss": 3.431644692765638,
+ "tokens_seen": 2202468352
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688706118355065,
+ "loss": 1.2601,
+ "theoretical_loss": 3.4316364089579445,
+ "tokens_seen": 2202533888
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688545636910732,
+ "loss": 1.1866,
+ "theoretical_loss": 3.431628125465743,
+ "tokens_seen": 2202599424
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688385155466399,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4316198422890127,
+ "tokens_seen": 2202664960
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688224674022066,
+ "loss": 1.2937,
+ "theoretical_loss": 3.4316115594277323,
+ "tokens_seen": 2202730496
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002688064192577733,
+ "loss": 1.2418,
+ "theoretical_loss": 3.43160327688188,
+ "tokens_seen": 2202796032
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026879037111334,
+ "loss": 1.1501,
+ "theoretical_loss": 3.4315949946514346,
+ "tokens_seen": 2202861568
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002687743229689067,
+ "loss": 1.2672,
+ "theoretical_loss": 3.431586712736375,
+ "tokens_seen": 2202927104
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002687582748244734,
+ "loss": 1.1957,
+ "theoretical_loss": 3.4315784311366793,
+ "tokens_seen": 2202992640
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002687422266800401,
+ "loss": 1.251,
+ "theoretical_loss": 3.4315701498523263,
+ "tokens_seen": 2203058176
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002687261785356068,
+ "loss": 1.2124,
+ "theoretical_loss": 3.431561868883295,
+ "tokens_seen": 2203123712
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002687101303911735,
+ "loss": 1.2345,
+ "theoretical_loss": 3.431553588229563,
+ "tokens_seen": 2203189248
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002686940822467402,
+ "loss": 1.2779,
+ "theoretical_loss": 3.4315453078911102,
+ "tokens_seen": 2203254784
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002686780341023069,
+ "loss": 1.2664,
+ "theoretical_loss": 3.4315370278679143,
+ "tokens_seen": 2203320320
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002686619859578736,
+ "loss": 1.2021,
+ "theoretical_loss": 3.431528748159954,
+ "tokens_seen": 2203385856
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002686459378134403,
+ "loss": 1.235,
+ "theoretical_loss": 3.4315204687672085,
+ "tokens_seen": 2203451392
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000268629889669007,
+ "loss": 1.3057,
+ "theoretical_loss": 3.4315121896896557,
+ "tokens_seen": 2203516928
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002686138415245737,
+ "loss": 1.0619,
+ "theoretical_loss": 3.4315039109272747,
+ "tokens_seen": 2203582464
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002685977933801404,
+ "loss": 1.1764,
+ "theoretical_loss": 3.4314956324800443,
+ "tokens_seen": 2203648000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002685817452357071,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4314873543479427,
+ "tokens_seen": 2203713536
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002685656970912738,
+ "loss": 1.1819,
+ "theoretical_loss": 3.4314790765309486,
+ "tokens_seen": 2203779072
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002685496489468405,
+ "loss": 1.4157,
+ "theoretical_loss": 3.4314707990290407,
+ "tokens_seen": 2203844608
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002685336008024072,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4314625218421977,
+ "tokens_seen": 2203910144
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026851755265797393,
+ "loss": 1.2681,
+ "theoretical_loss": 3.431454244970398,
+ "tokens_seen": 2203975680
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026850150451354063,
+ "loss": 1.2259,
+ "theoretical_loss": 3.431445968413621,
+ "tokens_seen": 2204041216
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026848545636910733,
+ "loss": 1.1786,
+ "theoretical_loss": 3.431437692171844,
+ "tokens_seen": 2204106752
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026846940822467403,
+ "loss": 1.2506,
+ "theoretical_loss": 3.431429416245047,
+ "tokens_seen": 2204172288
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026845336008024073,
+ "loss": 1.3745,
+ "theoretical_loss": 3.4314211406332076,
+ "tokens_seen": 2204237824
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026843731193580743,
+ "loss": 1.1679,
+ "theoretical_loss": 3.4314128653363056,
+ "tokens_seen": 2204303360
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026842126379137413,
+ "loss": 1.2446,
+ "theoretical_loss": 3.4314045903543184,
+ "tokens_seen": 2204368896
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026840521564694083,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4313963156872256,
+ "tokens_seen": 2204434432
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026838916750250753,
+ "loss": 1.2546,
+ "theoretical_loss": 3.431388041335005,
+ "tokens_seen": 2204499968
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026837311935807423,
+ "loss": 1.2129,
+ "theoretical_loss": 3.4313797672976363,
+ "tokens_seen": 2204565504
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026835707121364093,
+ "loss": 1.267,
+ "theoretical_loss": 3.431371493575097,
+ "tokens_seen": 2204631040
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026834102306920763,
+ "loss": 1.318,
+ "theoretical_loss": 3.4313632201673667,
+ "tokens_seen": 2204696576
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026832497492477433,
+ "loss": 1.293,
+ "theoretical_loss": 3.4313549470744236,
+ "tokens_seen": 2204762112
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026830892678034104,
+ "loss": 1.2878,
+ "theoretical_loss": 3.431346674296247,
+ "tokens_seen": 2204827648
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026829287863590774,
+ "loss": 1.3415,
+ "theoretical_loss": 3.4313384018328144,
+ "tokens_seen": 2204893184
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026827683049147444,
+ "loss": 1.1541,
+ "theoretical_loss": 3.4313301296841052,
+ "tokens_seen": 2204958720
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026826078234704114,
+ "loss": 1.2497,
+ "theoretical_loss": 3.431321857850098,
+ "tokens_seen": 2205024256
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026824473420260784,
+ "loss": 1.1723,
+ "theoretical_loss": 3.4313135863307718,
+ "tokens_seen": 2205089792
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026822868605817454,
+ "loss": 1.2063,
+ "theoretical_loss": 3.431305315126105,
+ "tokens_seen": 2205155328
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026821263791374124,
+ "loss": 1.2542,
+ "theoretical_loss": 3.431297044236076,
+ "tokens_seen": 2205220864
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 738571,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2584826946258545,
+ "objective/train/theoretical_loss": 3.431288773660664,
+ "objective/train/tokens_used": 2225746400,
+ "theoretical_loss": 3.431288773660664,
+ "tokens_seen": 2205286400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026819658976930794,
+ "loss": 1.2553,
+ "theoretical_loss": 3.431288773660664,
+ "tokens_seen": 2205286400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026818054162487464,
+ "loss": 1.2945,
+ "theoretical_loss": 3.431280503399847,
+ "tokens_seen": 2205351936
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026816449348044134,
+ "loss": 1.298,
+ "theoretical_loss": 3.4312722334536043,
+ "tokens_seen": 2205417472
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026814844533600804,
+ "loss": 1.2904,
+ "theoretical_loss": 3.4312639638219142,
+ "tokens_seen": 2205483008
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026813239719157474,
+ "loss": 1.3272,
+ "theoretical_loss": 3.4312556945047556,
+ "tokens_seen": 2205548544
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026811634904714144,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4312474255021073,
+ "tokens_seen": 2205614080
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026810030090270814,
+ "loss": 1.2232,
+ "theoretical_loss": 3.431239156813948,
+ "tokens_seen": 2205679616
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026808425275827484,
+ "loss": 1.251,
+ "theoretical_loss": 3.431230888440256,
+ "tokens_seen": 2205745152
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026806820461384154,
+ "loss": 1.2148,
+ "theoretical_loss": 3.43122262038101,
+ "tokens_seen": 2205810688
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026805215646940825,
+ "loss": 1.1803,
+ "theoretical_loss": 3.4312143526361893,
+ "tokens_seen": 2205876224
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026803610832497495,
+ "loss": 1.282,
+ "theoretical_loss": 3.431206085205772,
+ "tokens_seen": 2205941760
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026802006018054165,
+ "loss": 1.1219,
+ "theoretical_loss": 3.4311978180897373,
+ "tokens_seen": 2206007296
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026800401203610835,
+ "loss": 1.1861,
+ "theoretical_loss": 3.431189551288064,
+ "tokens_seen": 2206072832
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026798796389167505,
+ "loss": 1.293,
+ "theoretical_loss": 3.4311812848007297,
+ "tokens_seen": 2206138368
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026797191574724175,
+ "loss": 1.2752,
+ "theoretical_loss": 3.4311730186277143,
+ "tokens_seen": 2206203904
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026795586760280845,
+ "loss": 1.2216,
+ "theoretical_loss": 3.431164752768996,
+ "tokens_seen": 2206269440
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026793981945837515,
+ "loss": 1.3175,
+ "theoretical_loss": 3.4311564872245532,
+ "tokens_seen": 2206334976
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026792377131394185,
+ "loss": 1.3222,
+ "theoretical_loss": 3.4311482219943654,
+ "tokens_seen": 2206400512
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026790772316950855,
+ "loss": 1.2963,
+ "theoretical_loss": 3.431139957078411,
+ "tokens_seen": 2206466048
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026789167502507525,
+ "loss": 1.2396,
+ "theoretical_loss": 3.4311316924766686,
+ "tokens_seen": 2206531584
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026787562688064195,
+ "loss": 1.2999,
+ "theoretical_loss": 3.431123428189117,
+ "tokens_seen": 2206597120
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026785957873620865,
+ "loss": 1.3069,
+ "theoretical_loss": 3.431115164215735,
+ "tokens_seen": 2206662656
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026784353059177535,
+ "loss": 1.3743,
+ "theoretical_loss": 3.4311069005565007,
+ "tokens_seen": 2206728192
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026782748244734205,
+ "loss": 1.2237,
+ "theoretical_loss": 3.431098637211394,
+ "tokens_seen": 2206793728
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026781143430290876,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4310903741803926,
+ "tokens_seen": 2206859264
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026779538615847546,
+ "loss": 1.1995,
+ "theoretical_loss": 3.4310821114634757,
+ "tokens_seen": 2206924800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026777933801404216,
+ "loss": 1.2101,
+ "theoretical_loss": 3.431073849060622,
+ "tokens_seen": 2206990336
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026776328986960886,
+ "loss": 1.2514,
+ "theoretical_loss": 3.43106558697181,
+ "tokens_seen": 2207055872
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026774724172517556,
+ "loss": 1.2366,
+ "theoretical_loss": 3.4310573251970187,
+ "tokens_seen": 2207121408
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026773119358074226,
+ "loss": 1.3865,
+ "theoretical_loss": 3.4310490637362268,
+ "tokens_seen": 2207186944
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026771514543630896,
+ "loss": 1.1972,
+ "theoretical_loss": 3.431040802589413,
+ "tokens_seen": 2207252480
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026769909729187566,
+ "loss": 1.3258,
+ "theoretical_loss": 3.431032541756556,
+ "tokens_seen": 2207318016
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676830491474423,
+ "loss": 1.2836,
+ "theoretical_loss": 3.431024281237635,
+ "tokens_seen": 2207383552
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000267667001003009,
+ "loss": 1.2401,
+ "theoretical_loss": 3.4310160210326277,
+ "tokens_seen": 2207449088
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676509528585757,
+ "loss": 1.3105,
+ "theoretical_loss": 3.431007761141514,
+ "tokens_seen": 2207514624
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676349047141424,
+ "loss": 1.2134,
+ "theoretical_loss": 3.4309995015642722,
+ "tokens_seen": 2207580160
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676349047141424,
+ "loss": 1.3403,
+ "theoretical_loss": 3.430991242300881,
+ "tokens_seen": 2207645696
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676188565697091,
+ "loss": 1.3762,
+ "theoretical_loss": 3.430982983351319,
+ "tokens_seen": 2207711232
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002676028084252758,
+ "loss": 1.316,
+ "theoretical_loss": 3.430974724715565,
+ "tokens_seen": 2207776768
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002675867602808425,
+ "loss": 1.2614,
+ "theoretical_loss": 3.430966466393598,
+ "tokens_seen": 2207842304
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002675707121364092,
+ "loss": 1.2531,
+ "theoretical_loss": 3.430958208385397,
+ "tokens_seen": 2207907840
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002675546639919759,
+ "loss": 1.1853,
+ "theoretical_loss": 3.4309499506909398,
+ "tokens_seen": 2207973376
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002675386158475426,
+ "loss": 1.3063,
+ "theoretical_loss": 3.4309416933102064,
+ "tokens_seen": 2208038912
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002675225677031093,
+ "loss": 1.3596,
+ "theoretical_loss": 3.4309334362431745,
+ "tokens_seen": 2208104448
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000267506519558676,
+ "loss": 1.2408,
+ "theoretical_loss": 3.4309251794898237,
+ "tokens_seen": 2208169984
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674904714142427,
+ "loss": 1.2789,
+ "theoretical_loss": 3.430916923050132,
+ "tokens_seen": 2208235520
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674744232698094,
+ "loss": 1.4038,
+ "theoretical_loss": 3.430908666924079,
+ "tokens_seen": 2208301056
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674583751253761,
+ "loss": 1.2123,
+ "theoretical_loss": 3.430900411111643,
+ "tokens_seen": 2208366592
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674423269809428,
+ "loss": 1.1715,
+ "theoretical_loss": 3.4308921556128027,
+ "tokens_seen": 2208432128
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674262788365095,
+ "loss": 1.2378,
+ "theoretical_loss": 3.430883900427537,
+ "tokens_seen": 2208497664
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 739468,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2530725002288818,
+ "objective/train/theoretical_loss": 3.4308756455558247,
+ "objective/train/tokens_used": 2229023200,
+ "theoretical_loss": 3.4308756455558247,
+ "tokens_seen": 2208563200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002674102306920762,
+ "loss": 1.1894,
+ "theoretical_loss": 3.4308756455558247,
+ "tokens_seen": 2208563200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002673941825476429,
+ "loss": 1.2515,
+ "theoretical_loss": 3.430867390997645,
+ "tokens_seen": 2208628736
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002673781344032096,
+ "loss": 1.2938,
+ "theoretical_loss": 3.430859136752976,
+ "tokens_seen": 2208694272
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002673620862587763,
+ "loss": 1.2841,
+ "theoretical_loss": 3.4308508828217965,
+ "tokens_seen": 2208759808
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000267346038114343,
+ "loss": 1.2789,
+ "theoretical_loss": 3.430842629204086,
+ "tokens_seen": 2208825344
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002673299899699097,
+ "loss": 1.1935,
+ "theoretical_loss": 3.4308343758998223,
+ "tokens_seen": 2208890880
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002673139418254764,
+ "loss": 1.2613,
+ "theoretical_loss": 3.4308261229089854,
+ "tokens_seen": 2208956416
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672978936810431,
+ "loss": 1.3262,
+ "theoretical_loss": 3.4308178702315533,
+ "tokens_seen": 2209021952
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672818455366098,
+ "loss": 1.3713,
+ "theoretical_loss": 3.4308096178675047,
+ "tokens_seen": 2209087488
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672657973921766,
+ "loss": 1.2207,
+ "theoretical_loss": 3.430801365816819,
+ "tokens_seen": 2209153024
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672497492477433,
+ "loss": 1.3124,
+ "theoretical_loss": 3.4307931140794743,
+ "tokens_seen": 2209218560
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026723370110331,
+ "loss": 1.2379,
+ "theoretical_loss": 3.4307848626554502,
+ "tokens_seen": 2209284096
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672176529588767,
+ "loss": 1.2952,
+ "theoretical_loss": 3.430776611544725,
+ "tokens_seen": 2209349632
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002672016048144434,
+ "loss": 1.2849,
+ "theoretical_loss": 3.4307683607472774,
+ "tokens_seen": 2209415168
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671855566700101,
+ "loss": 1.2368,
+ "theoretical_loss": 3.4307601102630865,
+ "tokens_seen": 2209480704
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671695085255768,
+ "loss": 1.2818,
+ "theoretical_loss": 3.4307518600921307,
+ "tokens_seen": 2209546240
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671534603811435,
+ "loss": 1.2495,
+ "theoretical_loss": 3.43074361023439,
+ "tokens_seen": 2209611776
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671374122367102,
+ "loss": 1.2528,
+ "theoretical_loss": 3.430735360689842,
+ "tokens_seen": 2209677312
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671213640922769,
+ "loss": 1.2654,
+ "theoretical_loss": 3.430727111458465,
+ "tokens_seen": 2209742848
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002671053159478436,
+ "loss": 1.3642,
+ "theoretical_loss": 3.43071886254024,
+ "tokens_seen": 2209808384
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002670892678034103,
+ "loss": 1.1572,
+ "theoretical_loss": 3.430710613935144,
+ "tokens_seen": 2209873920
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000267073219658977,
+ "loss": 1.2414,
+ "theoretical_loss": 3.4307023656431563,
+ "tokens_seen": 2209939456
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002670571715145437,
+ "loss": 1.3203,
+ "theoretical_loss": 3.4306941176642556,
+ "tokens_seen": 2210004992
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026704112337011033,
+ "loss": 1.1671,
+ "theoretical_loss": 3.4306858699984213,
+ "tokens_seen": 2210070528
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026702507522567703,
+ "loss": 1.3222,
+ "theoretical_loss": 3.4306776226456317,
+ "tokens_seen": 2210136064
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026700902708124373,
+ "loss": 1.3078,
+ "theoretical_loss": 3.430669375605866,
+ "tokens_seen": 2210201600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026699297893681043,
+ "loss": 1.245,
+ "theoretical_loss": 3.4306611288791027,
+ "tokens_seen": 2210267136
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026697693079237713,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4306528824653206,
+ "tokens_seen": 2210332672
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026696088264794383,
+ "loss": 1.2031,
+ "theoretical_loss": 3.430644636364499,
+ "tokens_seen": 2210398208
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026694483450351054,
+ "loss": 1.2827,
+ "theoretical_loss": 3.430636390576616,
+ "tokens_seen": 2210463744
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026692878635907724,
+ "loss": 1.1262,
+ "theoretical_loss": 3.4306281451016516,
+ "tokens_seen": 2210529280
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026691273821464394,
+ "loss": 1.2952,
+ "theoretical_loss": 3.4306198999395834,
+ "tokens_seen": 2210594816
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026689669007021064,
+ "loss": 1.2693,
+ "theoretical_loss": 3.430611655090391,
+ "tokens_seen": 2210660352
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026688064192577734,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4306034105540535,
+ "tokens_seen": 2210725888
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026686459378134404,
+ "loss": 1.2148,
+ "theoretical_loss": 3.4305951663305487,
+ "tokens_seen": 2210791424
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026684854563691074,
+ "loss": 1.3327,
+ "theoretical_loss": 3.4305869224198564,
+ "tokens_seen": 2210856960
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026683249749247744,
+ "loss": 1.2478,
+ "theoretical_loss": 3.4305786788219548,
+ "tokens_seen": 2210922496
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026681644934804414,
+ "loss": 1.2099,
+ "theoretical_loss": 3.4305704355368234,
+ "tokens_seen": 2210988032
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026680040120361084,
+ "loss": 1.282,
+ "theoretical_loss": 3.430562192564441,
+ "tokens_seen": 2211053568
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026678435305917754,
+ "loss": 1.2132,
+ "theoretical_loss": 3.4305539499047857,
+ "tokens_seen": 2211119104
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026676830491474424,
+ "loss": 1.1741,
+ "theoretical_loss": 3.4305457075578367,
+ "tokens_seen": 2211184640
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026675225677031094,
+ "loss": 1.2348,
+ "theoretical_loss": 3.4305374655235736,
+ "tokens_seen": 2211250176
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026673620862587764,
+ "loss": 1.1937,
+ "theoretical_loss": 3.4305292238019747,
+ "tokens_seen": 2211315712
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026672016048144434,
+ "loss": 1.2857,
+ "theoretical_loss": 3.430520982393019,
+ "tokens_seen": 2211381248
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026670411233701104,
+ "loss": 1.2855,
+ "theoretical_loss": 3.4305127412966847,
+ "tokens_seen": 2211446784
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026668806419257775,
+ "loss": 1.3192,
+ "theoretical_loss": 3.430504500512952,
+ "tokens_seen": 2211512320
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026667201604814445,
+ "loss": 1.2752,
+ "theoretical_loss": 3.4304962600417985,
+ "tokens_seen": 2211577856
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026665596790371115,
+ "loss": 1.3304,
+ "theoretical_loss": 3.430488019883204,
+ "tokens_seen": 2211643392
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026663991975927785,
+ "loss": 1.2509,
+ "theoretical_loss": 3.4304797800371465,
+ "tokens_seen": 2211708928
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026662387161484455,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4304715405036057,
+ "tokens_seen": 2211774464
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 740733,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.34280264377594,
+ "objective/train/theoretical_loss": 3.43046330128256,
+ "objective/train/tokens_used": 2232300000,
+ "theoretical_loss": 3.43046330128256,
+ "tokens_seen": 2211840000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026660782347041125,
+ "loss": 1.3035,
+ "theoretical_loss": 3.43046330128256,
+ "tokens_seen": 2211840000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026659177532597795,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4304550623739885,
+ "tokens_seen": 2211905536
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026657572718154465,
+ "loss": 1.2365,
+ "theoretical_loss": 3.4304468237778702,
+ "tokens_seen": 2211971072
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026655967903711135,
+ "loss": 1.3074,
+ "theoretical_loss": 3.4304385854941835,
+ "tokens_seen": 2212036608
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026654363089267805,
+ "loss": 1.3366,
+ "theoretical_loss": 3.430430347522908,
+ "tokens_seen": 2212102144
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026652758274824475,
+ "loss": 1.2185,
+ "theoretical_loss": 3.430422109864022,
+ "tokens_seen": 2212167680
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026651153460381145,
+ "loss": 1.2484,
+ "theoretical_loss": 3.430413872517505,
+ "tokens_seen": 2212233216
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026649548645937815,
+ "loss": 1.1955,
+ "theoretical_loss": 3.4304056354833357,
+ "tokens_seen": 2212298752
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026647943831494485,
+ "loss": 1.1223,
+ "theoretical_loss": 3.4303973987614924,
+ "tokens_seen": 2212364288
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026646339017051155,
+ "loss": 1.2724,
+ "theoretical_loss": 3.430389162351954,
+ "tokens_seen": 2212429824
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026644734202607825,
+ "loss": 1.2329,
+ "theoretical_loss": 3.430380926254701,
+ "tokens_seen": 2212495360
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026643129388164496,
+ "loss": 1.3075,
+ "theoretical_loss": 3.43037269046971,
+ "tokens_seen": 2212560896
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026641524573721166,
+ "loss": 1.2121,
+ "theoretical_loss": 3.430364454996962,
+ "tokens_seen": 2212626432
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026639919759277836,
+ "loss": 1.223,
+ "theoretical_loss": 3.4303562198364346,
+ "tokens_seen": 2212691968
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026638314944834506,
+ "loss": 1.2591,
+ "theoretical_loss": 3.4303479849881073,
+ "tokens_seen": 2212757504
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026636710130391176,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4303397504519584,
+ "tokens_seen": 2212823040
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026635105315947846,
+ "loss": 1.2788,
+ "theoretical_loss": 3.4303315162279677,
+ "tokens_seen": 2212888576
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026633500501504516,
+ "loss": 1.3317,
+ "theoretical_loss": 3.4303232823161136,
+ "tokens_seen": 2212954112
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026631895687061186,
+ "loss": 1.2292,
+ "theoretical_loss": 3.4303150487163747,
+ "tokens_seen": 2213019648
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026630290872617856,
+ "loss": 1.301,
+ "theoretical_loss": 3.4303068154287306,
+ "tokens_seen": 2213085184
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026628686058174526,
+ "loss": 1.2443,
+ "theoretical_loss": 3.43029858245316,
+ "tokens_seen": 2213150720
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002662708124373119,
+ "loss": 1.3267,
+ "theoretical_loss": 3.430290349789642,
+ "tokens_seen": 2213216256
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002662547642928786,
+ "loss": 1.2838,
+ "theoretical_loss": 3.4302821174381553,
+ "tokens_seen": 2213281792
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002662387161484453,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4302738853986785,
+ "tokens_seen": 2213347328
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000266222668004012,
+ "loss": 1.1883,
+ "theoretical_loss": 3.4302656536711913,
+ "tokens_seen": 2213412864
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002662066198595787,
+ "loss": 1.316,
+ "theoretical_loss": 3.430257422255672,
+ "tokens_seen": 2213478400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661905717151454,
+ "loss": 1.2652,
+ "theoretical_loss": 3.4302491911520994,
+ "tokens_seen": 2213543936
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661745235707121,
+ "loss": 1.3176,
+ "theoretical_loss": 3.4302409603604533,
+ "tokens_seen": 2213609472
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661584754262788,
+ "loss": 1.3476,
+ "theoretical_loss": 3.4302327298807116,
+ "tokens_seen": 2213675008
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661424272818455,
+ "loss": 1.1453,
+ "theoretical_loss": 3.4302244997128546,
+ "tokens_seen": 2213740544
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661263791374122,
+ "loss": 1.2118,
+ "theoretical_loss": 3.43021626985686,
+ "tokens_seen": 2213806080
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002661103309929789,
+ "loss": 1.2844,
+ "theoretical_loss": 3.4302080403127073,
+ "tokens_seen": 2213871616
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002660942828485456,
+ "loss": 1.1662,
+ "theoretical_loss": 3.4301998110803753,
+ "tokens_seen": 2213937152
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002660782347041123,
+ "loss": 1.2062,
+ "theoretical_loss": 3.4301915821598428,
+ "tokens_seen": 2214002688
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026606218655967907,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4301833535510893,
+ "tokens_seen": 2214068224
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026604613841524577,
+ "loss": 1.4501,
+ "theoretical_loss": 3.4301751252540935,
+ "tokens_seen": 2214133760
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026603009027081247,
+ "loss": 1.2543,
+ "theoretical_loss": 3.4301668972688337,
+ "tokens_seen": 2214199296
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026601404212637917,
+ "loss": 1.2645,
+ "theoretical_loss": 3.43015866959529,
+ "tokens_seen": 2214264832
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026599799398194587,
+ "loss": 1.3218,
+ "theoretical_loss": 3.430150442233441,
+ "tokens_seen": 2214330368
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002659819458375126,
+ "loss": 1.2636,
+ "theoretical_loss": 3.430142215183265,
+ "tokens_seen": 2214395904
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002659658976930793,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4301339884447417,
+ "tokens_seen": 2214461440
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000265949849548646,
+ "loss": 1.2001,
+ "theoretical_loss": 3.4301257620178496,
+ "tokens_seen": 2214526976
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002659338014042127,
+ "loss": 1.2217,
+ "theoretical_loss": 3.430117535902568,
+ "tokens_seen": 2214592512
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002659177532597794,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4301093100988758,
+ "tokens_seen": 2214658048
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002659017051153461,
+ "loss": 1.1896,
+ "theoretical_loss": 3.430101084606752,
+ "tokens_seen": 2214723584
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658856569709128,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4300928594261757,
+ "tokens_seen": 2214789120
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658696088264795,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4300846345571254,
+ "tokens_seen": 2214854656
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658535606820462,
+ "loss": 1.3232,
+ "theoretical_loss": 3.4300764099995806,
+ "tokens_seen": 2214920192
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658375125376129,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4300681857535196,
+ "tokens_seen": 2214985728
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658214643931796,
+ "loss": 1.283,
+ "theoretical_loss": 3.4300599618189223,
+ "tokens_seen": 2215051264
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 741599,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2668308019638062,
+ "objective/train/theoretical_loss": 3.4300517381957674,
+ "objective/train/tokens_used": 2235576800,
+ "theoretical_loss": 3.4300517381957674,
+ "tokens_seen": 2215116800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002658054162487463,
+ "loss": 1.246,
+ "theoretical_loss": 3.4300517381957674,
+ "tokens_seen": 2215116800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000265789368104313,
+ "loss": 1.2519,
+ "theoretical_loss": 3.4300435148840336,
+ "tokens_seen": 2215182336
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002657733199598797,
+ "loss": 1.2321,
+ "theoretical_loss": 3.4300352918837,
+ "tokens_seen": 2215247872
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002657572718154464,
+ "loss": 1.3017,
+ "theoretical_loss": 3.4300270691947454,
+ "tokens_seen": 2215313408
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002657412236710131,
+ "loss": 1.1633,
+ "theoretical_loss": 3.4300188468171497,
+ "tokens_seen": 2215378944
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002657251755265798,
+ "loss": 1.2899,
+ "theoretical_loss": 3.430010624750891,
+ "tokens_seen": 2215444480
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002657091273821465,
+ "loss": 1.1492,
+ "theoretical_loss": 3.430002402995948,
+ "tokens_seen": 2215510016
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002656930792377132,
+ "loss": 1.2309,
+ "theoretical_loss": 3.429994181552301,
+ "tokens_seen": 2215575552
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002656770310932799,
+ "loss": 1.3256,
+ "theoretical_loss": 3.429985960419928,
+ "tokens_seen": 2215641088
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002656609829488466,
+ "loss": 1.252,
+ "theoretical_loss": 3.429977739598808,
+ "tokens_seen": 2215706624
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026564493480441323,
+ "loss": 1.2215,
+ "theoretical_loss": 3.42996951908892,
+ "tokens_seen": 2215772160
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026562888665997993,
+ "loss": 1.2155,
+ "theoretical_loss": 3.4299612988902437,
+ "tokens_seen": 2215837696
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026561283851554663,
+ "loss": 1.2592,
+ "theoretical_loss": 3.429953079002758,
+ "tokens_seen": 2215903232
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026559679037111333,
+ "loss": 1.3305,
+ "theoretical_loss": 3.429944859426441,
+ "tokens_seen": 2215968768
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026558074222668003,
+ "loss": 1.2528,
+ "theoretical_loss": 3.429936640161273,
+ "tokens_seen": 2216034304
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026556469408224674,
+ "loss": 1.1434,
+ "theoretical_loss": 3.4299284212072316,
+ "tokens_seen": 2216099840
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026554864593781344,
+ "loss": 1.2164,
+ "theoretical_loss": 3.429920202564297,
+ "tokens_seen": 2216165376
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026553259779338014,
+ "loss": 1.2262,
+ "theoretical_loss": 3.429911984232448,
+ "tokens_seen": 2216230912
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026551654964894684,
+ "loss": 1.2188,
+ "theoretical_loss": 3.429903766211663,
+ "tokens_seen": 2216296448
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026550050150451354,
+ "loss": 1.2551,
+ "theoretical_loss": 3.429895548501922,
+ "tokens_seen": 2216361984
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026548445336008024,
+ "loss": 1.2345,
+ "theoretical_loss": 3.429887331103203,
+ "tokens_seen": 2216427520
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026546840521564694,
+ "loss": 1.3294,
+ "theoretical_loss": 3.4298791140154856,
+ "tokens_seen": 2216493056
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026545235707121364,
+ "loss": 1.2265,
+ "theoretical_loss": 3.429870897238749,
+ "tokens_seen": 2216558592
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026543630892678034,
+ "loss": 1.3159,
+ "theoretical_loss": 3.4298626807729713,
+ "tokens_seen": 2216624128
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026542026078234704,
+ "loss": 1.227,
+ "theoretical_loss": 3.429854464618133,
+ "tokens_seen": 2216689664
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026540421263791374,
+ "loss": 1.2221,
+ "theoretical_loss": 3.4298462487742123,
+ "tokens_seen": 2216755200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026538816449348044,
+ "loss": 1.2271,
+ "theoretical_loss": 3.4298380332411877,
+ "tokens_seen": 2216820736
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026537211634904714,
+ "loss": 1.2427,
+ "theoretical_loss": 3.4298298180190394,
+ "tokens_seen": 2216886272
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026535606820461384,
+ "loss": 1.2649,
+ "theoretical_loss": 3.4298216031077455,
+ "tokens_seen": 2216951808
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026534002006018054,
+ "loss": 1.1281,
+ "theoretical_loss": 3.429813388507286,
+ "tokens_seen": 2217017344
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026532397191574724,
+ "loss": 1.2233,
+ "theoretical_loss": 3.429805174217639,
+ "tokens_seen": 2217082880
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026530792377131395,
+ "loss": 1.2497,
+ "theoretical_loss": 3.429796960238784,
+ "tokens_seen": 2217148416
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026529187562688065,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4297887465707,
+ "tokens_seen": 2217213952
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026527582748244735,
+ "loss": 1.1977,
+ "theoretical_loss": 3.4297805332133664,
+ "tokens_seen": 2217279488
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026525977933801405,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4297723201667614,
+ "tokens_seen": 2217345024
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026524373119358075,
+ "loss": 1.2103,
+ "theoretical_loss": 3.429764107430865,
+ "tokens_seen": 2217410560
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026522768304914745,
+ "loss": 1.2586,
+ "theoretical_loss": 3.4297558950056555,
+ "tokens_seen": 2217476096
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026521163490471415,
+ "loss": 1.2126,
+ "theoretical_loss": 3.429747682891113,
+ "tokens_seen": 2217541632
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026519558676028085,
+ "loss": 1.2299,
+ "theoretical_loss": 3.429739471087215,
+ "tokens_seen": 2217607168
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026517953861584755,
+ "loss": 1.1767,
+ "theoretical_loss": 3.429731259593942,
+ "tokens_seen": 2217672704
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026516349047141425,
+ "loss": 1.1833,
+ "theoretical_loss": 3.4297230484112724,
+ "tokens_seen": 2217738240
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026514744232698095,
+ "loss": 1.3009,
+ "theoretical_loss": 3.4297148375391853,
+ "tokens_seen": 2217803776
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026513139418254765,
+ "loss": 1.3177,
+ "theoretical_loss": 3.4297066269776604,
+ "tokens_seen": 2217869312
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026511534603811435,
+ "loss": 1.2841,
+ "theoretical_loss": 3.4296984167266755,
+ "tokens_seen": 2217934848
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026509929789368105,
+ "loss": 1.237,
+ "theoretical_loss": 3.4296902067862107,
+ "tokens_seen": 2218000384
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026508324974924775,
+ "loss": 1.1843,
+ "theoretical_loss": 3.4296819971562447,
+ "tokens_seen": 2218065920
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026506720160481446,
+ "loss": 1.2867,
+ "theoretical_loss": 3.429673787836757,
+ "tokens_seen": 2218131456
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026505115346038116,
+ "loss": 1.249,
+ "theoretical_loss": 3.429665578827726,
+ "tokens_seen": 2218196992
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026503510531594786,
+ "loss": 1.2335,
+ "theoretical_loss": 3.4296573701291315,
+ "tokens_seen": 2218262528
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026501905717151456,
+ "loss": 1.2415,
+ "theoretical_loss": 3.429649161740952,
+ "tokens_seen": 2218328064
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 742689,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3651707172393799,
+ "objective/train/theoretical_loss": 3.429640953663167,
+ "objective/train/tokens_used": 2238853600,
+ "theoretical_loss": 3.429640953663167,
+ "tokens_seen": 2218393600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026500300902708126,
+ "loss": 1.3749,
+ "theoretical_loss": 3.429640953663167,
+ "tokens_seen": 2218393600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026498696088264796,
+ "loss": 1.1721,
+ "theoretical_loss": 3.4296327458957556,
+ "tokens_seen": 2218459136
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026497091273821466,
+ "loss": 1.2124,
+ "theoretical_loss": 3.4296245384386963,
+ "tokens_seen": 2218524672
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026495486459378136,
+ "loss": 1.1852,
+ "theoretical_loss": 3.429616331291969,
+ "tokens_seen": 2218590208
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026493881644934806,
+ "loss": 1.247,
+ "theoretical_loss": 3.4296081244555525,
+ "tokens_seen": 2218655744
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026492276830491476,
+ "loss": 1.28,
+ "theoretical_loss": 3.4295999179294254,
+ "tokens_seen": 2218721280
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026490672016048146,
+ "loss": 1.2093,
+ "theoretical_loss": 3.4295917117135675,
+ "tokens_seen": 2218786816
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026489067201604816,
+ "loss": 1.2738,
+ "theoretical_loss": 3.429583505807958,
+ "tokens_seen": 2218852352
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002648746238716148,
+ "loss": 1.3016,
+ "theoretical_loss": 3.4295753002125746,
+ "tokens_seen": 2218917888
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026485857572718156,
+ "loss": 1.2112,
+ "theoretical_loss": 3.4295670949273984,
+ "tokens_seen": 2218983424
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026484252758274826,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4295588899524074,
+ "tokens_seen": 2219048960
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026482647943831496,
+ "loss": 1.2925,
+ "theoretical_loss": 3.429550685287581,
+ "tokens_seen": 2219114496
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026481043129388167,
+ "loss": 1.2579,
+ "theoretical_loss": 3.4295424809328976,
+ "tokens_seen": 2219180032
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026479438314944837,
+ "loss": 1.1911,
+ "theoretical_loss": 3.4295342768883375,
+ "tokens_seen": 2219245568
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026477833500501507,
+ "loss": 1.202,
+ "theoretical_loss": 3.429526073153879,
+ "tokens_seen": 2219311104
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026476228686058177,
+ "loss": 1.2689,
+ "theoretical_loss": 3.4295178697295015,
+ "tokens_seen": 2219376640
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026474623871614847,
+ "loss": 1.2285,
+ "theoretical_loss": 3.429509666615184,
+ "tokens_seen": 2219442176
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026473019057171517,
+ "loss": 1.292,
+ "theoretical_loss": 3.429501463810906,
+ "tokens_seen": 2219507712
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026471414242728187,
+ "loss": 1.2811,
+ "theoretical_loss": 3.429493261316646,
+ "tokens_seen": 2219573248
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026469809428284857,
+ "loss": 1.3281,
+ "theoretical_loss": 3.4294850591323836,
+ "tokens_seen": 2219638784
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026468204613841527,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4294768572580976,
+ "tokens_seen": 2219704320
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026466599799398197,
+ "loss": 1.2102,
+ "theoretical_loss": 3.4294686556937677,
+ "tokens_seen": 2219769856
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026464994984954867,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4294604544393725,
+ "tokens_seen": 2219835392
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026463390170511537,
+ "loss": 1.2258,
+ "theoretical_loss": 3.4294522534948912,
+ "tokens_seen": 2219900928
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002646178535606821,
+ "loss": 1.2509,
+ "theoretical_loss": 3.4294440528603034,
+ "tokens_seen": 2219966464
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002646018054162488,
+ "loss": 1.2552,
+ "theoretical_loss": 3.4294358525355877,
+ "tokens_seen": 2220032000
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002645857572718155,
+ "loss": 1.1515,
+ "theoretical_loss": 3.4294276525207232,
+ "tokens_seen": 2220097536
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002645697091273822,
+ "loss": 1.3023,
+ "theoretical_loss": 3.4294194528156896,
+ "tokens_seen": 2220163072
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002645536609829489,
+ "loss": 1.233,
+ "theoretical_loss": 3.4294112534204655,
+ "tokens_seen": 2220228608
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002645376128385156,
+ "loss": 1.1878,
+ "theoretical_loss": 3.4294030543350305,
+ "tokens_seen": 2220294144
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002645215646940823,
+ "loss": 1.1959,
+ "theoretical_loss": 3.429394855559363,
+ "tokens_seen": 2220359680
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000264505516549649,
+ "loss": 1.2426,
+ "theoretical_loss": 3.4293866570934437,
+ "tokens_seen": 2220425216
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644894684052157,
+ "loss": 1.2454,
+ "theoretical_loss": 3.42937845893725,
+ "tokens_seen": 2220490752
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644734202607824,
+ "loss": 1.3891,
+ "theoretical_loss": 3.429370261090762,
+ "tokens_seen": 2220556288
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644573721163491,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4293620635539583,
+ "tokens_seen": 2220621824
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644413239719158,
+ "loss": 1.2472,
+ "theoretical_loss": 3.429353866326819,
+ "tokens_seen": 2220687360
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644252758274825,
+ "loss": 1.3108,
+ "theoretical_loss": 3.4293456694093223,
+ "tokens_seen": 2220752896
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002644092276830492,
+ "loss": 1.2307,
+ "theoretical_loss": 3.429337472801448,
+ "tokens_seen": 2220818432
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002643931795386159,
+ "loss": 1.1358,
+ "theoretical_loss": 3.4293292765031747,
+ "tokens_seen": 2220883968
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002643771313941826,
+ "loss": 1.1489,
+ "theoretical_loss": 3.429321080514482,
+ "tokens_seen": 2220949504
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002643610832497493,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4293128848353494,
+ "tokens_seen": 2221015040
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000264345035105316,
+ "loss": 1.174,
+ "theoretical_loss": 3.429304689465755,
+ "tokens_seen": 2221080576
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002643289869608827,
+ "loss": 1.292,
+ "theoretical_loss": 3.429296494405679,
+ "tokens_seen": 2221146112
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002643129388164494,
+ "loss": 1.2696,
+ "theoretical_loss": 3.4292882996551,
+ "tokens_seen": 2221211648
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002642968906720161,
+ "loss": 1.2637,
+ "theoretical_loss": 3.4292801052139974,
+ "tokens_seen": 2221277184
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002642808425275828,
+ "loss": 1.3076,
+ "theoretical_loss": 3.4292719110823504,
+ "tokens_seen": 2221342720
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002642647943831495,
+ "loss": 1.304,
+ "theoretical_loss": 3.429263717260138,
+ "tokens_seen": 2221408256
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002642487462387162,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4292555237473397,
+ "tokens_seen": 2221473792
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026423269809428283,
+ "loss": 1.2394,
+ "theoretical_loss": 3.4292473305439346,
+ "tokens_seen": 2221539328
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026421664994984953,
+ "loss": 1.2298,
+ "theoretical_loss": 3.4292391376499016,
+ "tokens_seen": 2221604864
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 743980,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3372083902359009,
+ "objective/train/theoretical_loss": 3.4292309450652203,
+ "objective/train/tokens_used": 2242130400,
+ "theoretical_loss": 3.4292309450652203,
+ "tokens_seen": 2221670400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026420060180541624,
+ "loss": 1.3112,
+ "theoretical_loss": 3.4292309450652203,
+ "tokens_seen": 2221670400
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026418455366098294,
+ "loss": 1.1827,
+ "theoretical_loss": 3.4292227527898693,
+ "tokens_seen": 2221735936
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026416850551654964,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4292145608238287,
+ "tokens_seen": 2221801472
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026415245737211634,
+ "loss": 1.3176,
+ "theoretical_loss": 3.429206369167077,
+ "tokens_seen": 2221867008
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026413640922768304,
+ "loss": 1.2815,
+ "theoretical_loss": 3.4291981778195932,
+ "tokens_seen": 2221932544
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026412036108324974,
+ "loss": 1.2655,
+ "theoretical_loss": 3.429189986781357,
+ "tokens_seen": 2221998080
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026410431293881644,
+ "loss": 1.2826,
+ "theoretical_loss": 3.429181796052348,
+ "tokens_seen": 2222063616
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026408826479438314,
+ "loss": 1.346,
+ "theoretical_loss": 3.4291736056325446,
+ "tokens_seen": 2222129152
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026407221664994984,
+ "loss": 1.2676,
+ "theoretical_loss": 3.4291654155219264,
+ "tokens_seen": 2222194688
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026405616850551654,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4291572257204725,
+ "tokens_seen": 2222260224
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026404012036108324,
+ "loss": 1.2655,
+ "theoretical_loss": 3.429149036228162,
+ "tokens_seen": 2222325760
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026402407221664994,
+ "loss": 1.252,
+ "theoretical_loss": 3.4291408470449745,
+ "tokens_seen": 2222391296
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026400802407221664,
+ "loss": 1.2551,
+ "theoretical_loss": 3.4291326581708885,
+ "tokens_seen": 2222456832
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026399197592778334,
+ "loss": 1.1958,
+ "theoretical_loss": 3.429124469605884,
+ "tokens_seen": 2222522368
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026397592778335004,
+ "loss": 1.2822,
+ "theoretical_loss": 3.42911628134994,
+ "tokens_seen": 2222587904
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026395987963891674,
+ "loss": 1.2139,
+ "theoretical_loss": 3.4291080934030354,
+ "tokens_seen": 2222653440
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026394383149448345,
+ "loss": 1.3089,
+ "theoretical_loss": 3.42909990576515,
+ "tokens_seen": 2222718976
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026392778335005015,
+ "loss": 1.2895,
+ "theoretical_loss": 3.429091718436262,
+ "tokens_seen": 2222784512
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026391173520561685,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4290835314163517,
+ "tokens_seen": 2222850048
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026389568706118355,
+ "loss": 1.2129,
+ "theoretical_loss": 3.429075344705398,
+ "tokens_seen": 2222915584
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026387963891675025,
+ "loss": 1.2081,
+ "theoretical_loss": 3.42906715830338,
+ "tokens_seen": 2222981120
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026386359077231695,
+ "loss": 1.1812,
+ "theoretical_loss": 3.429058972210277,
+ "tokens_seen": 2223046656
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026384754262788365,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4290507864260684,
+ "tokens_seen": 2223112192
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026383149448345035,
+ "loss": 1.2361,
+ "theoretical_loss": 3.429042600950733,
+ "tokens_seen": 2223177728
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026381544633901705,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4290344157842503,
+ "tokens_seen": 2223243264
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026379939819458375,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4290262309266,
+ "tokens_seen": 2223308800
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026378335005015045,
+ "loss": 1.2287,
+ "theoretical_loss": 3.4290180463777604,
+ "tokens_seen": 2223374336
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026376730190571715,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4290098621377116,
+ "tokens_seen": 2223439872
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026375125376128385,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4290016782064323,
+ "tokens_seen": 2223505408
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026373520561685055,
+ "loss": 1.253,
+ "theoretical_loss": 3.428993494583902,
+ "tokens_seen": 2223570944
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026371915747241725,
+ "loss": 1.1872,
+ "theoretical_loss": 3.4289853112701,
+ "tokens_seen": 2223636480
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026370310932798395,
+ "loss": 1.2781,
+ "theoretical_loss": 3.4289771282650054,
+ "tokens_seen": 2223702016
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026368706118355066,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4289689455685974,
+ "tokens_seen": 2223767552
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026367101303911736,
+ "loss": 1.2456,
+ "theoretical_loss": 3.4289607631808554,
+ "tokens_seen": 2223833088
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002636549648946841,
+ "loss": 1.266,
+ "theoretical_loss": 3.428952581101759,
+ "tokens_seen": 2223898624
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002636389167502508,
+ "loss": 1.3292,
+ "theoretical_loss": 3.4289443993312867,
+ "tokens_seen": 2223964160
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002636228686058175,
+ "loss": 1.1492,
+ "theoretical_loss": 3.4289362178694183,
+ "tokens_seen": 2224029696
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026360682046138416,
+ "loss": 1.1917,
+ "theoretical_loss": 3.4289280367161328,
+ "tokens_seen": 2224095232
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026359077231695086,
+ "loss": 1.3213,
+ "theoretical_loss": 3.42891985587141,
+ "tokens_seen": 2224160768
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026357472417251756,
+ "loss": 1.1221,
+ "theoretical_loss": 3.428911675335228,
+ "tokens_seen": 2224226304
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026355867602808426,
+ "loss": 1.1716,
+ "theoretical_loss": 3.4289034951075674,
+ "tokens_seen": 2224291840
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026354262788365096,
+ "loss": 1.1981,
+ "theoretical_loss": 3.428895315188407,
+ "tokens_seen": 2224357376
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026352657973921766,
+ "loss": 1.2274,
+ "theoretical_loss": 3.428887135577726,
+ "tokens_seen": 2224422912
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026351053159478436,
+ "loss": 1.291,
+ "theoretical_loss": 3.4288789562755033,
+ "tokens_seen": 2224488448
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026349448345035106,
+ "loss": 1.3646,
+ "theoretical_loss": 3.428870777281719,
+ "tokens_seen": 2224553984
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026347843530591776,
+ "loss": 1.2991,
+ "theoretical_loss": 3.4288625985963517,
+ "tokens_seen": 2224619520
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026346238716148446,
+ "loss": 1.2858,
+ "theoretical_loss": 3.4288544202193805,
+ "tokens_seen": 2224685056
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026344633901705117,
+ "loss": 1.294,
+ "theoretical_loss": 3.4288462421507857,
+ "tokens_seen": 2224750592
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026343029087261787,
+ "loss": 1.2628,
+ "theoretical_loss": 3.428838064390546,
+ "tokens_seen": 2224816128
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026341424272818457,
+ "loss": 1.3204,
+ "theoretical_loss": 3.42882988693864,
+ "tokens_seen": 2224881664
+ },
+ {
+ "epoch": 0.67,
+ "objective/train/docs_used": 744927,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2962076663970947,
+ "objective/train/theoretical_loss": 3.4288217097950486,
+ "objective/train/tokens_used": 2245407200,
+ "theoretical_loss": 3.4288217097950486,
+ "tokens_seen": 2224947200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026339819458375127,
+ "loss": 1.2519,
+ "theoretical_loss": 3.4288217097950486,
+ "tokens_seen": 2224947200
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026338214643931797,
+ "loss": 1.2846,
+ "theoretical_loss": 3.4288135329597496,
+ "tokens_seen": 2225012736
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026336609829488467,
+ "loss": 1.2197,
+ "theoretical_loss": 3.428805356432723,
+ "tokens_seen": 2225078272
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026335005015045137,
+ "loss": 1.3023,
+ "theoretical_loss": 3.428797180213948,
+ "tokens_seen": 2225143808
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026333400200601807,
+ "loss": 1.2095,
+ "theoretical_loss": 3.428789004303404,
+ "tokens_seen": 2225209344
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026331795386158477,
+ "loss": 1.1658,
+ "theoretical_loss": 3.42878082870107,
+ "tokens_seen": 2225274880
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026330190571715147,
+ "loss": 1.2342,
+ "theoretical_loss": 3.4287726534069254,
+ "tokens_seen": 2225340416
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026328585757271817,
+ "loss": 1.2725,
+ "theoretical_loss": 3.4287644784209497,
+ "tokens_seen": 2225405952
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026326980942828487,
+ "loss": 1.208,
+ "theoretical_loss": 3.428756303743122,
+ "tokens_seen": 2225471488
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026325376128385157,
+ "loss": 1.2908,
+ "theoretical_loss": 3.428748129373422,
+ "tokens_seen": 2225537024
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002632377131394183,
+ "loss": 1.2185,
+ "theoretical_loss": 3.4287399553118285,
+ "tokens_seen": 2225602560
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000263221664994985,
+ "loss": 1.2851,
+ "theoretical_loss": 3.428731781558321,
+ "tokens_seen": 2225668096
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002632056168505517,
+ "loss": 1.2984,
+ "theoretical_loss": 3.428723608112879,
+ "tokens_seen": 2225733632
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631895687061184,
+ "loss": 1.2627,
+ "theoretical_loss": 3.428715434975482,
+ "tokens_seen": 2225799168
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631735205616851,
+ "loss": 1.2726,
+ "theoretical_loss": 3.428707262146108,
+ "tokens_seen": 2225864704
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631574724172518,
+ "loss": 1.2096,
+ "theoretical_loss": 3.428699089624738,
+ "tokens_seen": 2225930240
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631414242728185,
+ "loss": 1.174,
+ "theoretical_loss": 3.4286909174113505,
+ "tokens_seen": 2225995776
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631253761283852,
+ "loss": 1.2857,
+ "theoretical_loss": 3.428682745505925,
+ "tokens_seen": 2226061312
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002631093279839519,
+ "loss": 1.3091,
+ "theoretical_loss": 3.4286745739084408,
+ "tokens_seen": 2226126848
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002630932798395186,
+ "loss": 1.1975,
+ "theoretical_loss": 3.428666402618877,
+ "tokens_seen": 2226192384
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002630772316950853,
+ "loss": 1.2904,
+ "theoretical_loss": 3.4286582316372134,
+ "tokens_seen": 2226257920
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000263061183550652,
+ "loss": 1.2741,
+ "theoretical_loss": 3.428650060963429,
+ "tokens_seen": 2226323456
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002630451354062187,
+ "loss": 1.2023,
+ "theoretical_loss": 3.428641890597503,
+ "tokens_seen": 2226388992
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002630290872617854,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4286337205394153,
+ "tokens_seen": 2226454528
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002630130391173521,
+ "loss": 1.2535,
+ "theoretical_loss": 3.428625550789145,
+ "tokens_seen": 2226520064
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629969909729188,
+ "loss": 1.2165,
+ "theoretical_loss": 3.428617381346671,
+ "tokens_seen": 2226585600
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629809428284855,
+ "loss": 1.2885,
+ "theoretical_loss": 3.4286092122119736,
+ "tokens_seen": 2226651136
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629648946840522,
+ "loss": 1.3736,
+ "theoretical_loss": 3.428601043385031,
+ "tokens_seen": 2226716672
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629488465396189,
+ "loss": 1.3208,
+ "theoretical_loss": 3.4285928748658234,
+ "tokens_seen": 2226782208
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629327983951856,
+ "loss": 1.2053,
+ "theoretical_loss": 3.4285847066543296,
+ "tokens_seen": 2226847744
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002629167502507523,
+ "loss": 1.2479,
+ "theoretical_loss": 3.428576538750529,
+ "tokens_seen": 2226913280
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.000262900702106319,
+ "loss": 1.2131,
+ "theoretical_loss": 3.4285683711544013,
+ "tokens_seen": 2226978816
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002628846539618857,
+ "loss": 1.1953,
+ "theoretical_loss": 3.428560203865926,
+ "tokens_seen": 2227044352
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002628686058174524,
+ "loss": 1.1443,
+ "theoretical_loss": 3.428552036885082,
+ "tokens_seen": 2227109888
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.0002628525576730191,
+ "loss": 1.3213,
+ "theoretical_loss": 3.4285438702118487,
+ "tokens_seen": 2227175424
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026283650952858573,
+ "loss": 1.2421,
+ "theoretical_loss": 3.4285357038462054,
+ "tokens_seen": 2227240960
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026282046138415244,
+ "loss": 1.2776,
+ "theoretical_loss": 3.428527537788132,
+ "tokens_seen": 2227306496
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026280441323971914,
+ "loss": 1.2285,
+ "theoretical_loss": 3.4285193720376075,
+ "tokens_seen": 2227372032
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 0.00026278836509528584,
+ "loss": 1.2301,
+ "theoretical_loss": 3.428511206594611,
+ "tokens_seen": 2227437568
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026277231695085254,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4285030414591224,
+ "tokens_seen": 2227503104
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026275626880641924,
+ "loss": 1.2532,
+ "theoretical_loss": 3.428494876631121,
+ "tokens_seen": 2227568640
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026274022066198594,
+ "loss": 1.1369,
+ "theoretical_loss": 3.428486712110586,
+ "tokens_seen": 2227634176
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026272417251755264,
+ "loss": 1.2456,
+ "theoretical_loss": 3.4284785478974964,
+ "tokens_seen": 2227699712
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026270812437311934,
+ "loss": 1.3122,
+ "theoretical_loss": 3.428470383991832,
+ "tokens_seen": 2227765248
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026269207622868604,
+ "loss": 1.284,
+ "theoretical_loss": 3.4284622203935724,
+ "tokens_seen": 2227830784
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026267602808425274,
+ "loss": 1.2617,
+ "theoretical_loss": 3.4284540571026967,
+ "tokens_seen": 2227896320
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026265997993981944,
+ "loss": 1.2117,
+ "theoretical_loss": 3.428445894119184,
+ "tokens_seen": 2227961856
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026264393179538614,
+ "loss": 1.1856,
+ "theoretical_loss": 3.4284377314430143,
+ "tokens_seen": 2228027392
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026262788365095284,
+ "loss": 1.2032,
+ "theoretical_loss": 3.428429569074167,
+ "tokens_seen": 2228092928
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026261183550651954,
+ "loss": 1.2223,
+ "theoretical_loss": 3.42842140701262,
+ "tokens_seen": 2228158464
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 746243,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2754054069519043,
+ "objective/train/theoretical_loss": 3.428413245258355,
+ "objective/train/tokens_used": 2248684000,
+ "theoretical_loss": 3.428413245258355,
+ "tokens_seen": 2228224000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026259578736208624,
+ "loss": 1.2565,
+ "theoretical_loss": 3.428413245258355,
+ "tokens_seen": 2228224000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026257973921765295,
+ "loss": 1.2181,
+ "theoretical_loss": 3.42840508381135,
+ "tokens_seen": 2228289536
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026256369107321965,
+ "loss": 1.2252,
+ "theoretical_loss": 3.4283969226715842,
+ "tokens_seen": 2228355072
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026254764292878635,
+ "loss": 1.2196,
+ "theoretical_loss": 3.428388761839038,
+ "tokens_seen": 2228420608
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026253159478435305,
+ "loss": 1.1996,
+ "theoretical_loss": 3.42838060131369,
+ "tokens_seen": 2228486144
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026251554663991975,
+ "loss": 1.2926,
+ "theoretical_loss": 3.42837244109552,
+ "tokens_seen": 2228551680
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026249949849548645,
+ "loss": 1.1935,
+ "theoretical_loss": 3.428364281184507,
+ "tokens_seen": 2228617216
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026248345035105315,
+ "loss": 1.2346,
+ "theoretical_loss": 3.428356121580631,
+ "tokens_seen": 2228682752
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026246740220661985,
+ "loss": 1.1546,
+ "theoretical_loss": 3.428347962283871,
+ "tokens_seen": 2228748288
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002624513540621866,
+ "loss": 1.2163,
+ "theoretical_loss": 3.4283398032942065,
+ "tokens_seen": 2228813824
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002624353059177533,
+ "loss": 1.3449,
+ "theoretical_loss": 3.4283316446116165,
+ "tokens_seen": 2228879360
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026241925777332,
+ "loss": 1.1917,
+ "theoretical_loss": 3.4283234862360814,
+ "tokens_seen": 2228944896
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002624032096288867,
+ "loss": 1.132,
+ "theoretical_loss": 3.42831532816758,
+ "tokens_seen": 2229010432
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623871614844534,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4283071704060912,
+ "tokens_seen": 2229075968
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623711133400201,
+ "loss": 1.2752,
+ "theoretical_loss": 3.428299012951595,
+ "tokens_seen": 2229141504
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623550651955868,
+ "loss": 1.3274,
+ "theoretical_loss": 3.4282908558040712,
+ "tokens_seen": 2229207040
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623390170511535,
+ "loss": 1.2011,
+ "theoretical_loss": 3.4282826989634985,
+ "tokens_seen": 2229272576
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623229689067202,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4282745424298566,
+ "tokens_seen": 2229338112
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002623069207622869,
+ "loss": 1.2052,
+ "theoretical_loss": 3.4282663862031253,
+ "tokens_seen": 2229403648
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002622908726178536,
+ "loss": 1.264,
+ "theoretical_loss": 3.4282582302832836,
+ "tokens_seen": 2229469184
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002622748244734203,
+ "loss": 1.2322,
+ "theoretical_loss": 3.428250074670311,
+ "tokens_seen": 2229534720
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000262258776328987,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4282419193641873,
+ "tokens_seen": 2229600256
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002622427281845537,
+ "loss": 1.1535,
+ "theoretical_loss": 3.428233764364891,
+ "tokens_seen": 2229665792
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002622266800401204,
+ "loss": 1.2632,
+ "theoretical_loss": 3.4282256096724026,
+ "tokens_seen": 2229731328
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026221063189568706,
+ "loss": 1.1906,
+ "theoretical_loss": 3.428217455286701,
+ "tokens_seen": 2229796864
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026219458375125376,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4282093012077652,
+ "tokens_seen": 2229862400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026217853560682046,
+ "loss": 1.1806,
+ "theoretical_loss": 3.428201147435576,
+ "tokens_seen": 2229927936
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026216248746238716,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4281929939701112,
+ "tokens_seen": 2229993472
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026214643931795386,
+ "loss": 1.2116,
+ "theoretical_loss": 3.4281848408113516,
+ "tokens_seen": 2230059008
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026213039117352056,
+ "loss": 1.2791,
+ "theoretical_loss": 3.428176687959276,
+ "tokens_seen": 2230124544
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026211434302908726,
+ "loss": 1.1573,
+ "theoretical_loss": 3.428168535413864,
+ "tokens_seen": 2230190080
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026209829488465396,
+ "loss": 1.2152,
+ "theoretical_loss": 3.428160383175095,
+ "tokens_seen": 2230255616
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026208224674022066,
+ "loss": 1.2047,
+ "theoretical_loss": 3.428152231242948,
+ "tokens_seen": 2230321152
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026206619859578737,
+ "loss": 1.3543,
+ "theoretical_loss": 3.428144079617404,
+ "tokens_seen": 2230386688
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026205015045135407,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4281359282984405,
+ "tokens_seen": 2230452224
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026203410230692077,
+ "loss": 1.2624,
+ "theoretical_loss": 3.428127777286038,
+ "tokens_seen": 2230517760
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026201805416248747,
+ "loss": 1.3008,
+ "theoretical_loss": 3.428119626580176,
+ "tokens_seen": 2230583296
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026200200601805417,
+ "loss": 1.2781,
+ "theoretical_loss": 3.428111476180834,
+ "tokens_seen": 2230648832
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026198595787362087,
+ "loss": 1.33,
+ "theoretical_loss": 3.4281033260879905,
+ "tokens_seen": 2230714368
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026196990972918757,
+ "loss": 1.3113,
+ "theoretical_loss": 3.428095176301626,
+ "tokens_seen": 2230779904
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026195386158475427,
+ "loss": 1.3817,
+ "theoretical_loss": 3.42808702682172,
+ "tokens_seen": 2230845440
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026193781344032097,
+ "loss": 1.271,
+ "theoretical_loss": 3.4280788776482516,
+ "tokens_seen": 2230910976
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026192176529588767,
+ "loss": 1.2731,
+ "theoretical_loss": 3.4280707287812002,
+ "tokens_seen": 2230976512
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026190571715145437,
+ "loss": 1.304,
+ "theoretical_loss": 3.4280625802205456,
+ "tokens_seen": 2231042048
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026188966900702107,
+ "loss": 1.2808,
+ "theoretical_loss": 3.4280544319662667,
+ "tokens_seen": 2231107584
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002618736208625878,
+ "loss": 1.2304,
+ "theoretical_loss": 3.428046284018344,
+ "tokens_seen": 2231173120
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002618575727181545,
+ "loss": 1.2817,
+ "theoretical_loss": 3.428038136376756,
+ "tokens_seen": 2231238656
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002618415245737212,
+ "loss": 1.2842,
+ "theoretical_loss": 3.4280299890414825,
+ "tokens_seen": 2231304192
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002618254764292879,
+ "loss": 1.29,
+ "theoretical_loss": 3.428021842012503,
+ "tokens_seen": 2231369728
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002618094282848546,
+ "loss": 1.1693,
+ "theoretical_loss": 3.4280136952897973,
+ "tokens_seen": 2231435264
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 747198,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.172689437866211,
+ "objective/train/theoretical_loss": 3.4280055488733447,
+ "objective/train/tokens_used": 2251960800,
+ "theoretical_loss": 3.4280055488733447,
+ "tokens_seen": 2231500800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002617933801404213,
+ "loss": 1.1688,
+ "theoretical_loss": 3.4280055488733447,
+ "tokens_seen": 2231500800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000261777331995988,
+ "loss": 1.2212,
+ "theoretical_loss": 3.427997402763124,
+ "tokens_seen": 2231566336
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002617612838515547,
+ "loss": 1.2918,
+ "theoretical_loss": 3.427989256959116,
+ "tokens_seen": 2231631872
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002617452357071214,
+ "loss": 1.2737,
+ "theoretical_loss": 3.427981111461299,
+ "tokens_seen": 2231697408
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002617291875626881,
+ "loss": 1.2546,
+ "theoretical_loss": 3.427972966269653,
+ "tokens_seen": 2231762944
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002617131394182548,
+ "loss": 1.2056,
+ "theoretical_loss": 3.4279648213841583,
+ "tokens_seen": 2231828480
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616970912738215,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4279566768047927,
+ "tokens_seen": 2231894016
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616810431293882,
+ "loss": 1.2207,
+ "theoretical_loss": 3.427948532531537,
+ "tokens_seen": 2231959552
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616649949849549,
+ "loss": 1.1649,
+ "theoretical_loss": 3.4279403885643704,
+ "tokens_seen": 2232025088
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616489468405216,
+ "loss": 1.1676,
+ "theoretical_loss": 3.427932244903272,
+ "tokens_seen": 2232090624
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616328986960883,
+ "loss": 1.2509,
+ "theoretical_loss": 3.427924101548222,
+ "tokens_seen": 2232156160
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000261616850551655,
+ "loss": 1.1529,
+ "theoretical_loss": 3.427915958499199,
+ "tokens_seen": 2232221696
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002616008024072217,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4279078157561838,
+ "tokens_seen": 2232287232
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615847542627884,
+ "loss": 1.133,
+ "theoretical_loss": 3.4278996733191547,
+ "tokens_seen": 2232352768
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615687061183551,
+ "loss": 1.164,
+ "theoretical_loss": 3.427891531188092,
+ "tokens_seen": 2232418304
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615526579739218,
+ "loss": 1.2439,
+ "theoretical_loss": 3.4278833893629748,
+ "tokens_seen": 2232483840
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615366098294885,
+ "loss": 1.2517,
+ "theoretical_loss": 3.4278752478437826,
+ "tokens_seen": 2232549376
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615205616850552,
+ "loss": 1.2513,
+ "theoretical_loss": 3.4278671066304955,
+ "tokens_seen": 2232614912
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002615045135406219,
+ "loss": 1.1903,
+ "theoretical_loss": 3.4278589657230922,
+ "tokens_seen": 2232680448
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002614884653961886,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4278508251215527,
+ "tokens_seen": 2232745984
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002614724172517553,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4278426848258565,
+ "tokens_seen": 2232811520
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000261456369107322,
+ "loss": 1.3,
+ "theoretical_loss": 3.4278345448359833,
+ "tokens_seen": 2232877056
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026144032096288864,
+ "loss": 1.1505,
+ "theoretical_loss": 3.427826405151912,
+ "tokens_seen": 2232942592
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026142427281845534,
+ "loss": 1.1819,
+ "theoretical_loss": 3.427818265773623,
+ "tokens_seen": 2233008128
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026140822467402204,
+ "loss": 1.1705,
+ "theoretical_loss": 3.427810126701095,
+ "tokens_seen": 2233073664
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026139217652958874,
+ "loss": 1.246,
+ "theoretical_loss": 3.4278019879343087,
+ "tokens_seen": 2233139200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026137612838515544,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4277938494732423,
+ "tokens_seen": 2233204736
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026136008024072214,
+ "loss": 1.2396,
+ "theoretical_loss": 3.427785711317876,
+ "tokens_seen": 2233270272
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026134403209628884,
+ "loss": 1.3277,
+ "theoretical_loss": 3.4277775734681892,
+ "tokens_seen": 2233335808
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026132798395185554,
+ "loss": 1.252,
+ "theoretical_loss": 3.4277694359241617,
+ "tokens_seen": 2233401344
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026131193580742224,
+ "loss": 1.2186,
+ "theoretical_loss": 3.427761298685773,
+ "tokens_seen": 2233466880
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026129588766298894,
+ "loss": 1.3087,
+ "theoretical_loss": 3.427753161753002,
+ "tokens_seen": 2233532416
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026127983951855564,
+ "loss": 1.2315,
+ "theoretical_loss": 3.4277450251258292,
+ "tokens_seen": 2233597952
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026126379137412234,
+ "loss": 1.2606,
+ "theoretical_loss": 3.427736888804234,
+ "tokens_seen": 2233663488
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002612477432296891,
+ "loss": 1.3204,
+ "theoretical_loss": 3.4277287527881954,
+ "tokens_seen": 2233729024
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002612316950852558,
+ "loss": 1.2508,
+ "theoretical_loss": 3.4277206170776933,
+ "tokens_seen": 2233794560
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002612156469408225,
+ "loss": 1.2384,
+ "theoretical_loss": 3.427712481672707,
+ "tokens_seen": 2233860096
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611995987963892,
+ "loss": 1.2048,
+ "theoretical_loss": 3.4277043465732167,
+ "tokens_seen": 2233925632
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611835506519559,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4276962117792014,
+ "tokens_seen": 2233991168
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611675025075226,
+ "loss": 1.23,
+ "theoretical_loss": 3.427688077290641,
+ "tokens_seen": 2234056704
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611514543630893,
+ "loss": 1.3504,
+ "theoretical_loss": 3.4276799431075147,
+ "tokens_seen": 2234122240
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000261135406218656,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4276718092298024,
+ "tokens_seen": 2234187776
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611193580742227,
+ "loss": 1.1954,
+ "theoretical_loss": 3.427663675657483,
+ "tokens_seen": 2234253312
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002611033099297894,
+ "loss": 1.278,
+ "theoretical_loss": 3.4276555423905375,
+ "tokens_seen": 2234318848
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610872617853561,
+ "loss": 1.2021,
+ "theoretical_loss": 3.427647409428944,
+ "tokens_seen": 2234384384
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610712136409228,
+ "loss": 1.2345,
+ "theoretical_loss": 3.4276392767726827,
+ "tokens_seen": 2234449920
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610551654964895,
+ "loss": 1.2396,
+ "theoretical_loss": 3.427631144421733,
+ "tokens_seen": 2234515456
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610391173520562,
+ "loss": 1.1555,
+ "theoretical_loss": 3.4276230123760754,
+ "tokens_seen": 2234580992
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610230692076229,
+ "loss": 1.2077,
+ "theoretical_loss": 3.427614880635688,
+ "tokens_seen": 2234646528
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002610070210631896,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4276067492005518,
+ "tokens_seen": 2234712064
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 748461,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2609285116195679,
+ "objective/train/theoretical_loss": 3.427598618070645,
+ "objective/train/tokens_used": 2255237600,
+ "theoretical_loss": 3.427598618070645,
+ "tokens_seen": 2234777600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002609909729187563,
+ "loss": 1.2526,
+ "theoretical_loss": 3.427598618070645,
+ "tokens_seen": 2234777600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000260974924774323,
+ "loss": 1.2286,
+ "theoretical_loss": 3.4275904872459484,
+ "tokens_seen": 2234843136
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002609588766298897,
+ "loss": 1.1968,
+ "theoretical_loss": 3.427582356726441,
+ "tokens_seen": 2234908672
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002609428284854564,
+ "loss": 1.2968,
+ "theoretical_loss": 3.4275742265121023,
+ "tokens_seen": 2234974208
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002609267803410231,
+ "loss": 1.2207,
+ "theoretical_loss": 3.4275660966029124,
+ "tokens_seen": 2235039744
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002609107321965898,
+ "loss": 1.2829,
+ "theoretical_loss": 3.42755796699885,
+ "tokens_seen": 2235105280
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002608946840521565,
+ "loss": 1.2631,
+ "theoretical_loss": 3.427549837699896,
+ "tokens_seen": 2235170816
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002608786359077232,
+ "loss": 1.2794,
+ "theoretical_loss": 3.427541708706029,
+ "tokens_seen": 2235236352
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002608625877632899,
+ "loss": 1.2915,
+ "theoretical_loss": 3.427533580017229,
+ "tokens_seen": 2235301888
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002608465396188566,
+ "loss": 1.1816,
+ "theoretical_loss": 3.427525451633475,
+ "tokens_seen": 2235367424
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002608304914744233,
+ "loss": 1.2738,
+ "theoretical_loss": 3.427517323554748,
+ "tokens_seen": 2235432960
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026081444332999,
+ "loss": 1.2457,
+ "theoretical_loss": 3.427509195781026,
+ "tokens_seen": 2235498496
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026079839518555666,
+ "loss": 1.1903,
+ "theoretical_loss": 3.42750106831229,
+ "tokens_seen": 2235564032
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026078234704112336,
+ "loss": 1.2621,
+ "theoretical_loss": 3.427492941148518,
+ "tokens_seen": 2235629568
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026076629889669006,
+ "loss": 1.3453,
+ "theoretical_loss": 3.4274848142896914,
+ "tokens_seen": 2235695104
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026075025075225676,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4274766877357887,
+ "tokens_seen": 2235760640
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026073420260782346,
+ "loss": 1.2067,
+ "theoretical_loss": 3.42746856148679,
+ "tokens_seen": 2235826176
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026071815446339016,
+ "loss": 1.2309,
+ "theoretical_loss": 3.4274604355426748,
+ "tokens_seen": 2235891712
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026070210631895687,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4274523099034226,
+ "tokens_seen": 2235957248
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026068605817452357,
+ "loss": 1.2818,
+ "theoretical_loss": 3.427444184569013,
+ "tokens_seen": 2236022784
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026067001003009027,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4274360595394255,
+ "tokens_seen": 2236088320
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026065396188565697,
+ "loss": 1.2914,
+ "theoretical_loss": 3.42742793481464,
+ "tokens_seen": 2236153856
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026063791374122367,
+ "loss": 1.1957,
+ "theoretical_loss": 3.4274198103946367,
+ "tokens_seen": 2236219392
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026062186559679037,
+ "loss": 1.3751,
+ "theoretical_loss": 3.427411686279394,
+ "tokens_seen": 2236284928
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026060581745235707,
+ "loss": 1.4579,
+ "theoretical_loss": 3.4274035624688923,
+ "tokens_seen": 2236350464
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026058976930792377,
+ "loss": 1.3407,
+ "theoretical_loss": 3.4273954389631114,
+ "tokens_seen": 2236416000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026057372116349047,
+ "loss": 1.2947,
+ "theoretical_loss": 3.4273873157620303,
+ "tokens_seen": 2236481536
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026055767301905717,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4273791928656294,
+ "tokens_seen": 2236547072
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026054162487462387,
+ "loss": 1.2538,
+ "theoretical_loss": 3.4273710702738875,
+ "tokens_seen": 2236612608
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026052557673019057,
+ "loss": 1.201,
+ "theoretical_loss": 3.4273629479867846,
+ "tokens_seen": 2236678144
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00026050952858575727,
+ "loss": 1.2066,
+ "theoretical_loss": 3.4273548260043007,
+ "tokens_seen": 2236743680
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000260493480441324,
+ "loss": 1.2592,
+ "theoretical_loss": 3.427346704326415,
+ "tokens_seen": 2236809216
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002604774322968907,
+ "loss": 1.1896,
+ "theoretical_loss": 3.427338582953108,
+ "tokens_seen": 2236874752
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002604613841524574,
+ "loss": 1.372,
+ "theoretical_loss": 3.427330461884358,
+ "tokens_seen": 2236940288
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002604453360080241,
+ "loss": 1.2002,
+ "theoretical_loss": 3.427322341120145,
+ "tokens_seen": 2237005824
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002604292878635908,
+ "loss": 1.297,
+ "theoretical_loss": 3.42731422066045,
+ "tokens_seen": 2237071360
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002604132397191575,
+ "loss": 1.2618,
+ "theoretical_loss": 3.427306100505251,
+ "tokens_seen": 2237136896
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603971915747242,
+ "loss": 1.2693,
+ "theoretical_loss": 3.4272979806545285,
+ "tokens_seen": 2237202432
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603811434302909,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4272898611082616,
+ "tokens_seen": 2237267968
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603650952858576,
+ "loss": 1.3611,
+ "theoretical_loss": 3.4272817418664308,
+ "tokens_seen": 2237333504
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603490471414243,
+ "loss": 1.2429,
+ "theoretical_loss": 3.4272736229290155,
+ "tokens_seen": 2237399040
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000260332998996991,
+ "loss": 1.2929,
+ "theoretical_loss": 3.4272655042959945,
+ "tokens_seen": 2237464576
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603169508525577,
+ "loss": 1.2511,
+ "theoretical_loss": 3.4272573859673487,
+ "tokens_seen": 2237530112
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002603009027081244,
+ "loss": 1.2249,
+ "theoretical_loss": 3.4272492679430573,
+ "tokens_seen": 2237595648
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602848545636911,
+ "loss": 1.2362,
+ "theoretical_loss": 3.4272411502230997,
+ "tokens_seen": 2237661184
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602688064192578,
+ "loss": 1.3199,
+ "theoretical_loss": 3.4272330328074556,
+ "tokens_seen": 2237726720
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602527582748245,
+ "loss": 1.1886,
+ "theoretical_loss": 3.4272249156961054,
+ "tokens_seen": 2237792256
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602367101303912,
+ "loss": 1.2064,
+ "theoretical_loss": 3.427216798889028,
+ "tokens_seen": 2237857792
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602206619859579,
+ "loss": 1.3207,
+ "theoretical_loss": 3.427208682386203,
+ "tokens_seen": 2237923328
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002602046138415246,
+ "loss": 1.1761,
+ "theoretical_loss": 3.427200566187611,
+ "tokens_seen": 2237988864
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 749372,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2783803939819336,
+ "objective/train/theoretical_loss": 3.427192450293231,
+ "objective/train/tokens_used": 2258514400,
+ "theoretical_loss": 3.427192450293231,
+ "tokens_seen": 2238054400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002601885656970913,
+ "loss": 1.3294,
+ "theoretical_loss": 3.427192450293231,
+ "tokens_seen": 2238054400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000260172517552658,
+ "loss": 1.3497,
+ "theoretical_loss": 3.4271843347030426,
+ "tokens_seen": 2238119936
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002601564694082247,
+ "loss": 1.2225,
+ "theoretical_loss": 3.4271762194170257,
+ "tokens_seen": 2238185472
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002601404212637914,
+ "loss": 1.2537,
+ "theoretical_loss": 3.42716810443516,
+ "tokens_seen": 2238251008
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002601243731193581,
+ "loss": 1.275,
+ "theoretical_loss": 3.427159989757425,
+ "tokens_seen": 2238316544
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002601083249749248,
+ "loss": 1.2975,
+ "theoretical_loss": 3.4271518753838013,
+ "tokens_seen": 2238382080
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002600922768304915,
+ "loss": 1.2252,
+ "theoretical_loss": 3.4271437613142672,
+ "tokens_seen": 2238447616
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002600762286860582,
+ "loss": 1.2945,
+ "theoretical_loss": 3.4271356475488033,
+ "tokens_seen": 2238513152
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002600601805416249,
+ "loss": 1.2099,
+ "theoretical_loss": 3.427127534087389,
+ "tokens_seen": 2238578688
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002600441323971916,
+ "loss": 1.2675,
+ "theoretical_loss": 3.427119420930004,
+ "tokens_seen": 2238644224
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002600280842527583,
+ "loss": 1.268,
+ "theoretical_loss": 3.4271113080766282,
+ "tokens_seen": 2238709760
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000260012036108325,
+ "loss": 1.3131,
+ "theoretical_loss": 3.4271031955272413,
+ "tokens_seen": 2238775296
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599959879638917,
+ "loss": 1.2708,
+ "theoretical_loss": 3.4270950832818228,
+ "tokens_seen": 2238840832
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599799398194584,
+ "loss": 1.2699,
+ "theoretical_loss": 3.427086971340353,
+ "tokens_seen": 2238906368
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599638916750251,
+ "loss": 1.2659,
+ "theoretical_loss": 3.4270788597028106,
+ "tokens_seen": 2238971904
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599478435305918,
+ "loss": 1.2474,
+ "theoretical_loss": 3.427070748369176,
+ "tokens_seen": 2239037440
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599317953861585,
+ "loss": 1.26,
+ "theoretical_loss": 3.4270626373394286,
+ "tokens_seen": 2239102976
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002599157472417252,
+ "loss": 1.3254,
+ "theoretical_loss": 3.427054526613549,
+ "tokens_seen": 2239168512
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598996990972919,
+ "loss": 1.2616,
+ "theoretical_loss": 3.4270464161915157,
+ "tokens_seen": 2239234048
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598836509528586,
+ "loss": 1.2158,
+ "theoretical_loss": 3.427038306073309,
+ "tokens_seen": 2239299584
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598676028084253,
+ "loss": 1.237,
+ "theoretical_loss": 3.4270301962589085,
+ "tokens_seen": 2239365120
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000259851554663992,
+ "loss": 1.2999,
+ "theoretical_loss": 3.4270220867482943,
+ "tokens_seen": 2239430656
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598355065195587,
+ "loss": 1.2559,
+ "theoretical_loss": 3.4270139775414457,
+ "tokens_seen": 2239496192
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598194583751254,
+ "loss": 1.2476,
+ "theoretical_loss": 3.4270058686383424,
+ "tokens_seen": 2239561728
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002598034102306921,
+ "loss": 1.2037,
+ "theoretical_loss": 3.4269977600389643,
+ "tokens_seen": 2239627264
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597873620862588,
+ "loss": 1.1522,
+ "theoretical_loss": 3.4269896517432916,
+ "tokens_seen": 2239692800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597713139418255,
+ "loss": 1.2902,
+ "theoretical_loss": 3.426981543751303,
+ "tokens_seen": 2239758336
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597552657973922,
+ "loss": 1.1457,
+ "theoretical_loss": 3.4269734360629793,
+ "tokens_seen": 2239823872
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597392176529589,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4269653286782997,
+ "tokens_seen": 2239889408
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597231695085256,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4269572215972435,
+ "tokens_seen": 2239954944
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002597071213640923,
+ "loss": 1.298,
+ "theoretical_loss": 3.4269491148197915,
+ "tokens_seen": 2240020480
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000259691073219659,
+ "loss": 1.2627,
+ "theoretical_loss": 3.426941008345923,
+ "tokens_seen": 2240086016
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002596750250752257,
+ "loss": 1.2419,
+ "theoretical_loss": 3.426932902175617,
+ "tokens_seen": 2240151552
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002596589769307924,
+ "loss": 1.35,
+ "theoretical_loss": 3.4269247963088545,
+ "tokens_seen": 2240217088
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002596429287863591,
+ "loss": 1.3486,
+ "theoretical_loss": 3.4269166907456143,
+ "tokens_seen": 2240282624
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002596268806419258,
+ "loss": 1.3065,
+ "theoretical_loss": 3.4269085854858767,
+ "tokens_seen": 2240348160
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002596108324974925,
+ "loss": 1.3444,
+ "theoretical_loss": 3.4269004805296213,
+ "tokens_seen": 2240413696
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002595947843530592,
+ "loss": 1.3182,
+ "theoretical_loss": 3.426892375876828,
+ "tokens_seen": 2240479232
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002595787362086259,
+ "loss": 1.307,
+ "theoretical_loss": 3.4268842715274763,
+ "tokens_seen": 2240544768
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002595626880641926,
+ "loss": 1.2944,
+ "theoretical_loss": 3.426876167481546,
+ "tokens_seen": 2240610304
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002595466399197593,
+ "loss": 1.1759,
+ "theoretical_loss": 3.4268680637390165,
+ "tokens_seen": 2240675840
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000259530591775326,
+ "loss": 1.3129,
+ "theoretical_loss": 3.426859960299869,
+ "tokens_seen": 2240741376
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002595145436308927,
+ "loss": 1.2801,
+ "theoretical_loss": 3.426851857164081,
+ "tokens_seen": 2240806912
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594984954864594,
+ "loss": 1.194,
+ "theoretical_loss": 3.4268437543316344,
+ "tokens_seen": 2240872448
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594824473420261,
+ "loss": 1.2469,
+ "theoretical_loss": 3.426835651802508,
+ "tokens_seen": 2240937984
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594663991975928,
+ "loss": 1.306,
+ "theoretical_loss": 3.426827549576682,
+ "tokens_seen": 2241003520
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594503510531595,
+ "loss": 1.13,
+ "theoretical_loss": 3.426819447654135,
+ "tokens_seen": 2241069056
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594343029087262,
+ "loss": 1.2889,
+ "theoretical_loss": 3.426811346034848,
+ "tokens_seen": 2241134592
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002594182547642929,
+ "loss": 1.3935,
+ "theoretical_loss": 3.426803244718801,
+ "tokens_seen": 2241200128
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025940220661985956,
+ "loss": 1.268,
+ "theoretical_loss": 3.4267951437059727,
+ "tokens_seen": 2241265664
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 750607,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2871696949005127,
+ "objective/train/theoretical_loss": 3.4267870429963434,
+ "objective/train/tokens_used": 2261791200,
+ "theoretical_loss": 3.4267870429963434,
+ "tokens_seen": 2241331200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025938615847542626,
+ "loss": 1.252,
+ "theoretical_loss": 3.4267870429963434,
+ "tokens_seen": 2241331200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025937011033099296,
+ "loss": 1.2497,
+ "theoretical_loss": 3.426778942589893,
+ "tokens_seen": 2241396736
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025935406218655966,
+ "loss": 1.2834,
+ "theoretical_loss": 3.426770842486601,
+ "tokens_seen": 2241462272
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025933801404212636,
+ "loss": 1.3216,
+ "theoretical_loss": 3.4267627426864475,
+ "tokens_seen": 2241527808
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025932196589769307,
+ "loss": 1.2537,
+ "theoretical_loss": 3.426754643189412,
+ "tokens_seen": 2241593344
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025930591775325977,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4267465439954745,
+ "tokens_seen": 2241658880
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025928986960882647,
+ "loss": 1.2709,
+ "theoretical_loss": 3.426738445104615,
+ "tokens_seen": 2241724416
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025927382146439317,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4267303465168126,
+ "tokens_seen": 2241789952
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025925777331995987,
+ "loss": 1.3044,
+ "theoretical_loss": 3.426722248232048,
+ "tokens_seen": 2241855488
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025924172517552657,
+ "loss": 1.2432,
+ "theoretical_loss": 3.4267141502503002,
+ "tokens_seen": 2241921024
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025922567703109327,
+ "loss": 1.1919,
+ "theoretical_loss": 3.426706052571549,
+ "tokens_seen": 2241986560
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025920962888665997,
+ "loss": 1.1862,
+ "theoretical_loss": 3.4266979551957757,
+ "tokens_seen": 2242052096
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025919358074222667,
+ "loss": 1.2909,
+ "theoretical_loss": 3.426689858122958,
+ "tokens_seen": 2242117632
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025917753259779337,
+ "loss": 1.1483,
+ "theoretical_loss": 3.426681761353077,
+ "tokens_seen": 2242183168
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025916148445336007,
+ "loss": 1.2412,
+ "theoretical_loss": 3.4266736648861125,
+ "tokens_seen": 2242248704
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025914543630892677,
+ "loss": 1.2736,
+ "theoretical_loss": 3.4266655687220435,
+ "tokens_seen": 2242314240
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002591293881644935,
+ "loss": 1.248,
+ "theoretical_loss": 3.4266574728608505,
+ "tokens_seen": 2242379776
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002591133400200602,
+ "loss": 1.3304,
+ "theoretical_loss": 3.426649377302513,
+ "tokens_seen": 2242445312
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590972918756269,
+ "loss": 1.2917,
+ "theoretical_loss": 3.426641282047011,
+ "tokens_seen": 2242510848
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590812437311936,
+ "loss": 1.2937,
+ "theoretical_loss": 3.4266331870943247,
+ "tokens_seen": 2242576384
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590651955867603,
+ "loss": 1.3004,
+ "theoretical_loss": 3.426625092444433,
+ "tokens_seen": 2242641920
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000259049147442327,
+ "loss": 1.3978,
+ "theoretical_loss": 3.4266169980973165,
+ "tokens_seen": 2242707456
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590330992978937,
+ "loss": 1.2985,
+ "theoretical_loss": 3.4266089040529546,
+ "tokens_seen": 2242772992
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590170511534604,
+ "loss": 1.2557,
+ "theoretical_loss": 3.4266008103113275,
+ "tokens_seen": 2242838528
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002590010030090271,
+ "loss": 1.2487,
+ "theoretical_loss": 3.4265927168724146,
+ "tokens_seen": 2242904064
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589849548645938,
+ "loss": 1.2023,
+ "theoretical_loss": 3.426584623736196,
+ "tokens_seen": 2242969600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589689067201605,
+ "loss": 1.2825,
+ "theoretical_loss": 3.4265765309026515,
+ "tokens_seen": 2243035136
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589528585757272,
+ "loss": 1.2971,
+ "theoretical_loss": 3.4265684383717607,
+ "tokens_seen": 2243100672
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589368104312939,
+ "loss": 1.2662,
+ "theoretical_loss": 3.426560346143504,
+ "tokens_seen": 2243166208
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589207622868606,
+ "loss": 1.2661,
+ "theoretical_loss": 3.426552254217861,
+ "tokens_seen": 2243231744
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002589047141424273,
+ "loss": 1.2408,
+ "theoretical_loss": 3.426544162594811,
+ "tokens_seen": 2243297280
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258888665997994,
+ "loss": 1.205,
+ "theoretical_loss": 3.4265360712743345,
+ "tokens_seen": 2243362816
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002588726178535607,
+ "loss": 1.2912,
+ "theoretical_loss": 3.4265279802564113,
+ "tokens_seen": 2243428352
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025885656970912744,
+ "loss": 1.228,
+ "theoretical_loss": 3.426519889541021,
+ "tokens_seen": 2243493888
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025884052156469414,
+ "loss": 1.2382,
+ "theoretical_loss": 3.4265117991281437,
+ "tokens_seen": 2243559424
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025882447342026084,
+ "loss": 1.2655,
+ "theoretical_loss": 3.4265037090177586,
+ "tokens_seen": 2243624960
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025880842527582754,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4264956192098466,
+ "tokens_seen": 2243690496
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025879237713139424,
+ "loss": 1.2381,
+ "theoretical_loss": 3.426487529704387,
+ "tokens_seen": 2243756032
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025877632898696094,
+ "loss": 1.1888,
+ "theoretical_loss": 3.4264794405013594,
+ "tokens_seen": 2243821568
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002587602808425276,
+ "loss": 1.2146,
+ "theoretical_loss": 3.426471351600744,
+ "tokens_seen": 2243887104
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002587442326980943,
+ "loss": 1.3117,
+ "theoretical_loss": 3.4264632630025202,
+ "tokens_seen": 2243952640
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258728184553661,
+ "loss": 1.2189,
+ "theoretical_loss": 3.4264551747066685,
+ "tokens_seen": 2244018176
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002587121364092277,
+ "loss": 1.234,
+ "theoretical_loss": 3.4264470867131687,
+ "tokens_seen": 2244083712
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586960882647944,
+ "loss": 1.2452,
+ "theoretical_loss": 3.4264389990220003,
+ "tokens_seen": 2244149248
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586800401203611,
+ "loss": 1.2628,
+ "theoretical_loss": 3.4264309116331435,
+ "tokens_seen": 2244214784
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586639919759278,
+ "loss": 1.2893,
+ "theoretical_loss": 3.426422824546578,
+ "tokens_seen": 2244280320
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586479438314945,
+ "loss": 1.2525,
+ "theoretical_loss": 3.4264147377622836,
+ "tokens_seen": 2244345856
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586318956870612,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4264066512802405,
+ "tokens_seen": 2244411392
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002586158475426279,
+ "loss": 1.2643,
+ "theoretical_loss": 3.4263985651004285,
+ "tokens_seen": 2244476928
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585997993981946,
+ "loss": 1.3556,
+ "theoretical_loss": 3.426390479222827,
+ "tokens_seen": 2244542464
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 751484,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1918597221374512,
+ "objective/train/theoretical_loss": 3.4263823936474163,
+ "objective/train/tokens_used": 2265068000,
+ "theoretical_loss": 3.4263823936474163,
+ "tokens_seen": 2244608000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585837512537613,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4263823936474163,
+ "tokens_seen": 2244608000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258567703109328,
+ "loss": 1.3167,
+ "theoretical_loss": 3.426374308374176,
+ "tokens_seen": 2244673536
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585516549648947,
+ "loss": 1.2461,
+ "theoretical_loss": 3.4263662234030865,
+ "tokens_seen": 2244739072
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585356068204614,
+ "loss": 1.119,
+ "theoretical_loss": 3.4263581387341273,
+ "tokens_seen": 2244804608
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585195586760281,
+ "loss": 1.1333,
+ "theoretical_loss": 3.426350054367278,
+ "tokens_seen": 2244870144
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002585035105315948,
+ "loss": 1.354,
+ "theoretical_loss": 3.426341970302519,
+ "tokens_seen": 2244935680
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002584874623871615,
+ "loss": 1.3536,
+ "theoretical_loss": 3.4263338865398305,
+ "tokens_seen": 2245001216
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002584714142427282,
+ "loss": 1.3903,
+ "theoretical_loss": 3.426325803079192,
+ "tokens_seen": 2245066752
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002584553660982949,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4263177199205828,
+ "tokens_seen": 2245132288
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002584393179538616,
+ "loss": 1.2298,
+ "theoretical_loss": 3.4263096370639836,
+ "tokens_seen": 2245197824
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002584232698094283,
+ "loss": 1.3126,
+ "theoretical_loss": 3.426301554509374,
+ "tokens_seen": 2245263360
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258407221664995,
+ "loss": 1.3224,
+ "theoretical_loss": 3.426293472256734,
+ "tokens_seen": 2245328896
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583911735205617,
+ "loss": 1.2536,
+ "theoretical_loss": 3.4262853903060435,
+ "tokens_seen": 2245394432
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583751253761284,
+ "loss": 1.2142,
+ "theoretical_loss": 3.4262773086572818,
+ "tokens_seen": 2245459968
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583590772316951,
+ "loss": 1.1508,
+ "theoretical_loss": 3.42626922731043,
+ "tokens_seen": 2245525504
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583430290872618,
+ "loss": 1.1756,
+ "theoretical_loss": 3.426261146265467,
+ "tokens_seen": 2245591040
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583269809428285,
+ "loss": 1.1798,
+ "theoretical_loss": 3.4262530655223733,
+ "tokens_seen": 2245656576
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002583109327983952,
+ "loss": 1.2357,
+ "theoretical_loss": 3.4262449850811287,
+ "tokens_seen": 2245722112
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002582948846539619,
+ "loss": 1.2656,
+ "theoretical_loss": 3.426236904941713,
+ "tokens_seen": 2245787648
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002582788365095286,
+ "loss": 1.3233,
+ "theoretical_loss": 3.426228825104106,
+ "tokens_seen": 2245853184
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002582627883650953,
+ "loss": 1.3315,
+ "theoretical_loss": 3.426220745568288,
+ "tokens_seen": 2245918720
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258246740220662,
+ "loss": 1.3161,
+ "theoretical_loss": 3.4262126663342385,
+ "tokens_seen": 2245984256
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002582306920762287,
+ "loss": 1.3041,
+ "theoretical_loss": 3.4262045874019376,
+ "tokens_seen": 2246049792
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002582146439317954,
+ "loss": 1.2157,
+ "theoretical_loss": 3.426196508771365,
+ "tokens_seen": 2246115328
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581985957873621,
+ "loss": 1.2135,
+ "theoretical_loss": 3.426188430442501,
+ "tokens_seen": 2246180864
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581825476429288,
+ "loss": 1.2739,
+ "theoretical_loss": 3.426180352415326,
+ "tokens_seen": 2246246400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581664994984955,
+ "loss": 1.2549,
+ "theoretical_loss": 3.4261722746898187,
+ "tokens_seen": 2246311936
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581504513540622,
+ "loss": 1.3188,
+ "theoretical_loss": 3.4261641972659596,
+ "tokens_seen": 2246377472
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581344032096289,
+ "loss": 1.3489,
+ "theoretical_loss": 3.4261561201437285,
+ "tokens_seen": 2246443008
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581183550651956,
+ "loss": 1.2,
+ "theoretical_loss": 3.426148043323106,
+ "tokens_seen": 2246508544
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002581023069207623,
+ "loss": 1.3102,
+ "theoretical_loss": 3.4261399668040715,
+ "tokens_seen": 2246574080
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000258086258776329,
+ "loss": 1.3588,
+ "theoretical_loss": 3.426131890586605,
+ "tokens_seen": 2246639616
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002580702106318957,
+ "loss": 1.3681,
+ "theoretical_loss": 3.4261238146706865,
+ "tokens_seen": 2246705152
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002580541624874624,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4261157390562955,
+ "tokens_seen": 2246770688
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002580381143430291,
+ "loss": 1.2944,
+ "theoretical_loss": 3.4261076637434127,
+ "tokens_seen": 2246836224
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002580220661985958,
+ "loss": 1.4175,
+ "theoretical_loss": 3.4260995887320176,
+ "tokens_seen": 2246901760
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002580060180541625,
+ "loss": 1.3159,
+ "theoretical_loss": 3.4260915140220902,
+ "tokens_seen": 2246967296
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025798996990972916,
+ "loss": 1.3104,
+ "theoretical_loss": 3.4260834396136106,
+ "tokens_seen": 2247032832
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025797392176529586,
+ "loss": 1.2345,
+ "theoretical_loss": 3.426075365506559,
+ "tokens_seen": 2247098368
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025795787362086257,
+ "loss": 1.2815,
+ "theoretical_loss": 3.4260672917009147,
+ "tokens_seen": 2247163904
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025794182547642927,
+ "loss": 1.2066,
+ "theoretical_loss": 3.4260592181966576,
+ "tokens_seen": 2247229440
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025792577733199597,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4260511449937683,
+ "tokens_seen": 2247294976
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025790972918756267,
+ "loss": 1.2326,
+ "theoretical_loss": 3.426043072092227,
+ "tokens_seen": 2247360512
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025789368104312937,
+ "loss": 1.3732,
+ "theoretical_loss": 3.4260349994920123,
+ "tokens_seen": 2247426048
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025787763289869607,
+ "loss": 1.2563,
+ "theoretical_loss": 3.4260269271931056,
+ "tokens_seen": 2247491584
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025786158475426277,
+ "loss": 1.3138,
+ "theoretical_loss": 3.426018855195486,
+ "tokens_seen": 2247557120
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025784553660982947,
+ "loss": 1.0993,
+ "theoretical_loss": 3.426010783499134,
+ "tokens_seen": 2247622656
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025782948846539617,
+ "loss": 1.1972,
+ "theoretical_loss": 3.426002712104029,
+ "tokens_seen": 2247688192
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025781344032096287,
+ "loss": 1.1782,
+ "theoretical_loss": 3.425994641010152,
+ "tokens_seen": 2247753728
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025779739217652957,
+ "loss": 1.253,
+ "theoretical_loss": 3.425986570217481,
+ "tokens_seen": 2247819264
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 752699,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1567444801330566,
+ "objective/train/theoretical_loss": 3.4259784997259985,
+ "objective/train/tokens_used": 2268344800,
+ "theoretical_loss": 3.4259784997259985,
+ "tokens_seen": 2247884800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025778134403209627,
+ "loss": 1.1792,
+ "theoretical_loss": 3.4259784997259985,
+ "tokens_seen": 2247884800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025776529588766297,
+ "loss": 1.2379,
+ "theoretical_loss": 3.425970429535683,
+ "tokens_seen": 2247950336
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002577492477432297,
+ "loss": 1.2762,
+ "theoretical_loss": 3.425962359646514,
+ "tokens_seen": 2248015872
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002577331995987964,
+ "loss": 1.179,
+ "theoretical_loss": 3.425954290058473,
+ "tokens_seen": 2248081408
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002577171514543631,
+ "loss": 1.2794,
+ "theoretical_loss": 3.425946220771539,
+ "tokens_seen": 2248146944
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002577011033099298,
+ "loss": 1.2385,
+ "theoretical_loss": 3.425938151785692,
+ "tokens_seen": 2248212480
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002576850551654965,
+ "loss": 1.2444,
+ "theoretical_loss": 3.425930083100912,
+ "tokens_seen": 2248278016
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002576690070210632,
+ "loss": 1.2346,
+ "theoretical_loss": 3.4259220147171794,
+ "tokens_seen": 2248343552
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025765295887662993,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4259139466344743,
+ "tokens_seen": 2248409088
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025763691073219663,
+ "loss": 1.3066,
+ "theoretical_loss": 3.4259058788527756,
+ "tokens_seen": 2248474624
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025762086258776333,
+ "loss": 1.2267,
+ "theoretical_loss": 3.4258978113720646,
+ "tokens_seen": 2248540160
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025760481444333003,
+ "loss": 1.2654,
+ "theoretical_loss": 3.425889744192321,
+ "tokens_seen": 2248605696
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025758876629889673,
+ "loss": 1.2398,
+ "theoretical_loss": 3.425881677313524,
+ "tokens_seen": 2248671232
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025757271815446343,
+ "loss": 1.2718,
+ "theoretical_loss": 3.425873610735654,
+ "tokens_seen": 2248736768
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025755667001003014,
+ "loss": 1.259,
+ "theoretical_loss": 3.4258655444586914,
+ "tokens_seen": 2248802304
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025754062186559684,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4258574784826163,
+ "tokens_seen": 2248867840
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025752457372116354,
+ "loss": 1.2773,
+ "theoretical_loss": 3.425849412807408,
+ "tokens_seen": 2248933376
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025750852557673024,
+ "loss": 1.1889,
+ "theoretical_loss": 3.425841347433047,
+ "tokens_seen": 2248998912
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025749247743229694,
+ "loss": 1.2735,
+ "theoretical_loss": 3.425833282359513,
+ "tokens_seen": 2249064448
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025747642928786364,
+ "loss": 1.3222,
+ "theoretical_loss": 3.4258252175867865,
+ "tokens_seen": 2249129984
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025746038114343034,
+ "loss": 1.184,
+ "theoretical_loss": 3.425817153114847,
+ "tokens_seen": 2249195520
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025744433299899704,
+ "loss": 1.2551,
+ "theoretical_loss": 3.4258090889436748,
+ "tokens_seen": 2249261056
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025742828485456374,
+ "loss": 1.2662,
+ "theoretical_loss": 3.42580102507325,
+ "tokens_seen": 2249326592
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025741223671013044,
+ "loss": 1.2804,
+ "theoretical_loss": 3.4257929615035523,
+ "tokens_seen": 2249392128
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025739618856569714,
+ "loss": 1.3647,
+ "theoretical_loss": 3.425784898234562,
+ "tokens_seen": 2249457664
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025738014042126384,
+ "loss": 1.1655,
+ "theoretical_loss": 3.4257768352662588,
+ "tokens_seen": 2249523200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002573640922768305,
+ "loss": 1.2565,
+ "theoretical_loss": 3.425768772598623,
+ "tokens_seen": 2249588736
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002573480441323972,
+ "loss": 1.2632,
+ "theoretical_loss": 3.4257607102316348,
+ "tokens_seen": 2249654272
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002573319959879639,
+ "loss": 1.2603,
+ "theoretical_loss": 3.425752648165274,
+ "tokens_seen": 2249719808
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002573159478435306,
+ "loss": 1.2878,
+ "theoretical_loss": 3.425744586399521,
+ "tokens_seen": 2249785344
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572998996990973,
+ "loss": 1.1345,
+ "theoretical_loss": 3.4257365249343543,
+ "tokens_seen": 2249850880
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000257283851554664,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4257284637697563,
+ "tokens_seen": 2249916416
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572678034102307,
+ "loss": 1.2999,
+ "theoretical_loss": 3.4257204029057053,
+ "tokens_seen": 2249981952
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572517552657974,
+ "loss": 1.2553,
+ "theoretical_loss": 3.425712342342182,
+ "tokens_seen": 2250047488
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572357071213641,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4257042820791663,
+ "tokens_seen": 2250113024
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572196589769308,
+ "loss": 1.2514,
+ "theoretical_loss": 3.4256962221166383,
+ "tokens_seen": 2250178560
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002572036108324975,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4256881624545783,
+ "tokens_seen": 2250244096
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002571875626880642,
+ "loss": 1.1187,
+ "theoretical_loss": 3.425680103092966,
+ "tokens_seen": 2250309632
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002571715145436309,
+ "loss": 1.1878,
+ "theoretical_loss": 3.4256720440317814,
+ "tokens_seen": 2250375168
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002571554663991976,
+ "loss": 1.3023,
+ "theoretical_loss": 3.4256639852710045,
+ "tokens_seen": 2250440704
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002571394182547643,
+ "loss": 1.2913,
+ "theoretical_loss": 3.425655926810616,
+ "tokens_seen": 2250506240
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000257123370110331,
+ "loss": 1.227,
+ "theoretical_loss": 3.425647868650595,
+ "tokens_seen": 2250571776
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002571073219658977,
+ "loss": 1.2792,
+ "theoretical_loss": 3.4256398107909227,
+ "tokens_seen": 2250637312
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570912738214644,
+ "loss": 1.3699,
+ "theoretical_loss": 3.425631753231578,
+ "tokens_seen": 2250702848
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570752256770311,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4256236959725417,
+ "tokens_seen": 2250768384
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570591775325978,
+ "loss": 1.1681,
+ "theoretical_loss": 3.4256156390137935,
+ "tokens_seen": 2250833920
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570431293881645,
+ "loss": 1.2877,
+ "theoretical_loss": 3.4256075823553136,
+ "tokens_seen": 2250899456
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570270812437312,
+ "loss": 1.3389,
+ "theoretical_loss": 3.4255995259970824,
+ "tokens_seen": 2250964992
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002570110330992979,
+ "loss": 1.2114,
+ "theoretical_loss": 3.4255914699390795,
+ "tokens_seen": 2251030528
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002569949849548646,
+ "loss": 1.198,
+ "theoretical_loss": 3.425583414181285,
+ "tokens_seen": 2251096064
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 753469,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.215314269065857,
+ "objective/train/theoretical_loss": 3.4255753587236786,
+ "objective/train/tokens_used": 2271621600,
+ "theoretical_loss": 3.4255753587236786,
+ "tokens_seen": 2251161600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002569789368104313,
+ "loss": 1.1527,
+ "theoretical_loss": 3.4255753587236786,
+ "tokens_seen": 2251161600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000256962888665998,
+ "loss": 1.1617,
+ "theoretical_loss": 3.4255673035662415,
+ "tokens_seen": 2251227136
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002569468405215647,
+ "loss": 1.2568,
+ "theoretical_loss": 3.425559248708953,
+ "tokens_seen": 2251292672
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002569307923771314,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4255511941517933,
+ "tokens_seen": 2251358208
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002569147442326981,
+ "loss": 1.352,
+ "theoretical_loss": 3.4255431398947427,
+ "tokens_seen": 2251423744
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568986960882648,
+ "loss": 1.2929,
+ "theoretical_loss": 3.425535085937781,
+ "tokens_seen": 2251489280
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568826479438315,
+ "loss": 1.2623,
+ "theoretical_loss": 3.425527032280888,
+ "tokens_seen": 2251554816
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568665997993982,
+ "loss": 1.3188,
+ "theoretical_loss": 3.4255189789240443,
+ "tokens_seen": 2251620352
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568505516549649,
+ "loss": 1.3254,
+ "theoretical_loss": 3.42551092586723,
+ "tokens_seen": 2251685888
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568345035105316,
+ "loss": 1.2982,
+ "theoretical_loss": 3.425502873110425,
+ "tokens_seen": 2251751424
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002568184553660983,
+ "loss": 1.2842,
+ "theoretical_loss": 3.4254948206536096,
+ "tokens_seen": 2251816960
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000256802407221665,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4254867684967634,
+ "tokens_seen": 2251882496
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567863590772317,
+ "loss": 1.2819,
+ "theoretical_loss": 3.425478716639867,
+ "tokens_seen": 2251948032
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567703109327984,
+ "loss": 1.2873,
+ "theoretical_loss": 3.4254706650829005,
+ "tokens_seen": 2252013568
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567542627883651,
+ "loss": 1.1428,
+ "theoretical_loss": 3.4254626138258435,
+ "tokens_seen": 2252079104
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567382146439318,
+ "loss": 1.2405,
+ "theoretical_loss": 3.4254545628686768,
+ "tokens_seen": 2252144640
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567221664994985,
+ "loss": 1.1668,
+ "theoretical_loss": 3.42544651221138,
+ "tokens_seen": 2252210176
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002567061183550652,
+ "loss": 1.3322,
+ "theoretical_loss": 3.4254384618539326,
+ "tokens_seen": 2252275712
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002566900702106319,
+ "loss": 1.3657,
+ "theoretical_loss": 3.425430411796316,
+ "tokens_seen": 2252341248
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002566740220661986,
+ "loss": 1.2937,
+ "theoretical_loss": 3.42542236203851,
+ "tokens_seen": 2252406784
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002566579739217653,
+ "loss": 1.2307,
+ "theoretical_loss": 3.425414312580494,
+ "tokens_seen": 2252472320
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000256641925777332,
+ "loss": 1.2622,
+ "theoretical_loss": 3.425406263422249,
+ "tokens_seen": 2252537856
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002566258776328987,
+ "loss": 1.2367,
+ "theoretical_loss": 3.4253982145637543,
+ "tokens_seen": 2252603392
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002566098294884654,
+ "loss": 1.2514,
+ "theoretical_loss": 3.425390166004991,
+ "tokens_seen": 2252668928
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025659378134403206,
+ "loss": 1.3413,
+ "theoretical_loss": 3.425382117745938,
+ "tokens_seen": 2252734464
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025657773319959877,
+ "loss": 1.3059,
+ "theoretical_loss": 3.425374069786576,
+ "tokens_seen": 2252800000
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025656168505516547,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4253660221268856,
+ "tokens_seen": 2252865536
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025654563691073217,
+ "loss": 1.3065,
+ "theoretical_loss": 3.425357974766846,
+ "tokens_seen": 2252931072
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025652958876629887,
+ "loss": 1.2044,
+ "theoretical_loss": 3.425349927706438,
+ "tokens_seen": 2252996608
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025651354062186557,
+ "loss": 1.3195,
+ "theoretical_loss": 3.4253418809456417,
+ "tokens_seen": 2253062144
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025649749247743227,
+ "loss": 1.1761,
+ "theoretical_loss": 3.4253338344844373,
+ "tokens_seen": 2253127680
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025648144433299897,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4253257883228043,
+ "tokens_seen": 2253193216
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025646539618856567,
+ "loss": 1.2669,
+ "theoretical_loss": 3.425317742460723,
+ "tokens_seen": 2253258752
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002564493480441324,
+ "loss": 1.2762,
+ "theoretical_loss": 3.425309696898174,
+ "tokens_seen": 2253324288
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002564332998996991,
+ "loss": 1.1921,
+ "theoretical_loss": 3.4253016516351376,
+ "tokens_seen": 2253389824
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002564172517552658,
+ "loss": 1.2248,
+ "theoretical_loss": 3.4252936066715933,
+ "tokens_seen": 2253455360
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002564012036108325,
+ "loss": 1.3451,
+ "theoretical_loss": 3.4252855620075215,
+ "tokens_seen": 2253520896
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025638515546639923,
+ "loss": 1.138,
+ "theoretical_loss": 3.4252775176429022,
+ "tokens_seen": 2253586432
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025636910732196593,
+ "loss": 1.1937,
+ "theoretical_loss": 3.425269473577716,
+ "tokens_seen": 2253651968
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025635305917753263,
+ "loss": 1.2648,
+ "theoretical_loss": 3.4252614298119424,
+ "tokens_seen": 2253717504
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025633701103309933,
+ "loss": 1.1999,
+ "theoretical_loss": 3.425253386345562,
+ "tokens_seen": 2253783040
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025632096288866603,
+ "loss": 1.3652,
+ "theoretical_loss": 3.425245343178555,
+ "tokens_seen": 2253848576
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025630491474423273,
+ "loss": 1.2079,
+ "theoretical_loss": 3.4252373003109007,
+ "tokens_seen": 2253914112
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025628886659979943,
+ "loss": 1.2153,
+ "theoretical_loss": 3.4252292577425805,
+ "tokens_seen": 2253979648
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025627281845536613,
+ "loss": 1.2059,
+ "theoretical_loss": 3.425221215473574,
+ "tokens_seen": 2254045184
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025625677031093283,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4252131735038613,
+ "tokens_seen": 2254110720
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025624072216649953,
+ "loss": 1.2685,
+ "theoretical_loss": 3.4252051318334225,
+ "tokens_seen": 2254176256
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025622467402206623,
+ "loss": 1.3108,
+ "theoretical_loss": 3.425197090462238,
+ "tokens_seen": 2254241792
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025620862587763293,
+ "loss": 1.3067,
+ "theoretical_loss": 3.425189049390288,
+ "tokens_seen": 2254307328
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025619257773319964,
+ "loss": 1.2607,
+ "theoretical_loss": 3.4251810086175523,
+ "tokens_seen": 2254372864
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 754585,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1932061910629272,
+ "objective/train/theoretical_loss": 3.425172968144011,
+ "objective/train/tokens_used": 2274898400,
+ "theoretical_loss": 3.425172968144011,
+ "tokens_seen": 2254438400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025617652958876634,
+ "loss": 1.2115,
+ "theoretical_loss": 3.425172968144011,
+ "tokens_seen": 2254438400
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025616048144433304,
+ "loss": 1.1674,
+ "theoretical_loss": 3.425164927969645,
+ "tokens_seen": 2254503936
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025614443329989974,
+ "loss": 1.2241,
+ "theoretical_loss": 3.425156888094434,
+ "tokens_seen": 2254569472
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025612838515546644,
+ "loss": 1.1817,
+ "theoretical_loss": 3.425148848518358,
+ "tokens_seen": 2254635008
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025611233701103314,
+ "loss": 1.2647,
+ "theoretical_loss": 3.4251408092413973,
+ "tokens_seen": 2254700544
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025609628886659984,
+ "loss": 1.3923,
+ "theoretical_loss": 3.425132770263532,
+ "tokens_seen": 2254766080
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025608024072216654,
+ "loss": 1.221,
+ "theoretical_loss": 3.425124731584743,
+ "tokens_seen": 2254831616
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025606419257773324,
+ "loss": 1.1826,
+ "theoretical_loss": 3.4251166932050094,
+ "tokens_seen": 2254897152
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025604814443329994,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4251086551243124,
+ "tokens_seen": 2254962688
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025603209628886664,
+ "loss": 1.2202,
+ "theoretical_loss": 3.4251006173426313,
+ "tokens_seen": 2255028224
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025601604814443334,
+ "loss": 1.3595,
+ "theoretical_loss": 3.4250925798599465,
+ "tokens_seen": 2255093760
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025600000000000004,
+ "loss": 1.2818,
+ "theoretical_loss": 3.4250845426762386,
+ "tokens_seen": 2255159296
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025598395185556674,
+ "loss": 1.1745,
+ "theoretical_loss": 3.4250765057914876,
+ "tokens_seen": 2255224832
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002559679037111334,
+ "loss": 1.2497,
+ "theoretical_loss": 3.4250684692056734,
+ "tokens_seen": 2255290368
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002559518555667001,
+ "loss": 1.2763,
+ "theoretical_loss": 3.4250604329187766,
+ "tokens_seen": 2255355904
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002559358074222668,
+ "loss": 1.1942,
+ "theoretical_loss": 3.425052396930777,
+ "tokens_seen": 2255421440
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002559197592778335,
+ "loss": 1.2346,
+ "theoretical_loss": 3.4250443612416555,
+ "tokens_seen": 2255486976
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002559037111334002,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4250363258513916,
+ "tokens_seen": 2255552512
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002558876629889669,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4250282907599656,
+ "tokens_seen": 2255618048
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002558716148445336,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4250202559673575,
+ "tokens_seen": 2255683584
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002558555667001003,
+ "loss": 1.3108,
+ "theoretical_loss": 3.4250122214735486,
+ "tokens_seen": 2255749120
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255839518555667,
+ "loss": 1.3033,
+ "theoretical_loss": 3.4250041872785175,
+ "tokens_seen": 2255814656
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002558234704112337,
+ "loss": 1.2607,
+ "theoretical_loss": 3.4249961533822457,
+ "tokens_seen": 2255880192
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002558074222668004,
+ "loss": 1.2957,
+ "theoretical_loss": 3.424988119784713,
+ "tokens_seen": 2255945728
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557913741223671,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4249800864859,
+ "tokens_seen": 2256011264
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557753259779338,
+ "loss": 1.2041,
+ "theoretical_loss": 3.424972053485786,
+ "tokens_seen": 2256076800
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557592778335005,
+ "loss": 1.3353,
+ "theoretical_loss": 3.4249640207843517,
+ "tokens_seen": 2256142336
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557432296890672,
+ "loss": 1.2474,
+ "theoretical_loss": 3.424955988381577,
+ "tokens_seen": 2256207872
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557271815446339,
+ "loss": 1.2486,
+ "theoretical_loss": 3.424947956277443,
+ "tokens_seen": 2256273408
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002557111334002006,
+ "loss": 1.3202,
+ "theoretical_loss": 3.4249399244719294,
+ "tokens_seen": 2256338944
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002556950852557673,
+ "loss": 1.304,
+ "theoretical_loss": 3.424931892965016,
+ "tokens_seen": 2256404480
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255679037111334,
+ "loss": 1.3286,
+ "theoretical_loss": 3.4249238617566835,
+ "tokens_seen": 2256470016
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002556629889669007,
+ "loss": 1.1692,
+ "theoretical_loss": 3.4249158308469125,
+ "tokens_seen": 2256535552
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002556469408224674,
+ "loss": 1.1953,
+ "theoretical_loss": 3.4249078002356823,
+ "tokens_seen": 2256601088
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002556308926780341,
+ "loss": 1.347,
+ "theoretical_loss": 3.4248997699229737,
+ "tokens_seen": 2256666624
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002556148445336008,
+ "loss": 1.272,
+ "theoretical_loss": 3.424891739908767,
+ "tokens_seen": 2256732160
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555987963891675,
+ "loss": 1.31,
+ "theoretical_loss": 3.424883710193042,
+ "tokens_seen": 2256797696
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555827482447342,
+ "loss": 1.3133,
+ "theoretical_loss": 3.4248756807757794,
+ "tokens_seen": 2256863232
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555667001003009,
+ "loss": 1.3467,
+ "theoretical_loss": 3.4248676516569594,
+ "tokens_seen": 2256928768
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555506519558676,
+ "loss": 1.235,
+ "theoretical_loss": 3.424859622836562,
+ "tokens_seen": 2256994304
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555346038114343,
+ "loss": 1.2618,
+ "theoretical_loss": 3.4248515943145676,
+ "tokens_seen": 2257059840
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255518555667001,
+ "loss": 1.2116,
+ "theoretical_loss": 3.4248435660909564,
+ "tokens_seen": 2257125376
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002555025075225677,
+ "loss": 1.2972,
+ "theoretical_loss": 3.4248355381657083,
+ "tokens_seen": 2257190912
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554864593781344,
+ "loss": 1.3585,
+ "theoretical_loss": 3.424827510538804,
+ "tokens_seen": 2257256448
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554704112337011,
+ "loss": 1.2303,
+ "theoretical_loss": 3.424819483210224,
+ "tokens_seen": 2257321984
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554543630892678,
+ "loss": 1.2153,
+ "theoretical_loss": 3.424811456179948,
+ "tokens_seen": 2257387520
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554383149448345,
+ "loss": 1.2916,
+ "theoretical_loss": 3.4248034294479566,
+ "tokens_seen": 2257453056
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554222668004012,
+ "loss": 1.2517,
+ "theoretical_loss": 3.4247954030142296,
+ "tokens_seen": 2257518592
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002554062186559679,
+ "loss": 1.2494,
+ "theoretical_loss": 3.4247873768787476,
+ "tokens_seen": 2257584128
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002553901705115346,
+ "loss": 1.18,
+ "theoretical_loss": 3.424779351041491,
+ "tokens_seen": 2257649664
+ },
+ {
+ "epoch": 0.68,
+ "objective/train/docs_used": 755500,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2898519039154053,
+ "objective/train/theoretical_loss": 3.42477132550244,
+ "objective/train/tokens_used": 2278175200,
+ "theoretical_loss": 3.42477132550244,
+ "tokens_seen": 2257715200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002553741223671013,
+ "loss": 1.3005,
+ "theoretical_loss": 3.42477132550244,
+ "tokens_seen": 2257715200
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255358074222668,
+ "loss": 1.2849,
+ "theoretical_loss": 3.4247633002615743,
+ "tokens_seen": 2257780736
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002553420260782347,
+ "loss": 1.3798,
+ "theoretical_loss": 3.424755275318875,
+ "tokens_seen": 2257846272
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002553259779338014,
+ "loss": 1.2874,
+ "theoretical_loss": 3.424747250674322,
+ "tokens_seen": 2257911808
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002553099297893681,
+ "loss": 1.1557,
+ "theoretical_loss": 3.4247392263278953,
+ "tokens_seen": 2257977344
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552938816449348,
+ "loss": 1.3045,
+ "theoretical_loss": 3.4247312022795757,
+ "tokens_seen": 2258042880
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552778335005015,
+ "loss": 1.2919,
+ "theoretical_loss": 3.424723178529343,
+ "tokens_seen": 2258108416
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552617853560682,
+ "loss": 1.1924,
+ "theoretical_loss": 3.4247151550771777,
+ "tokens_seen": 2258173952
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552457372116349,
+ "loss": 1.2231,
+ "theoretical_loss": 3.42470713192306,
+ "tokens_seen": 2258239488
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552296890672016,
+ "loss": 1.1726,
+ "theoretical_loss": 3.4246991090669705,
+ "tokens_seen": 2258305024
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002552136409227683,
+ "loss": 1.2395,
+ "theoretical_loss": 3.424691086508889,
+ "tokens_seen": 2258370560
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255197592778335,
+ "loss": 1.2466,
+ "theoretical_loss": 3.424683064248796,
+ "tokens_seen": 2258436096
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551815446339017,
+ "loss": 1.2591,
+ "theoretical_loss": 3.4246750422866716,
+ "tokens_seen": 2258501632
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551654964894684,
+ "loss": 1.1827,
+ "theoretical_loss": 3.424667020622497,
+ "tokens_seen": 2258567168
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551494483450351,
+ "loss": 1.1719,
+ "theoretical_loss": 3.424658999256251,
+ "tokens_seen": 2258632704
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551334002006018,
+ "loss": 1.2197,
+ "theoretical_loss": 3.4246509781879153,
+ "tokens_seen": 2258698240
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551173520561685,
+ "loss": 1.3275,
+ "theoretical_loss": 3.424642957417469,
+ "tokens_seen": 2258763776
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002551013039117352,
+ "loss": 1.2389,
+ "theoretical_loss": 3.424634936944893,
+ "tokens_seen": 2258829312
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002550852557673019,
+ "loss": 1.2772,
+ "theoretical_loss": 3.4246269167701677,
+ "tokens_seen": 2258894848
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002550692076228686,
+ "loss": 1.2107,
+ "theoretical_loss": 3.424618896893273,
+ "tokens_seen": 2258960384
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.0002550531594784353,
+ "loss": 1.3469,
+ "theoretical_loss": 3.4246108773141897,
+ "tokens_seen": 2259025920
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.000255037111334002,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4246028580328973,
+ "tokens_seen": 2259091456
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025502106318956873,
+ "loss": 1.3107,
+ "theoretical_loss": 3.4245948390493774,
+ "tokens_seen": 2259156992
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025500501504513543,
+ "loss": 1.228,
+ "theoretical_loss": 3.424586820363609,
+ "tokens_seen": 2259222528
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025498896690070213,
+ "loss": 1.2238,
+ "theoretical_loss": 3.4245788019755734,
+ "tokens_seen": 2259288064
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025497291875626883,
+ "loss": 1.4206,
+ "theoretical_loss": 3.42457078388525,
+ "tokens_seen": 2259353600
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025495687061183553,
+ "loss": 1.2871,
+ "theoretical_loss": 3.4245627660926194,
+ "tokens_seen": 2259419136
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025494082246740223,
+ "loss": 1.239,
+ "theoretical_loss": 3.4245547485976626,
+ "tokens_seen": 2259484672
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025492477432296893,
+ "loss": 1.176,
+ "theoretical_loss": 3.4245467314003593,
+ "tokens_seen": 2259550208
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025490872617853563,
+ "loss": 1.3052,
+ "theoretical_loss": 3.42453871450069,
+ "tokens_seen": 2259615744
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025489267803410233,
+ "loss": 1.2952,
+ "theoretical_loss": 3.4245306978986347,
+ "tokens_seen": 2259681280
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025487662988966903,
+ "loss": 1.324,
+ "theoretical_loss": 3.424522681594174,
+ "tokens_seen": 2259746816
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025486058174523573,
+ "loss": 1.2491,
+ "theoretical_loss": 3.4245146655872882,
+ "tokens_seen": 2259812352
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025484453360080243,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4245066498779577,
+ "tokens_seen": 2259877888
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025482848545636913,
+ "loss": 1.1547,
+ "theoretical_loss": 3.4244986344661625,
+ "tokens_seen": 2259943424
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025481243731193584,
+ "loss": 1.2935,
+ "theoretical_loss": 3.424490619351883,
+ "tokens_seen": 2260008960
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025479638916750254,
+ "loss": 1.1152,
+ "theoretical_loss": 3.4244826045351004,
+ "tokens_seen": 2260074496
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025478034102306924,
+ "loss": 1.3116,
+ "theoretical_loss": 3.4244745900157936,
+ "tokens_seen": 2260140032
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025476429287863594,
+ "loss": 1.2537,
+ "theoretical_loss": 3.4244665757939443,
+ "tokens_seen": 2260205568
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025474824473420264,
+ "loss": 1.2539,
+ "theoretical_loss": 3.424458561869532,
+ "tokens_seen": 2260271104
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025473219658976934,
+ "loss": 1.2467,
+ "theoretical_loss": 3.4244505482425365,
+ "tokens_seen": 2260336640
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025471614844533604,
+ "loss": 1.2374,
+ "theoretical_loss": 3.4244425349129397,
+ "tokens_seen": 2260402176
+ },
+ {
+ "epoch": 0.68,
+ "learning_rate": 0.00025470010030090274,
+ "loss": 1.3068,
+ "theoretical_loss": 3.4244345218807206,
+ "tokens_seen": 2260467712
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025468405215646944,
+ "loss": 1.2747,
+ "theoretical_loss": 3.42442650914586,
+ "tokens_seen": 2260533248
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025466800401203614,
+ "loss": 1.1944,
+ "theoretical_loss": 3.4244184967083386,
+ "tokens_seen": 2260598784
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025465195586760284,
+ "loss": 1.0917,
+ "theoretical_loss": 3.4244104845681367,
+ "tokens_seen": 2260664320
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025463590772316954,
+ "loss": 1.1865,
+ "theoretical_loss": 3.424402472725234,
+ "tokens_seen": 2260729856
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025461985957873624,
+ "loss": 1.1873,
+ "theoretical_loss": 3.4243944611796113,
+ "tokens_seen": 2260795392
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025460381143430294,
+ "loss": 1.2376,
+ "theoretical_loss": 3.424386449931249,
+ "tokens_seen": 2260860928
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025458776328986964,
+ "loss": 1.4568,
+ "theoretical_loss": 3.424378438980127,
+ "tokens_seen": 2260926464
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 756703,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1879603862762451,
+ "objective/train/theoretical_loss": 3.424370428326226,
+ "objective/train/tokens_used": 2281452000,
+ "theoretical_loss": 3.424370428326226,
+ "tokens_seen": 2260992000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025457171514543634,
+ "loss": 1.1289,
+ "theoretical_loss": 3.424370428326226,
+ "tokens_seen": 2260992000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000254555667001003,
+ "loss": 1.2832,
+ "theoretical_loss": 3.424362417969527,
+ "tokens_seen": 2261057536
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002545396188565697,
+ "loss": 1.3101,
+ "theoretical_loss": 3.424354407910009,
+ "tokens_seen": 2261123072
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002545235707121364,
+ "loss": 1.2765,
+ "theoretical_loss": 3.4243463981476534,
+ "tokens_seen": 2261188608
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002545075225677031,
+ "loss": 1.3419,
+ "theoretical_loss": 3.4243383886824406,
+ "tokens_seen": 2261254144
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544914744232698,
+ "loss": 1.1994,
+ "theoretical_loss": 3.42433037951435,
+ "tokens_seen": 2261319680
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544754262788365,
+ "loss": 1.217,
+ "theoretical_loss": 3.4243223706433628,
+ "tokens_seen": 2261385216
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544593781344032,
+ "loss": 1.3251,
+ "theoretical_loss": 3.4243143620694587,
+ "tokens_seen": 2261450752
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544433299899699,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4243063537926193,
+ "tokens_seen": 2261516288
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544272818455366,
+ "loss": 1.2834,
+ "theoretical_loss": 3.424298345812824,
+ "tokens_seen": 2261581824
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002544112337011033,
+ "loss": 1.1707,
+ "theoretical_loss": 3.424290338130053,
+ "tokens_seen": 2261647360
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025439518555667,
+ "loss": 1.2109,
+ "theoretical_loss": 3.4242823307442873,
+ "tokens_seen": 2261712896
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002543791374122367,
+ "loss": 1.2241,
+ "theoretical_loss": 3.424274323655507,
+ "tokens_seen": 2261778432
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002543630892678034,
+ "loss": 1.0865,
+ "theoretical_loss": 3.4242663168636924,
+ "tokens_seen": 2261843968
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002543470411233701,
+ "loss": 1.2122,
+ "theoretical_loss": 3.424258310368824,
+ "tokens_seen": 2261909504
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002543309929789368,
+ "loss": 1.2553,
+ "theoretical_loss": 3.424250304170882,
+ "tokens_seen": 2261975040
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002543149448345035,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4242422982698475,
+ "tokens_seen": 2262040576
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542988966900702,
+ "loss": 1.2306,
+ "theoretical_loss": 3.4242342926657,
+ "tokens_seen": 2262106112
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542828485456369,
+ "loss": 1.2504,
+ "theoretical_loss": 3.42422628735842,
+ "tokens_seen": 2262171648
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542668004012036,
+ "loss": 1.2626,
+ "theoretical_loss": 3.4242182823479883,
+ "tokens_seen": 2262237184
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542507522567703,
+ "loss": 1.2583,
+ "theoretical_loss": 3.4242102776343852,
+ "tokens_seen": 2262302720
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000254234704112337,
+ "loss": 1.1822,
+ "theoretical_loss": 3.424202273217591,
+ "tokens_seen": 2262368256
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542186559679037,
+ "loss": 1.2763,
+ "theoretical_loss": 3.424194269097586,
+ "tokens_seen": 2262433792
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002542026078234704,
+ "loss": 1.2583,
+ "theoretical_loss": 3.4241862652743507,
+ "tokens_seen": 2262499328
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541865596790371,
+ "loss": 1.352,
+ "theoretical_loss": 3.4241782617478655,
+ "tokens_seen": 2262564864
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541705115346038,
+ "loss": 1.2293,
+ "theoretical_loss": 3.424170258518111,
+ "tokens_seen": 2262630400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541544633901705,
+ "loss": 1.1916,
+ "theoretical_loss": 3.424162255585067,
+ "tokens_seen": 2262695936
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541384152457372,
+ "loss": 1.2708,
+ "theoretical_loss": 3.4241542529487146,
+ "tokens_seen": 2262761472
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541223671013039,
+ "loss": 1.227,
+ "theoretical_loss": 3.424146250609034,
+ "tokens_seen": 2262827008
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002541063189568706,
+ "loss": 1.225,
+ "theoretical_loss": 3.4241382485660052,
+ "tokens_seen": 2262892544
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002540902708124373,
+ "loss": 1.2593,
+ "theoretical_loss": 3.424130246819609,
+ "tokens_seen": 2262958080
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000254074222668004,
+ "loss": 1.2914,
+ "theoretical_loss": 3.4241222453698263,
+ "tokens_seen": 2263023616
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002540581745235707,
+ "loss": 1.3057,
+ "theoretical_loss": 3.4241142442166366,
+ "tokens_seen": 2263089152
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025404212637913747,
+ "loss": 1.1903,
+ "theoretical_loss": 3.4241062433600207,
+ "tokens_seen": 2263154688
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025402607823470417,
+ "loss": 1.2713,
+ "theoretical_loss": 3.424098242799959,
+ "tokens_seen": 2263220224
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025401003009027087,
+ "loss": 1.1481,
+ "theoretical_loss": 3.4240902425364323,
+ "tokens_seen": 2263285760
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025399398194583757,
+ "loss": 1.2993,
+ "theoretical_loss": 3.42408224256942,
+ "tokens_seen": 2263351296
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025397793380140427,
+ "loss": 1.2071,
+ "theoretical_loss": 3.4240742428989037,
+ "tokens_seen": 2263416832
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025396188565697097,
+ "loss": 1.2496,
+ "theoretical_loss": 3.424066243524863,
+ "tokens_seen": 2263482368
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025394583751253767,
+ "loss": 1.2413,
+ "theoretical_loss": 3.424058244447279,
+ "tokens_seen": 2263547904
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002539297893681043,
+ "loss": 1.1994,
+ "theoretical_loss": 3.4240502456661313,
+ "tokens_seen": 2263613440
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000253913741223671,
+ "loss": 1.1629,
+ "theoretical_loss": 3.424042247181401,
+ "tokens_seen": 2263678976
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538976930792377,
+ "loss": 1.3103,
+ "theoretical_loss": 3.4240342489930686,
+ "tokens_seen": 2263744512
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538816449348044,
+ "loss": 1.2784,
+ "theoretical_loss": 3.4240262511011137,
+ "tokens_seen": 2263810048
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538655967903711,
+ "loss": 1.1305,
+ "theoretical_loss": 3.4240182535055177,
+ "tokens_seen": 2263875584
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538495486459378,
+ "loss": 1.2864,
+ "theoretical_loss": 3.4240102562062606,
+ "tokens_seen": 2263941120
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538335005015045,
+ "loss": 1.1519,
+ "theoretical_loss": 3.424002259203323,
+ "tokens_seen": 2264006656
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538174523570712,
+ "loss": 1.1906,
+ "theoretical_loss": 3.4239942624966853,
+ "tokens_seen": 2264072192
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002538014042126379,
+ "loss": 1.4173,
+ "theoretical_loss": 3.4239862660863274,
+ "tokens_seen": 2264137728
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002537853560682046,
+ "loss": 1.1836,
+ "theoretical_loss": 3.4239782699722308,
+ "tokens_seen": 2264203264
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 757625,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2602403163909912,
+ "objective/train/theoretical_loss": 3.423970274154375,
+ "objective/train/tokens_used": 2284728800,
+ "theoretical_loss": 3.423970274154375,
+ "tokens_seen": 2264268800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002537693079237713,
+ "loss": 1.2912,
+ "theoretical_loss": 3.423970274154375,
+ "tokens_seen": 2264268800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000253753259779338,
+ "loss": 1.1354,
+ "theoretical_loss": 3.4239622786327413,
+ "tokens_seen": 2264334336
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002537372116349047,
+ "loss": 1.157,
+ "theoretical_loss": 3.4239542834073093,
+ "tokens_seen": 2264399872
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002537211634904714,
+ "loss": 1.3084,
+ "theoretical_loss": 3.4239462884780596,
+ "tokens_seen": 2264465408
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002537051153460381,
+ "loss": 1.2993,
+ "theoretical_loss": 3.423938293844973,
+ "tokens_seen": 2264530944
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002536890672016048,
+ "loss": 1.2535,
+ "theoretical_loss": 3.4239302995080303,
+ "tokens_seen": 2264596480
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002536730190571715,
+ "loss": 1.2121,
+ "theoretical_loss": 3.423922305467211,
+ "tokens_seen": 2264662016
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002536569709127382,
+ "loss": 1.2473,
+ "theoretical_loss": 3.4239143117224966,
+ "tokens_seen": 2264727552
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025364092276830493,
+ "loss": 1.2107,
+ "theoretical_loss": 3.4239063182738665,
+ "tokens_seen": 2264793088
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025362487462387163,
+ "loss": 1.2191,
+ "theoretical_loss": 3.4238983251213018,
+ "tokens_seen": 2264858624
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025360882647943833,
+ "loss": 1.2205,
+ "theoretical_loss": 3.423890332264783,
+ "tokens_seen": 2264924160
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025359277833500503,
+ "loss": 1.1555,
+ "theoretical_loss": 3.4238823397042903,
+ "tokens_seen": 2264989696
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025357673019057173,
+ "loss": 1.2833,
+ "theoretical_loss": 3.4238743474398046,
+ "tokens_seen": 2265055232
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025356068204613843,
+ "loss": 1.2103,
+ "theoretical_loss": 3.423866355471306,
+ "tokens_seen": 2265120768
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025354463390170513,
+ "loss": 1.1955,
+ "theoretical_loss": 3.423858363798775,
+ "tokens_seen": 2265186304
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025352858575727183,
+ "loss": 1.2207,
+ "theoretical_loss": 3.4238503724221925,
+ "tokens_seen": 2265251840
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025351253761283853,
+ "loss": 1.2607,
+ "theoretical_loss": 3.423842381341538,
+ "tokens_seen": 2265317376
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025349648946840523,
+ "loss": 1.2788,
+ "theoretical_loss": 3.423834390556793,
+ "tokens_seen": 2265382912
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025348044132397193,
+ "loss": 1.2468,
+ "theoretical_loss": 3.423826400067937,
+ "tokens_seen": 2265448448
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025346439317953863,
+ "loss": 1.2413,
+ "theoretical_loss": 3.423818409874952,
+ "tokens_seen": 2265513984
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025344834503510534,
+ "loss": 1.3598,
+ "theoretical_loss": 3.423810419977817,
+ "tokens_seen": 2265579520
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025343229689067204,
+ "loss": 1.2453,
+ "theoretical_loss": 3.423802430376513,
+ "tokens_seen": 2265645056
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025341624874623874,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4237944410710206,
+ "tokens_seen": 2265710592
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025340020060180544,
+ "loss": 1.2374,
+ "theoretical_loss": 3.42378645206132,
+ "tokens_seen": 2265776128
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025338415245737214,
+ "loss": 1.2984,
+ "theoretical_loss": 3.4237784633473924,
+ "tokens_seen": 2265841664
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025336810431293884,
+ "loss": 1.1565,
+ "theoretical_loss": 3.4237704749292175,
+ "tokens_seen": 2265907200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025335205616850554,
+ "loss": 1.327,
+ "theoretical_loss": 3.423762486806776,
+ "tokens_seen": 2265972736
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025333600802407224,
+ "loss": 1.2541,
+ "theoretical_loss": 3.423754498980049,
+ "tokens_seen": 2266038272
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025331995987963894,
+ "loss": 1.1624,
+ "theoretical_loss": 3.423746511449016,
+ "tokens_seen": 2266103808
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025330391173520564,
+ "loss": 1.2236,
+ "theoretical_loss": 3.4237385242136584,
+ "tokens_seen": 2266169344
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025328786359077234,
+ "loss": 1.2687,
+ "theoretical_loss": 3.423730537273956,
+ "tokens_seen": 2266234880
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025327181544633904,
+ "loss": 1.1889,
+ "theoretical_loss": 3.42372255062989,
+ "tokens_seen": 2266300416
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025325576730190574,
+ "loss": 1.2716,
+ "theoretical_loss": 3.42371456428144,
+ "tokens_seen": 2266365952
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025323971915747244,
+ "loss": 1.1959,
+ "theoretical_loss": 3.4237065782285874,
+ "tokens_seen": 2266431488
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025322367101303914,
+ "loss": 1.1599,
+ "theoretical_loss": 3.4236985924713124,
+ "tokens_seen": 2266497024
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025320762286860584,
+ "loss": 1.1607,
+ "theoretical_loss": 3.423690607009595,
+ "tokens_seen": 2266562560
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025319157472417255,
+ "loss": 1.2472,
+ "theoretical_loss": 3.4236826218434167,
+ "tokens_seen": 2266628096
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025317552657973925,
+ "loss": 1.3503,
+ "theoretical_loss": 3.4236746369727573,
+ "tokens_seen": 2266693632
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002531594784353059,
+ "loss": 1.2719,
+ "theoretical_loss": 3.4236666523975976,
+ "tokens_seen": 2266759168
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002531434302908726,
+ "loss": 1.226,
+ "theoretical_loss": 3.423658668117918,
+ "tokens_seen": 2266824704
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002531273821464393,
+ "loss": 1.283,
+ "theoretical_loss": 3.423650684133699,
+ "tokens_seen": 2266890240
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000253111334002006,
+ "loss": 1.2581,
+ "theoretical_loss": 3.423642700444921,
+ "tokens_seen": 2266955776
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530952858575727,
+ "loss": 1.1902,
+ "theoretical_loss": 3.423634717051565,
+ "tokens_seen": 2267021312
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530792377131394,
+ "loss": 1.1994,
+ "theoretical_loss": 3.4236267339536113,
+ "tokens_seen": 2267086848
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530631895687061,
+ "loss": 1.2128,
+ "theoretical_loss": 3.4236187511510403,
+ "tokens_seen": 2267152384
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530471414242728,
+ "loss": 1.2073,
+ "theoretical_loss": 3.423610768643832,
+ "tokens_seen": 2267217920
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530310932798395,
+ "loss": 1.2642,
+ "theoretical_loss": 3.4236027864319682,
+ "tokens_seen": 2267283456
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002530150451354062,
+ "loss": 1.1769,
+ "theoretical_loss": 3.423594804515429,
+ "tokens_seen": 2267348992
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529989969909729,
+ "loss": 1.301,
+ "theoretical_loss": 3.4235868228941944,
+ "tokens_seen": 2267414528
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529829488465396,
+ "loss": 1.1972,
+ "theoretical_loss": 3.4235788415682453,
+ "tokens_seen": 2267480064
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 758824,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.331571340560913,
+ "objective/train/theoretical_loss": 3.423570860537562,
+ "objective/train/tokens_used": 2288005600,
+ "theoretical_loss": 3.423570860537562,
+ "tokens_seen": 2267545600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529669007021063,
+ "loss": 1.3733,
+ "theoretical_loss": 3.423570860537562,
+ "tokens_seen": 2267545600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000252950852557673,
+ "loss": 1.0821,
+ "theoretical_loss": 3.4235628798021254,
+ "tokens_seen": 2267611136
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529348044132397,
+ "loss": 1.2548,
+ "theoretical_loss": 3.423554899361916,
+ "tokens_seen": 2267676672
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529187562688064,
+ "loss": 1.2521,
+ "theoretical_loss": 3.423546919216914,
+ "tokens_seen": 2267742208
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002529027081243731,
+ "loss": 1.2387,
+ "theoretical_loss": 3.4235389393671003,
+ "tokens_seen": 2267807744
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002528866599799398,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4235309598124553,
+ "tokens_seen": 2267873280
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002528706118355065,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4235229805529594,
+ "tokens_seen": 2267938816
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002528545636910732,
+ "loss": 1.2381,
+ "theoretical_loss": 3.423515001588594,
+ "tokens_seen": 2268004352
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025283851554663996,
+ "loss": 1.2329,
+ "theoretical_loss": 3.4235070229193383,
+ "tokens_seen": 2268069888
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025282246740220666,
+ "loss": 1.2003,
+ "theoretical_loss": 3.423499044545174,
+ "tokens_seen": 2268135424
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025280641925777336,
+ "loss": 1.2083,
+ "theoretical_loss": 3.423491066466081,
+ "tokens_seen": 2268200960
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025279037111334006,
+ "loss": 1.1842,
+ "theoretical_loss": 3.42348308868204,
+ "tokens_seen": 2268266496
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025277432296890676,
+ "loss": 1.1414,
+ "theoretical_loss": 3.4234751111930315,
+ "tokens_seen": 2268332032
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025275827482447346,
+ "loss": 1.2506,
+ "theoretical_loss": 3.4234671339990363,
+ "tokens_seen": 2268397568
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025274222668004016,
+ "loss": 1.2079,
+ "theoretical_loss": 3.4234591571000355,
+ "tokens_seen": 2268463104
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025272617853560686,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4234511804960084,
+ "tokens_seen": 2268528640
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025271013039117356,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4234432041869365,
+ "tokens_seen": 2268594176
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025269408224674026,
+ "loss": 1.2663,
+ "theoretical_loss": 3.4234352281727998,
+ "tokens_seen": 2268659712
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025267803410230697,
+ "loss": 1.3754,
+ "theoretical_loss": 3.423427252453579,
+ "tokens_seen": 2268725248
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025266198595787367,
+ "loss": 1.209,
+ "theoretical_loss": 3.4234192770292555,
+ "tokens_seen": 2268790784
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025264593781344037,
+ "loss": 1.2915,
+ "theoretical_loss": 3.423411301899809,
+ "tokens_seen": 2268856320
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025262988966900707,
+ "loss": 1.2496,
+ "theoretical_loss": 3.4234033270652198,
+ "tokens_seen": 2268921856
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025261384152457377,
+ "loss": 1.227,
+ "theoretical_loss": 3.4233953525254694,
+ "tokens_seen": 2268987392
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025259779338014047,
+ "loss": 1.2188,
+ "theoretical_loss": 3.423387378280538,
+ "tokens_seen": 2269052928
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025258174523570717,
+ "loss": 1.2183,
+ "theoretical_loss": 3.423379404330406,
+ "tokens_seen": 2269118464
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025256569709127387,
+ "loss": 1.1318,
+ "theoretical_loss": 3.4233714306750542,
+ "tokens_seen": 2269184000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025254964894684057,
+ "loss": 1.2962,
+ "theoretical_loss": 3.4233634573144633,
+ "tokens_seen": 2269249536
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025253360080240727,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4233554842486136,
+ "tokens_seen": 2269315072
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002525175526579739,
+ "loss": 1.1076,
+ "theoretical_loss": 3.423347511477486,
+ "tokens_seen": 2269380608
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002525015045135406,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4233395390010606,
+ "tokens_seen": 2269446144
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002524854563691073,
+ "loss": 1.2971,
+ "theoretical_loss": 3.4233315668193183,
+ "tokens_seen": 2269511680
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000252469408224674,
+ "loss": 1.2631,
+ "theoretical_loss": 3.42332359493224,
+ "tokens_seen": 2269577216
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002524533600802407,
+ "loss": 1.2931,
+ "theoretical_loss": 3.4233156233398057,
+ "tokens_seen": 2269642752
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002524373119358074,
+ "loss": 1.2804,
+ "theoretical_loss": 3.4233076520419963,
+ "tokens_seen": 2269708288
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002524212637913741,
+ "loss": 1.3802,
+ "theoretical_loss": 3.4232996810387926,
+ "tokens_seen": 2269773824
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002524052156469408,
+ "loss": 1.1924,
+ "theoretical_loss": 3.423291710330175,
+ "tokens_seen": 2269839360
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002523891675025075,
+ "loss": 1.1484,
+ "theoretical_loss": 3.423283739916124,
+ "tokens_seen": 2269904896
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002523731193580742,
+ "loss": 1.3077,
+ "theoretical_loss": 3.42327576979662,
+ "tokens_seen": 2269970432
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002523570712136409,
+ "loss": 1.349,
+ "theoretical_loss": 3.4232677999716445,
+ "tokens_seen": 2270035968
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002523410230692076,
+ "loss": 1.3718,
+ "theoretical_loss": 3.4232598304411774,
+ "tokens_seen": 2270101504
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002523249749247743,
+ "loss": 1.3387,
+ "theoretical_loss": 3.4232518612051996,
+ "tokens_seen": 2270167040
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000252308926780341,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4232438922636916,
+ "tokens_seen": 2270232576
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002522928786359077,
+ "loss": 1.2954,
+ "theoretical_loss": 3.423235923616634,
+ "tokens_seen": 2270298112
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025227683049147443,
+ "loss": 1.2451,
+ "theoretical_loss": 3.423227955264007,
+ "tokens_seen": 2270363648
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025226078234704113,
+ "loss": 1.3469,
+ "theoretical_loss": 3.4232199872057922,
+ "tokens_seen": 2270429184
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025224473420260783,
+ "loss": 1.117,
+ "theoretical_loss": 3.4232120194419693,
+ "tokens_seen": 2270494720
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025222868605817453,
+ "loss": 1.2371,
+ "theoretical_loss": 3.4232040519725193,
+ "tokens_seen": 2270560256
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025221263791374123,
+ "loss": 1.2156,
+ "theoretical_loss": 3.423196084797423,
+ "tokens_seen": 2270625792
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025219658976930793,
+ "loss": 1.2532,
+ "theoretical_loss": 3.423188117916661,
+ "tokens_seen": 2270691328
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025218054162487463,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4231801513302136,
+ "tokens_seen": 2270756864
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 759696,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3452606201171875,
+ "objective/train/theoretical_loss": 3.4231721850380614,
+ "objective/train/tokens_used": 2291282400,
+ "theoretical_loss": 3.4231721850380614,
+ "tokens_seen": 2270822400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025216449348044133,
+ "loss": 1.3798,
+ "theoretical_loss": 3.4231721850380614,
+ "tokens_seen": 2270822400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025214844533600803,
+ "loss": 1.1902,
+ "theoretical_loss": 3.423164219040186,
+ "tokens_seen": 2270887936
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025213239719157473,
+ "loss": 1.2556,
+ "theoretical_loss": 3.4231562533365665,
+ "tokens_seen": 2270953472
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025211634904714143,
+ "loss": 1.3508,
+ "theoretical_loss": 3.4231482879271846,
+ "tokens_seen": 2271019008
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025210030090270813,
+ "loss": 1.1938,
+ "theoretical_loss": 3.4231403228120207,
+ "tokens_seen": 2271084544
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025208425275827483,
+ "loss": 1.2114,
+ "theoretical_loss": 3.4231323579910553,
+ "tokens_seen": 2271150080
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025206820461384154,
+ "loss": 1.2922,
+ "theoretical_loss": 3.4231243934642697,
+ "tokens_seen": 2271215616
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025205215646940824,
+ "loss": 1.2219,
+ "theoretical_loss": 3.4231164292316434,
+ "tokens_seen": 2271281152
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025203610832497494,
+ "loss": 1.2933,
+ "theoretical_loss": 3.423108465293158,
+ "tokens_seen": 2271346688
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025202006018054164,
+ "loss": 1.3095,
+ "theoretical_loss": 3.423100501648794,
+ "tokens_seen": 2271412224
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025200401203610834,
+ "loss": 1.254,
+ "theoretical_loss": 3.4230925382985316,
+ "tokens_seen": 2271477760
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025198796389167504,
+ "loss": 1.2416,
+ "theoretical_loss": 3.4230845752423518,
+ "tokens_seen": 2271543296
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025197191574724174,
+ "loss": 1.2451,
+ "theoretical_loss": 3.4230766124802345,
+ "tokens_seen": 2271608832
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025195586760280844,
+ "loss": 1.3325,
+ "theoretical_loss": 3.423068650012162,
+ "tokens_seen": 2271674368
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025193981945837514,
+ "loss": 1.2924,
+ "theoretical_loss": 3.4230606878381136,
+ "tokens_seen": 2271739904
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025192377131394184,
+ "loss": 1.2111,
+ "theoretical_loss": 3.4230527259580703,
+ "tokens_seen": 2271805440
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025190772316950854,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4230447643720128,
+ "tokens_seen": 2271870976
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025189167502507524,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4230368030799223,
+ "tokens_seen": 2271936512
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025187562688064194,
+ "loss": 1.2232,
+ "theoretical_loss": 3.423028842081778,
+ "tokens_seen": 2272002048
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025185957873620864,
+ "loss": 1.305,
+ "theoretical_loss": 3.423020881377562,
+ "tokens_seen": 2272067584
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025184353059177534,
+ "loss": 1.2474,
+ "theoretical_loss": 3.423012920967255,
+ "tokens_seen": 2272133120
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025182748244734204,
+ "loss": 1.2309,
+ "theoretical_loss": 3.4230049608508364,
+ "tokens_seen": 2272198656
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025181143430290875,
+ "loss": 1.2731,
+ "theoretical_loss": 3.422997001028288,
+ "tokens_seen": 2272264192
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025179538615847545,
+ "loss": 1.2984,
+ "theoretical_loss": 3.42298904149959,
+ "tokens_seen": 2272329728
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025177933801404215,
+ "loss": 1.3255,
+ "theoretical_loss": 3.422981082264723,
+ "tokens_seen": 2272395264
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025176328986960885,
+ "loss": 1.206,
+ "theoretical_loss": 3.4229731233236684,
+ "tokens_seen": 2272460800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002517472417251755,
+ "loss": 1.1753,
+ "theoretical_loss": 3.422965164676406,
+ "tokens_seen": 2272526336
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002517311935807422,
+ "loss": 1.3235,
+ "theoretical_loss": 3.422957206322917,
+ "tokens_seen": 2272591872
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002517151454363089,
+ "loss": 1.1814,
+ "theoretical_loss": 3.4229492482631816,
+ "tokens_seen": 2272657408
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002516990972918756,
+ "loss": 1.2417,
+ "theoretical_loss": 3.4229412904971808,
+ "tokens_seen": 2272722944
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002516830491474423,
+ "loss": 1.295,
+ "theoretical_loss": 3.422933333024895,
+ "tokens_seen": 2272788480
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000251667001003009,
+ "loss": 1.2484,
+ "theoretical_loss": 3.4229253758463063,
+ "tokens_seen": 2272854016
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025165095285857575,
+ "loss": 1.2029,
+ "theoretical_loss": 3.4229174189613936,
+ "tokens_seen": 2272919552
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025163490471414245,
+ "loss": 1.2615,
+ "theoretical_loss": 3.422909462370138,
+ "tokens_seen": 2272985088
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025161885656970915,
+ "loss": 1.1895,
+ "theoretical_loss": 3.4229015060725207,
+ "tokens_seen": 2273050624
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025160280842527585,
+ "loss": 1.2615,
+ "theoretical_loss": 3.4228935500685225,
+ "tokens_seen": 2273116160
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025158676028084255,
+ "loss": 1.288,
+ "theoretical_loss": 3.422885594358123,
+ "tokens_seen": 2273181696
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025157071213640926,
+ "loss": 1.2891,
+ "theoretical_loss": 3.422877638941304,
+ "tokens_seen": 2273247232
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025155466399197596,
+ "loss": 1.3102,
+ "theoretical_loss": 3.422869683818046,
+ "tokens_seen": 2273312768
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025153861584754266,
+ "loss": 1.2433,
+ "theoretical_loss": 3.42286172898833,
+ "tokens_seen": 2273378304
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025152256770310936,
+ "loss": 1.2624,
+ "theoretical_loss": 3.4228537744521357,
+ "tokens_seen": 2273443840
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025150651955867606,
+ "loss": 1.2897,
+ "theoretical_loss": 3.4228458202094445,
+ "tokens_seen": 2273509376
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025149047141424276,
+ "loss": 1.2954,
+ "theoretical_loss": 3.422837866260237,
+ "tokens_seen": 2273574912
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025147442326980946,
+ "loss": 1.2628,
+ "theoretical_loss": 3.422829912604494,
+ "tokens_seen": 2273640448
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025145837512537616,
+ "loss": 1.3347,
+ "theoretical_loss": 3.422821959242196,
+ "tokens_seen": 2273705984
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025144232698094286,
+ "loss": 1.3089,
+ "theoretical_loss": 3.4228140061733234,
+ "tokens_seen": 2273771520
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025142627883650956,
+ "loss": 1.2281,
+ "theoretical_loss": 3.422806053397858,
+ "tokens_seen": 2273837056
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025141023069207626,
+ "loss": 1.2387,
+ "theoretical_loss": 3.4227981009157795,
+ "tokens_seen": 2273902592
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025139418254764296,
+ "loss": 1.2452,
+ "theoretical_loss": 3.4227901487270693,
+ "tokens_seen": 2273968128
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025137813440320966,
+ "loss": 1.3538,
+ "theoretical_loss": 3.4227821968317076,
+ "tokens_seen": 2274033664
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 761000,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.278111219406128,
+ "objective/train/theoretical_loss": 3.422774245229675,
+ "objective/train/tokens_used": 2294559200,
+ "theoretical_loss": 3.422774245229675,
+ "tokens_seen": 2274099200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025136208625877636,
+ "loss": 1.2351,
+ "theoretical_loss": 3.422774245229675,
+ "tokens_seen": 2274099200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025134603811434306,
+ "loss": 1.1902,
+ "theoretical_loss": 3.4227662939209536,
+ "tokens_seen": 2274164736
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025132998996990976,
+ "loss": 1.2983,
+ "theoretical_loss": 3.422758342905522,
+ "tokens_seen": 2274230272
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025131394182547647,
+ "loss": 1.3415,
+ "theoretical_loss": 3.422750392183363,
+ "tokens_seen": 2274295808
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025129789368104317,
+ "loss": 1.2504,
+ "theoretical_loss": 3.4227424417544556,
+ "tokens_seen": 2274361344
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025128184553660987,
+ "loss": 1.2615,
+ "theoretical_loss": 3.422734491618782,
+ "tokens_seen": 2274426880
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025126579739217657,
+ "loss": 1.2974,
+ "theoretical_loss": 3.4227265417763215,
+ "tokens_seen": 2274492416
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025124974924774327,
+ "loss": 1.2285,
+ "theoretical_loss": 3.422718592227056,
+ "tokens_seen": 2274557952
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025123370110330997,
+ "loss": 1.2849,
+ "theoretical_loss": 3.4227106429709657,
+ "tokens_seen": 2274623488
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025121765295887667,
+ "loss": 1.2496,
+ "theoretical_loss": 3.4227026940080316,
+ "tokens_seen": 2274689024
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025120160481444337,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4226947453382337,
+ "tokens_seen": 2274754560
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025118555667001007,
+ "loss": 1.2033,
+ "theoretical_loss": 3.4226867969615538,
+ "tokens_seen": 2274820096
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025116950852557677,
+ "loss": 1.1145,
+ "theoretical_loss": 3.4226788488779722,
+ "tokens_seen": 2274885632
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025115346038114347,
+ "loss": 1.2668,
+ "theoretical_loss": 3.4226709010874696,
+ "tokens_seen": 2274951168
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025113741223671017,
+ "loss": 1.2375,
+ "theoretical_loss": 3.422662953590027,
+ "tokens_seen": 2275016704
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002511213640922768,
+ "loss": 1.3184,
+ "theoretical_loss": 3.4226550063856247,
+ "tokens_seen": 2275082240
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002511053159478435,
+ "loss": 1.3255,
+ "theoretical_loss": 3.4226470594742437,
+ "tokens_seen": 2275147776
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002510892678034102,
+ "loss": 1.2455,
+ "theoretical_loss": 3.4226391128558644,
+ "tokens_seen": 2275213312
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002510732196589769,
+ "loss": 1.2343,
+ "theoretical_loss": 3.4226311665304685,
+ "tokens_seen": 2275278848
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002510571715145436,
+ "loss": 1.2811,
+ "theoretical_loss": 3.422623220498036,
+ "tokens_seen": 2275344384
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002510411233701103,
+ "loss": 1.1684,
+ "theoretical_loss": 3.4226152747585474,
+ "tokens_seen": 2275409920
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000251025075225677,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4226073293119845,
+ "tokens_seen": 2275475456
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002510090270812437,
+ "loss": 1.264,
+ "theoretical_loss": 3.4225993841583273,
+ "tokens_seen": 2275540992
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509929789368104,
+ "loss": 1.237,
+ "theoretical_loss": 3.4225914392975563,
+ "tokens_seen": 2275606528
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509769307923771,
+ "loss": 1.2718,
+ "theoretical_loss": 3.422583494729653,
+ "tokens_seen": 2275672064
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509608826479438,
+ "loss": 1.3219,
+ "theoretical_loss": 3.422575550454598,
+ "tokens_seen": 2275737600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509448345035105,
+ "loss": 1.2766,
+ "theoretical_loss": 3.422567606472372,
+ "tokens_seen": 2275803136
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509287863590772,
+ "loss": 1.2475,
+ "theoretical_loss": 3.4225596627829553,
+ "tokens_seen": 2275868672
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002509127382146439,
+ "loss": 1.1821,
+ "theoretical_loss": 3.422551719386329,
+ "tokens_seen": 2275934208
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025089669007021063,
+ "loss": 1.2273,
+ "theoretical_loss": 3.4225437762824744,
+ "tokens_seen": 2275999744
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025088064192577733,
+ "loss": 1.2221,
+ "theoretical_loss": 3.422535833471372,
+ "tokens_seen": 2276065280
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025086459378134403,
+ "loss": 1.1338,
+ "theoretical_loss": 3.4225278909530017,
+ "tokens_seen": 2276130816
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025084854563691073,
+ "loss": 1.2709,
+ "theoretical_loss": 3.4225199487273454,
+ "tokens_seen": 2276196352
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025083249749247743,
+ "loss": 1.2279,
+ "theoretical_loss": 3.4225120067943835,
+ "tokens_seen": 2276261888
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025081644934804413,
+ "loss": 1.2021,
+ "theoretical_loss": 3.4225040651540968,
+ "tokens_seen": 2276327424
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025080040120361083,
+ "loss": 1.1562,
+ "theoretical_loss": 3.422496123806466,
+ "tokens_seen": 2276392960
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025078435305917753,
+ "loss": 1.2187,
+ "theoretical_loss": 3.422488182751472,
+ "tokens_seen": 2276458496
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025076830491474423,
+ "loss": 1.2989,
+ "theoretical_loss": 3.4224802419890956,
+ "tokens_seen": 2276524032
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025075225677031093,
+ "loss": 1.2342,
+ "theoretical_loss": 3.422472301519317,
+ "tokens_seen": 2276589568
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025073620862587763,
+ "loss": 1.1945,
+ "theoretical_loss": 3.422464361342118,
+ "tokens_seen": 2276655104
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025072016048144433,
+ "loss": 1.2482,
+ "theoretical_loss": 3.422456421457479,
+ "tokens_seen": 2276720640
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025070411233701104,
+ "loss": 1.2803,
+ "theoretical_loss": 3.4224484818653806,
+ "tokens_seen": 2276786176
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025068806419257774,
+ "loss": 1.2759,
+ "theoretical_loss": 3.422440542565804,
+ "tokens_seen": 2276851712
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025067201604814444,
+ "loss": 1.2826,
+ "theoretical_loss": 3.42243260355873,
+ "tokens_seen": 2276917248
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025065596790371114,
+ "loss": 1.2774,
+ "theoretical_loss": 3.4224246648441383,
+ "tokens_seen": 2276982784
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025063991975927784,
+ "loss": 1.2701,
+ "theoretical_loss": 3.422416726422011,
+ "tokens_seen": 2277048320
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025062387161484454,
+ "loss": 1.1897,
+ "theoretical_loss": 3.422408788292328,
+ "tokens_seen": 2277113856
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025060782347041124,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4224008504550714,
+ "tokens_seen": 2277179392
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025059177532597794,
+ "loss": 1.184,
+ "theoretical_loss": 3.4223929129102206,
+ "tokens_seen": 2277244928
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025057572718154464,
+ "loss": 1.3507,
+ "theoretical_loss": 3.4223849756577573,
+ "tokens_seen": 2277310464
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 761954,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1957485675811768,
+ "objective/train/theoretical_loss": 3.422377038697662,
+ "objective/train/tokens_used": 2297836000,
+ "theoretical_loss": 3.422377038697662,
+ "tokens_seen": 2277376000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025055967903711134,
+ "loss": 1.2288,
+ "theoretical_loss": 3.422377038697662,
+ "tokens_seen": 2277376000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025054363089267804,
+ "loss": 1.3385,
+ "theoretical_loss": 3.422369102029915,
+ "tokens_seen": 2277441536
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025052758274824474,
+ "loss": 1.1808,
+ "theoretical_loss": 3.422361165654498,
+ "tokens_seen": 2277507072
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025051153460381144,
+ "loss": 1.1632,
+ "theoretical_loss": 3.422353229571392,
+ "tokens_seen": 2277572608
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025049548645937814,
+ "loss": 1.3477,
+ "theoretical_loss": 3.4223452937805767,
+ "tokens_seen": 2277638144
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025047943831494484,
+ "loss": 1.3277,
+ "theoretical_loss": 3.422337358282033,
+ "tokens_seen": 2277703680
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025046339017051154,
+ "loss": 1.2989,
+ "theoretical_loss": 3.4223294230757433,
+ "tokens_seen": 2277769216
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025044734202607825,
+ "loss": 1.3211,
+ "theoretical_loss": 3.422321488161687,
+ "tokens_seen": 2277834752
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025043129388164495,
+ "loss": 1.3001,
+ "theoretical_loss": 3.422313553539845,
+ "tokens_seen": 2277900288
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025041524573721165,
+ "loss": 1.259,
+ "theoretical_loss": 3.422305619210199,
+ "tokens_seen": 2277965824
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025039919759277835,
+ "loss": 1.2461,
+ "theoretical_loss": 3.4222976851727287,
+ "tokens_seen": 2278031360
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025038314944834505,
+ "loss": 1.25,
+ "theoretical_loss": 3.4222897514274155,
+ "tokens_seen": 2278096896
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025036710130391175,
+ "loss": 1.2492,
+ "theoretical_loss": 3.4222818179742407,
+ "tokens_seen": 2278162432
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025035105315947845,
+ "loss": 1.1898,
+ "theoretical_loss": 3.4222738848131846,
+ "tokens_seen": 2278227968
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025033500501504515,
+ "loss": 1.2585,
+ "theoretical_loss": 3.422265951944228,
+ "tokens_seen": 2278293504
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025031895687061185,
+ "loss": 1.1689,
+ "theoretical_loss": 3.4222580193673515,
+ "tokens_seen": 2278359040
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025030290872617855,
+ "loss": 1.1538,
+ "theoretical_loss": 3.422250087082537,
+ "tokens_seen": 2278424576
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025028686058174525,
+ "loss": 1.1711,
+ "theoretical_loss": 3.4222421550897644,
+ "tokens_seen": 2278490112
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025027081243731195,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4222342233890144,
+ "tokens_seen": 2278555648
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025025476429287865,
+ "loss": 1.1542,
+ "theoretical_loss": 3.422226291980269,
+ "tokens_seen": 2278621184
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025023871614844535,
+ "loss": 1.2361,
+ "theoretical_loss": 3.422218360863508,
+ "tokens_seen": 2278686720
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025022266800401205,
+ "loss": 1.2583,
+ "theoretical_loss": 3.422210430038712,
+ "tokens_seen": 2278752256
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025020661985957875,
+ "loss": 1.2362,
+ "theoretical_loss": 3.4222024995058633,
+ "tokens_seen": 2278817792
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025019057171514546,
+ "loss": 1.2091,
+ "theoretical_loss": 3.422194569264941,
+ "tokens_seen": 2278883328
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025017452357071216,
+ "loss": 1.2911,
+ "theoretical_loss": 3.4221866393159273,
+ "tokens_seen": 2278948864
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025015847542627886,
+ "loss": 1.2593,
+ "theoretical_loss": 3.422178709658803,
+ "tokens_seen": 2279014400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025014242728184556,
+ "loss": 1.2402,
+ "theoretical_loss": 3.422170780293548,
+ "tokens_seen": 2279079936
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025012637913741226,
+ "loss": 1.2509,
+ "theoretical_loss": 3.422162851220144,
+ "tokens_seen": 2279145472
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025011033099297896,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4221549224385717,
+ "tokens_seen": 2279211008
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025009428284854566,
+ "loss": 1.3057,
+ "theoretical_loss": 3.422146993948811,
+ "tokens_seen": 2279276544
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025007823470411236,
+ "loss": 1.3934,
+ "theoretical_loss": 3.4221390657508444,
+ "tokens_seen": 2279342080
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025006218655967906,
+ "loss": 1.2668,
+ "theoretical_loss": 3.422131137844652,
+ "tokens_seen": 2279407616
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025004613841524576,
+ "loss": 1.243,
+ "theoretical_loss": 3.4221232102302146,
+ "tokens_seen": 2279473152
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025003009027081246,
+ "loss": 1.227,
+ "theoretical_loss": 3.422115282907513,
+ "tokens_seen": 2279538688
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00025001404212637916,
+ "loss": 1.2628,
+ "theoretical_loss": 3.4221073558765283,
+ "tokens_seen": 2279604224
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024999799398194586,
+ "loss": 1.2096,
+ "theoretical_loss": 3.422099429137241,
+ "tokens_seen": 2279669760
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024998194583751256,
+ "loss": 1.1972,
+ "theoretical_loss": 3.422091502689633,
+ "tokens_seen": 2279735296
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024996589769307926,
+ "loss": 1.2533,
+ "theoretical_loss": 3.4220835765336837,
+ "tokens_seen": 2279800832
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024994984954864597,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4220756506693752,
+ "tokens_seen": 2279866368
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024993380140421267,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4220677250966878,
+ "tokens_seen": 2279931904
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024991775325977937,
+ "loss": 1.256,
+ "theoretical_loss": 3.4220597998156026,
+ "tokens_seen": 2279997440
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024990170511534607,
+ "loss": 1.1136,
+ "theoretical_loss": 3.4220518748261,
+ "tokens_seen": 2280062976
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024988565697091277,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4220439501281623,
+ "tokens_seen": 2280128512
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024986960882647947,
+ "loss": 1.2216,
+ "theoretical_loss": 3.4220360257217686,
+ "tokens_seen": 2280194048
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024985356068204617,
+ "loss": 1.2576,
+ "theoretical_loss": 3.4220281016069007,
+ "tokens_seen": 2280259584
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024983751253761287,
+ "loss": 1.2055,
+ "theoretical_loss": 3.4220201777835397,
+ "tokens_seen": 2280325120
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024982146439317957,
+ "loss": 1.1951,
+ "theoretical_loss": 3.422012254251666,
+ "tokens_seen": 2280390656
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024980541624874627,
+ "loss": 1.2612,
+ "theoretical_loss": 3.4220043310112604,
+ "tokens_seen": 2280456192
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024978936810431297,
+ "loss": 1.1299,
+ "theoretical_loss": 3.421996408062305,
+ "tokens_seen": 2280521728
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024977331995987967,
+ "loss": 1.4156,
+ "theoretical_loss": 3.421988485404779,
+ "tokens_seen": 2280587264
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 763124,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3014075756072998,
+ "objective/train/theoretical_loss": 3.421980563038664,
+ "objective/train/tokens_used": 2301112800,
+ "theoretical_loss": 3.421980563038664,
+ "tokens_seen": 2280652800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024975727181544637,
+ "loss": 1.3021,
+ "theoretical_loss": 3.421980563038664,
+ "tokens_seen": 2280652800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002497412236710131,
+ "loss": 1.2063,
+ "theoretical_loss": 3.4219726409639413,
+ "tokens_seen": 2280718336
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002497251755265797,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4219647191805915,
+ "tokens_seen": 2280783872
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002497091273821464,
+ "loss": 1.243,
+ "theoretical_loss": 3.421956797688596,
+ "tokens_seen": 2280849408
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496930792377131,
+ "loss": 1.207,
+ "theoretical_loss": 3.4219488764879347,
+ "tokens_seen": 2280914944
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496770310932798,
+ "loss": 1.2472,
+ "theoretical_loss": 3.421940955578589,
+ "tokens_seen": 2280980480
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496609829488465,
+ "loss": 1.3039,
+ "theoretical_loss": 3.42193303496054,
+ "tokens_seen": 2281046016
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496449348044132,
+ "loss": 1.2362,
+ "theoretical_loss": 3.4219251146337686,
+ "tokens_seen": 2281111552
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496288866599799,
+ "loss": 1.3183,
+ "theoretical_loss": 3.4219171945982554,
+ "tokens_seen": 2281177088
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002496128385155466,
+ "loss": 1.3315,
+ "theoretical_loss": 3.421909274853982,
+ "tokens_seen": 2281242624
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002495967903711133,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4219013554009283,
+ "tokens_seen": 2281308160
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024958074222668,
+ "loss": 1.3365,
+ "theoretical_loss": 3.4218934362390763,
+ "tokens_seen": 2281373696
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002495646940822467,
+ "loss": 1.2075,
+ "theoretical_loss": 3.4218855173684064,
+ "tokens_seen": 2281439232
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002495486459378134,
+ "loss": 1.1919,
+ "theoretical_loss": 3.421877598788899,
+ "tokens_seen": 2281504768
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024953259779338013,
+ "loss": 1.2921,
+ "theoretical_loss": 3.421869680500536,
+ "tokens_seen": 2281570304
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024951654964894683,
+ "loss": 1.3631,
+ "theoretical_loss": 3.421861762503298,
+ "tokens_seen": 2281635840
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024950050150451353,
+ "loss": 1.2799,
+ "theoretical_loss": 3.4218538447971656,
+ "tokens_seen": 2281701376
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024948445336008023,
+ "loss": 1.1703,
+ "theoretical_loss": 3.4218459273821202,
+ "tokens_seen": 2281766912
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024946840521564693,
+ "loss": 1.1603,
+ "theoretical_loss": 3.4218380102581425,
+ "tokens_seen": 2281832448
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024945235707121363,
+ "loss": 1.2096,
+ "theoretical_loss": 3.4218300934252133,
+ "tokens_seen": 2281897984
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024943630892678033,
+ "loss": 1.3404,
+ "theoretical_loss": 3.421822176883314,
+ "tokens_seen": 2281963520
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024942026078234703,
+ "loss": 1.2757,
+ "theoretical_loss": 3.421814260632425,
+ "tokens_seen": 2282029056
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024940421263791373,
+ "loss": 1.2191,
+ "theoretical_loss": 3.421806344672527,
+ "tokens_seen": 2282094592
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024938816449348043,
+ "loss": 1.27,
+ "theoretical_loss": 3.4217984290036023,
+ "tokens_seen": 2282160128
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024937211634904713,
+ "loss": 1.3596,
+ "theoretical_loss": 3.4217905136256306,
+ "tokens_seen": 2282225664
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024935606820461383,
+ "loss": 1.303,
+ "theoretical_loss": 3.4217825985385932,
+ "tokens_seen": 2282291200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024934002006018053,
+ "loss": 1.289,
+ "theoretical_loss": 3.4217746837424716,
+ "tokens_seen": 2282356736
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024932397191574724,
+ "loss": 1.3012,
+ "theoretical_loss": 3.4217667692372458,
+ "tokens_seen": 2282422272
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024930792377131394,
+ "loss": 1.201,
+ "theoretical_loss": 3.421758855022897,
+ "tokens_seen": 2282487808
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024929187562688064,
+ "loss": 1.2263,
+ "theoretical_loss": 3.4217509410994067,
+ "tokens_seen": 2282553344
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024927582748244734,
+ "loss": 1.1863,
+ "theoretical_loss": 3.4217430274667557,
+ "tokens_seen": 2282618880
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024925977933801404,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4217351141249246,
+ "tokens_seen": 2282684416
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002492437311935808,
+ "loss": 1.1624,
+ "theoretical_loss": 3.421727201073894,
+ "tokens_seen": 2282749952
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002492276830491475,
+ "loss": 1.2393,
+ "theoretical_loss": 3.4217192883136462,
+ "tokens_seen": 2282815488
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002492116349047142,
+ "loss": 1.1816,
+ "theoretical_loss": 3.4217113758441613,
+ "tokens_seen": 2282881024
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002491955867602809,
+ "loss": 1.1797,
+ "theoretical_loss": 3.42170346366542,
+ "tokens_seen": 2282946560
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002491795386158476,
+ "loss": 1.1957,
+ "theoretical_loss": 3.421695551777404,
+ "tokens_seen": 2283012096
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002491634904714143,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4216876401800933,
+ "tokens_seen": 2283077632
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000249147442326981,
+ "loss": 1.333,
+ "theoretical_loss": 3.42167972887347,
+ "tokens_seen": 2283143168
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002491313941825477,
+ "loss": 1.2469,
+ "theoretical_loss": 3.4216718178575145,
+ "tokens_seen": 2283208704
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002491153460381144,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4216639071322077,
+ "tokens_seen": 2283274240
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002490992978936811,
+ "loss": 1.1901,
+ "theoretical_loss": 3.4216559966975306,
+ "tokens_seen": 2283339776
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024908324974924775,
+ "loss": 1.287,
+ "theoretical_loss": 3.4216480865534646,
+ "tokens_seen": 2283405312
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024906720160481445,
+ "loss": 1.3362,
+ "theoretical_loss": 3.42164017669999,
+ "tokens_seen": 2283470848
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024905115346038115,
+ "loss": 1.2318,
+ "theoretical_loss": 3.4216322671370882,
+ "tokens_seen": 2283536384
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024903510531594785,
+ "loss": 1.2531,
+ "theoretical_loss": 3.4216243578647405,
+ "tokens_seen": 2283601920
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024901905717151455,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4216164488829275,
+ "tokens_seen": 2283667456
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024900300902708125,
+ "loss": 1.2873,
+ "theoretical_loss": 3.42160854019163,
+ "tokens_seen": 2283732992
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024898696088264795,
+ "loss": 1.3486,
+ "theoretical_loss": 3.421600631790829,
+ "tokens_seen": 2283798528
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024897091273821465,
+ "loss": 1.2556,
+ "theoretical_loss": 3.421592723680506,
+ "tokens_seen": 2283864064
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 764062,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2559454441070557,
+ "objective/train/theoretical_loss": 3.421584815860642,
+ "objective/train/tokens_used": 2304389600,
+ "theoretical_loss": 3.421584815860642,
+ "tokens_seen": 2283929600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024895486459378135,
+ "loss": 1.3086,
+ "theoretical_loss": 3.421584815860642,
+ "tokens_seen": 2283929600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024893881644934805,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4215769083312173,
+ "tokens_seen": 2283995136
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024892276830491475,
+ "loss": 1.275,
+ "theoretical_loss": 3.4215690010922133,
+ "tokens_seen": 2284060672
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024890672016048145,
+ "loss": 1.2679,
+ "theoretical_loss": 3.4215610941436108,
+ "tokens_seen": 2284126208
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024889067201604815,
+ "loss": 1.1957,
+ "theoretical_loss": 3.4215531874853915,
+ "tokens_seen": 2284191744
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024887462387161485,
+ "loss": 1.1776,
+ "theoretical_loss": 3.4215452811175355,
+ "tokens_seen": 2284257280
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024885857572718155,
+ "loss": 1.2411,
+ "theoretical_loss": 3.4215373750400246,
+ "tokens_seen": 2284322816
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024884252758274825,
+ "loss": 1.1065,
+ "theoretical_loss": 3.421529469252839,
+ "tokens_seen": 2284388352
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024882647943831496,
+ "loss": 1.3828,
+ "theoretical_loss": 3.4215215637559604,
+ "tokens_seen": 2284453888
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024881043129388166,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4215136585493697,
+ "tokens_seen": 2284519424
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024879438314944836,
+ "loss": 1.3173,
+ "theoretical_loss": 3.4215057536330473,
+ "tokens_seen": 2284584960
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024877833500501506,
+ "loss": 1.2537,
+ "theoretical_loss": 3.421497849006975,
+ "tokens_seen": 2284650496
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024876228686058176,
+ "loss": 1.269,
+ "theoretical_loss": 3.4214899446711335,
+ "tokens_seen": 2284716032
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024874623871614846,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4214820406255035,
+ "tokens_seen": 2284781568
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024873019057171516,
+ "loss": 1.1967,
+ "theoretical_loss": 3.4214741368700663,
+ "tokens_seen": 2284847104
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024871414242728186,
+ "loss": 1.2141,
+ "theoretical_loss": 3.4214662334048036,
+ "tokens_seen": 2284912640
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024869809428284856,
+ "loss": 1.2936,
+ "theoretical_loss": 3.4214583302296955,
+ "tokens_seen": 2284978176
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024868204613841526,
+ "loss": 1.1737,
+ "theoretical_loss": 3.421450427344723,
+ "tokens_seen": 2285043712
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024866599799398196,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4214425247498674,
+ "tokens_seen": 2285109248
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024864994984954866,
+ "loss": 1.3074,
+ "theoretical_loss": 3.42143462244511,
+ "tokens_seen": 2285174784
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024863390170511536,
+ "loss": 1.2719,
+ "theoretical_loss": 3.4214267204304316,
+ "tokens_seen": 2285240320
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024861785356068206,
+ "loss": 1.2865,
+ "theoretical_loss": 3.4214188187058134,
+ "tokens_seen": 2285305856
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024860180541624876,
+ "loss": 1.2344,
+ "theoretical_loss": 3.421410917271236,
+ "tokens_seen": 2285371392
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024858575727181546,
+ "loss": 1.3586,
+ "theoretical_loss": 3.4214030161266806,
+ "tokens_seen": 2285436928
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024856970912738217,
+ "loss": 1.1407,
+ "theoretical_loss": 3.4213951152721287,
+ "tokens_seen": 2285502464
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024855366098294887,
+ "loss": 1.2684,
+ "theoretical_loss": 3.421387214707561,
+ "tokens_seen": 2285568000
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024853761283851557,
+ "loss": 1.2,
+ "theoretical_loss": 3.421379314432958,
+ "tokens_seen": 2285633536
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024852156469408227,
+ "loss": 1.3126,
+ "theoretical_loss": 3.4213714144483016,
+ "tokens_seen": 2285699072
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024850551654964897,
+ "loss": 1.2138,
+ "theoretical_loss": 3.4213635147535726,
+ "tokens_seen": 2285764608
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024848946840521567,
+ "loss": 1.2638,
+ "theoretical_loss": 3.4213556153487517,
+ "tokens_seen": 2285830144
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024847342026078237,
+ "loss": 1.2949,
+ "theoretical_loss": 3.42134771623382,
+ "tokens_seen": 2285895680
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024845737211634907,
+ "loss": 1.2191,
+ "theoretical_loss": 3.4213398174087595,
+ "tokens_seen": 2285961216
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024844132397191577,
+ "loss": 1.1829,
+ "theoretical_loss": 3.42133191887355,
+ "tokens_seen": 2286026752
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024842527582748247,
+ "loss": 1.2199,
+ "theoretical_loss": 3.421324020628173,
+ "tokens_seen": 2286092288
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024840922768304917,
+ "loss": 1.1818,
+ "theoretical_loss": 3.42131612267261,
+ "tokens_seen": 2286157824
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024839317953861587,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4213082250068414,
+ "tokens_seen": 2286223360
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002483771313941826,
+ "loss": 1.2637,
+ "theoretical_loss": 3.4213003276308487,
+ "tokens_seen": 2286288896
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002483610832497493,
+ "loss": 1.2902,
+ "theoretical_loss": 3.4212924305446126,
+ "tokens_seen": 2286354432
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000248345035105316,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4212845337481146,
+ "tokens_seen": 2286419968
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002483289869608827,
+ "loss": 1.3549,
+ "theoretical_loss": 3.4212766372413355,
+ "tokens_seen": 2286485504
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002483129388164493,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4212687410242566,
+ "tokens_seen": 2286551040
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000248296890672016,
+ "loss": 1.2563,
+ "theoretical_loss": 3.421260845096858,
+ "tokens_seen": 2286616576
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482808425275827,
+ "loss": 1.2451,
+ "theoretical_loss": 3.4212529494591224,
+ "tokens_seen": 2286682112
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482647943831494,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4212450541110293,
+ "tokens_seen": 2286747648
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482487462387161,
+ "loss": 1.1951,
+ "theoretical_loss": 3.421237159052561,
+ "tokens_seen": 2286813184
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482326980942828,
+ "loss": 1.2896,
+ "theoretical_loss": 3.421229264283698,
+ "tokens_seen": 2286878720
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482326980942828,
+ "loss": 1.1929,
+ "theoretical_loss": 3.421221369804421,
+ "tokens_seen": 2286944256
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482166499498495,
+ "loss": 1.1349,
+ "theoretical_loss": 3.421213475614712,
+ "tokens_seen": 2287009792
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002482006018054162,
+ "loss": 1.298,
+ "theoretical_loss": 3.4212055817145517,
+ "tokens_seen": 2287075328
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002481845536609829,
+ "loss": 1.2029,
+ "theoretical_loss": 3.4211976881039208,
+ "tokens_seen": 2287140864
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 765245,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3369064331054688,
+ "objective/train/theoretical_loss": 3.4211897947828005,
+ "objective/train/tokens_used": 2307666400,
+ "theoretical_loss": 3.4211897947828005,
+ "tokens_seen": 2287206400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002481685055165496,
+ "loss": 1.405,
+ "theoretical_loss": 3.4211897947828005,
+ "tokens_seen": 2287206400
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024815245737211633,
+ "loss": 1.2553,
+ "theoretical_loss": 3.421181901751172,
+ "tokens_seen": 2287271936
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024813640922768303,
+ "loss": 1.238,
+ "theoretical_loss": 3.4211740090090172,
+ "tokens_seen": 2287337472
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024812036108324973,
+ "loss": 1.252,
+ "theoretical_loss": 3.4211661165563156,
+ "tokens_seen": 2287403008
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024810431293881643,
+ "loss": 1.3089,
+ "theoretical_loss": 3.4211582243930496,
+ "tokens_seen": 2287468544
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024808826479438313,
+ "loss": 1.2312,
+ "theoretical_loss": 3.4211503325191996,
+ "tokens_seen": 2287534080
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024807221664994983,
+ "loss": 1.2604,
+ "theoretical_loss": 3.421142440934747,
+ "tokens_seen": 2287599616
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024805616850551653,
+ "loss": 1.2303,
+ "theoretical_loss": 3.421134549639673,
+ "tokens_seen": 2287665152
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002480401203610833,
+ "loss": 1.2721,
+ "theoretical_loss": 3.421126658633958,
+ "tokens_seen": 2287730688
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024802407221665,
+ "loss": 1.1785,
+ "theoretical_loss": 3.421118767917584,
+ "tokens_seen": 2287796224
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002480080240722167,
+ "loss": 1.3106,
+ "theoretical_loss": 3.421110877490532,
+ "tokens_seen": 2287861760
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479919759277834,
+ "loss": 1.1664,
+ "theoretical_loss": 3.421102987352782,
+ "tokens_seen": 2287927296
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479759277833501,
+ "loss": 1.2783,
+ "theoretical_loss": 3.421095097504317,
+ "tokens_seen": 2287992832
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479598796389168,
+ "loss": 1.2455,
+ "theoretical_loss": 3.4210872079451162,
+ "tokens_seen": 2288058368
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479438314944835,
+ "loss": 1.1549,
+ "theoretical_loss": 3.4210793186751616,
+ "tokens_seen": 2288123904
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479277833500502,
+ "loss": 1.1562,
+ "theoretical_loss": 3.4210714296944347,
+ "tokens_seen": 2288189440
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002479117352056169,
+ "loss": 1.2328,
+ "theoretical_loss": 3.421063541002916,
+ "tokens_seen": 2288254976
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002478956870611836,
+ "loss": 1.2258,
+ "theoretical_loss": 3.4210556526005864,
+ "tokens_seen": 2288320512
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002478796389167503,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4210477644874278,
+ "tokens_seen": 2288386048
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000247863590772317,
+ "loss": 1.1928,
+ "theoretical_loss": 3.421039876663421,
+ "tokens_seen": 2288451584
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002478475426278837,
+ "loss": 1.2939,
+ "theoretical_loss": 3.421031989128547,
+ "tokens_seen": 2288517120
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002478314944834504,
+ "loss": 1.1944,
+ "theoretical_loss": 3.4210241018827867,
+ "tokens_seen": 2288582656
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002478154463390171,
+ "loss": 1.2884,
+ "theoretical_loss": 3.421016214926121,
+ "tokens_seen": 2288648192
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477993981945838,
+ "loss": 1.216,
+ "theoretical_loss": 3.4210083282585324,
+ "tokens_seen": 2288713728
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477833500501505,
+ "loss": 1.323,
+ "theoretical_loss": 3.421000441880001,
+ "tokens_seen": 2288779264
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477673019057172,
+ "loss": 1.234,
+ "theoretical_loss": 3.420992555790508,
+ "tokens_seen": 2288844800
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477512537612839,
+ "loss": 1.1943,
+ "theoretical_loss": 3.4209846699900344,
+ "tokens_seen": 2288910336
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477352056168506,
+ "loss": 1.2016,
+ "theoretical_loss": 3.4209767844785617,
+ "tokens_seen": 2288975872
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002477191574724173,
+ "loss": 1.2223,
+ "theoretical_loss": 3.4209688992560707,
+ "tokens_seen": 2289041408
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000247703109327984,
+ "loss": 1.3215,
+ "theoretical_loss": 3.4209610143225433,
+ "tokens_seen": 2289106944
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024768706118355065,
+ "loss": 1.1826,
+ "theoretical_loss": 3.4209531296779594,
+ "tokens_seen": 2289172480
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024767101303911735,
+ "loss": 1.2606,
+ "theoretical_loss": 3.420945245322301,
+ "tokens_seen": 2289238016
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024765496489468405,
+ "loss": 1.3239,
+ "theoretical_loss": 3.420937361255549,
+ "tokens_seen": 2289303552
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024763891675025075,
+ "loss": 1.335,
+ "theoretical_loss": 3.4209294774776846,
+ "tokens_seen": 2289369088
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024762286860581745,
+ "loss": 1.3064,
+ "theoretical_loss": 3.420921593988689,
+ "tokens_seen": 2289434624
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024760682046138415,
+ "loss": 1.2655,
+ "theoretical_loss": 3.420913710788543,
+ "tokens_seen": 2289500160
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024759077231695085,
+ "loss": 1.3181,
+ "theoretical_loss": 3.420905827877228,
+ "tokens_seen": 2289565696
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024757472417251755,
+ "loss": 1.2623,
+ "theoretical_loss": 3.4208979452547252,
+ "tokens_seen": 2289631232
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024755867602808425,
+ "loss": 1.187,
+ "theoretical_loss": 3.420890062921016,
+ "tokens_seen": 2289696768
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024754262788365095,
+ "loss": 1.2983,
+ "theoretical_loss": 3.420882180876081,
+ "tokens_seen": 2289762304
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024752657973921765,
+ "loss": 1.2948,
+ "theoretical_loss": 3.4208742991199017,
+ "tokens_seen": 2289827840
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024751053159478435,
+ "loss": 1.3701,
+ "theoretical_loss": 3.4208664176524595,
+ "tokens_seen": 2289893376
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024749448345035105,
+ "loss": 1.3513,
+ "theoretical_loss": 3.4208585364737347,
+ "tokens_seen": 2289958912
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024747843530591775,
+ "loss": 1.1207,
+ "theoretical_loss": 3.4208506555837093,
+ "tokens_seen": 2290024448
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024746238716148445,
+ "loss": 1.3078,
+ "theoretical_loss": 3.420842774982364,
+ "tokens_seen": 2290089984
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024744633901705116,
+ "loss": 1.264,
+ "theoretical_loss": 3.4208348946696803,
+ "tokens_seen": 2290155520
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024743029087261786,
+ "loss": 1.3753,
+ "theoretical_loss": 3.420827014645639,
+ "tokens_seen": 2290221056
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024741424272818456,
+ "loss": 1.2255,
+ "theoretical_loss": 3.4208191349102215,
+ "tokens_seen": 2290286592
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024739819458375126,
+ "loss": 1.296,
+ "theoretical_loss": 3.420811255463409,
+ "tokens_seen": 2290352128
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024738214643931796,
+ "loss": 1.2068,
+ "theoretical_loss": 3.420803376305183,
+ "tokens_seen": 2290417664
+ },
+ {
+ "epoch": 0.69,
+ "objective/train/docs_used": 766171,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2719933986663818,
+ "objective/train/theoretical_loss": 3.420795497435524,
+ "objective/train/tokens_used": 2310943200,
+ "theoretical_loss": 3.420795497435524,
+ "tokens_seen": 2290483200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024736609829488466,
+ "loss": 1.2678,
+ "theoretical_loss": 3.420795497435524,
+ "tokens_seen": 2290483200
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024735005015045136,
+ "loss": 1.2002,
+ "theoretical_loss": 3.420787618854413,
+ "tokens_seen": 2290548736
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024733400200601806,
+ "loss": 1.2135,
+ "theoretical_loss": 3.420779740561832,
+ "tokens_seen": 2290614272
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024731795386158476,
+ "loss": 1.2334,
+ "theoretical_loss": 3.420771862557762,
+ "tokens_seen": 2290679808
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024730190571715146,
+ "loss": 1.1837,
+ "theoretical_loss": 3.420763984842184,
+ "tokens_seen": 2290745344
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024728585757271816,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4207561074150785,
+ "tokens_seen": 2290810880
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024726980942828486,
+ "loss": 1.2231,
+ "theoretical_loss": 3.420748230276428,
+ "tokens_seen": 2290876416
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024725376128385156,
+ "loss": 1.2383,
+ "theoretical_loss": 3.420740353426213,
+ "tokens_seen": 2290941952
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024723771313941826,
+ "loss": 1.2755,
+ "theoretical_loss": 3.4207324768644147,
+ "tokens_seen": 2291007488
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024722166499498496,
+ "loss": 1.3361,
+ "theoretical_loss": 3.420724600591014,
+ "tokens_seen": 2291073024
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024720561685055167,
+ "loss": 1.0879,
+ "theoretical_loss": 3.4207167246059926,
+ "tokens_seen": 2291138560
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024718956870611837,
+ "loss": 1.1967,
+ "theoretical_loss": 3.4207088489093316,
+ "tokens_seen": 2291204096
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024717352056168507,
+ "loss": 1.2709,
+ "theoretical_loss": 3.4207009735010123,
+ "tokens_seen": 2291269632
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024715747241725177,
+ "loss": 1.297,
+ "theoretical_loss": 3.4206930983810153,
+ "tokens_seen": 2291335168
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024714142427281847,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4206852235493224,
+ "tokens_seen": 2291400704
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024712537612838517,
+ "loss": 1.1992,
+ "theoretical_loss": 3.4206773490059144,
+ "tokens_seen": 2291466240
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024710932798395187,
+ "loss": 1.1943,
+ "theoretical_loss": 3.4206694747507727,
+ "tokens_seen": 2291531776
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024709327983951857,
+ "loss": 1.193,
+ "theoretical_loss": 3.4206616007838786,
+ "tokens_seen": 2291597312
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024707723169508527,
+ "loss": 1.228,
+ "theoretical_loss": 3.420653727105213,
+ "tokens_seen": 2291662848
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024706118355065197,
+ "loss": 1.251,
+ "theoretical_loss": 3.420645853714758,
+ "tokens_seen": 2291728384
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024704513540621867,
+ "loss": 1.3244,
+ "theoretical_loss": 3.4206379806124936,
+ "tokens_seen": 2291793920
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024702908726178537,
+ "loss": 1.2594,
+ "theoretical_loss": 3.420630107798402,
+ "tokens_seen": 2291859456
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.00024701303911735207,
+ "loss": 1.2738,
+ "theoretical_loss": 3.420622235272463,
+ "tokens_seen": 2291924992
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469969909729188,
+ "loss": 1.211,
+ "theoretical_loss": 3.4206143630346597,
+ "tokens_seen": 2291990528
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469809428284855,
+ "loss": 1.1404,
+ "theoretical_loss": 3.4206064910849716,
+ "tokens_seen": 2292056064
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469648946840522,
+ "loss": 1.3157,
+ "theoretical_loss": 3.420598619423381,
+ "tokens_seen": 2292121600
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469488465396189,
+ "loss": 1.2794,
+ "theoretical_loss": 3.420590748049869,
+ "tokens_seen": 2292187136
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469327983951856,
+ "loss": 1.1263,
+ "theoretical_loss": 3.4205828769644167,
+ "tokens_seen": 2292252672
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469167502507522,
+ "loss": 1.304,
+ "theoretical_loss": 3.420575006167005,
+ "tokens_seen": 2292318208
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002469007021063189,
+ "loss": 1.2076,
+ "theoretical_loss": 3.4205671356576155,
+ "tokens_seen": 2292383744
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002468846539618856,
+ "loss": 1.2339,
+ "theoretical_loss": 3.4205592654362293,
+ "tokens_seen": 2292449280
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002468686058174523,
+ "loss": 1.2402,
+ "theoretical_loss": 3.4205513955028275,
+ "tokens_seen": 2292514816
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000246852557673019,
+ "loss": 1.319,
+ "theoretical_loss": 3.4205435258573917,
+ "tokens_seen": 2292580352
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002468365095285858,
+ "loss": 1.228,
+ "theoretical_loss": 3.4205356564999025,
+ "tokens_seen": 2292645888
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002468204613841525,
+ "loss": 1.3041,
+ "theoretical_loss": 3.420527787430342,
+ "tokens_seen": 2292711424
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002468044132397192,
+ "loss": 1.2073,
+ "theoretical_loss": 3.420519918648691,
+ "tokens_seen": 2292776960
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002467883650952859,
+ "loss": 1.0901,
+ "theoretical_loss": 3.4205120501549304,
+ "tokens_seen": 2292842496
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002467723169508526,
+ "loss": 1.2486,
+ "theoretical_loss": 3.420504181949042,
+ "tokens_seen": 2292908032
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002467562688064193,
+ "loss": 1.1366,
+ "theoretical_loss": 3.420496314031006,
+ "tokens_seen": 2292973568
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.000246740220661986,
+ "loss": 1.2153,
+ "theoretical_loss": 3.420488446400805,
+ "tokens_seen": 2293039104
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002467241725175527,
+ "loss": 1.249,
+ "theoretical_loss": 3.42048057905842,
+ "tokens_seen": 2293104640
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002467081243731194,
+ "loss": 1.2397,
+ "theoretical_loss": 3.420472712003831,
+ "tokens_seen": 2293170176
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002466920762286861,
+ "loss": 1.1215,
+ "theoretical_loss": 3.420464845237021,
+ "tokens_seen": 2293235712
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002466760280842528,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4204569787579704,
+ "tokens_seen": 2293301248
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002466599799398195,
+ "loss": 1.3926,
+ "theoretical_loss": 3.4204491125666605,
+ "tokens_seen": 2293366784
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002466439317953862,
+ "loss": 1.2673,
+ "theoretical_loss": 3.420441246663072,
+ "tokens_seen": 2293432320
+ },
+ {
+ "epoch": 0.69,
+ "learning_rate": 0.0002466278836509529,
+ "loss": 1.2498,
+ "theoretical_loss": 3.4204333810471867,
+ "tokens_seen": 2293497856
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002466118355065196,
+ "loss": 1.1075,
+ "theoretical_loss": 3.4204255157189865,
+ "tokens_seen": 2293563392
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002465957873620863,
+ "loss": 1.3213,
+ "theoretical_loss": 3.420417650678451,
+ "tokens_seen": 2293628928
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000246579739217653,
+ "loss": 1.3007,
+ "theoretical_loss": 3.420409785925563,
+ "tokens_seen": 2293694464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.7368157089654065,
+ "debugging/entropy-1-grams": 5.260448350002519,
+ "debugging/length": 548.8333333333334,
+ "debugging/num_segments": 12,
+ "debugging/score": 0.05299388900931896,
+ "debugging/score_std": 0.02491763752851157,
+ "epoch": 0.7,
+ "objective/train/docs_used": 767417,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2914656400680542,
+ "objective/train/theoretical_loss": 3.4204019214603036,
+ "objective/train/tokens_used": 2314220000,
+ "theoretical_loss": 3.4204019214603036,
+ "tokens_seen": 2293760000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002465636910732197,
+ "loss": 1.2804,
+ "theoretical_loss": 3.4204019214603036,
+ "tokens_seen": 2293760000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002465476429287864,
+ "loss": 1.0869,
+ "theoretical_loss": 3.420394057282653,
+ "tokens_seen": 2293825536
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002465315947843531,
+ "loss": 1.1691,
+ "theoretical_loss": 3.4203861933925936,
+ "tokens_seen": 2293891072
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002465155466399198,
+ "loss": 1.2505,
+ "theoretical_loss": 3.420378329790106,
+ "tokens_seen": 2293956608
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464994984954865,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4203704664751715,
+ "tokens_seen": 2294022144
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464834503510532,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4203626034477717,
+ "tokens_seen": 2294087680
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464674022066199,
+ "loss": 1.1889,
+ "theoretical_loss": 3.420354740707888,
+ "tokens_seen": 2294153216
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464513540621866,
+ "loss": 1.1673,
+ "theoretical_loss": 3.420346878255501,
+ "tokens_seen": 2294218752
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464353059177533,
+ "loss": 1.175,
+ "theoretical_loss": 3.4203390160905927,
+ "tokens_seen": 2294284288
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024641925777332,
+ "loss": 1.1598,
+ "theoretical_loss": 3.4203311542131436,
+ "tokens_seen": 2294349824
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002464032096288867,
+ "loss": 1.3002,
+ "theoretical_loss": 3.4203232926231357,
+ "tokens_seen": 2294415360
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463871614844534,
+ "loss": 1.2029,
+ "theoretical_loss": 3.42031543132055,
+ "tokens_seen": 2294480896
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463711133400201,
+ "loss": 1.1759,
+ "theoretical_loss": 3.4203075703053676,
+ "tokens_seen": 2294546432
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463550651955868,
+ "loss": 1.2217,
+ "theoretical_loss": 3.42029970957757,
+ "tokens_seen": 2294611968
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463390170511535,
+ "loss": 1.2513,
+ "theoretical_loss": 3.4202918491371386,
+ "tokens_seen": 2294677504
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463229689067202,
+ "loss": 1.2122,
+ "theoretical_loss": 3.4202839889840546,
+ "tokens_seen": 2294743040
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002463069207622869,
+ "loss": 1.2356,
+ "theoretical_loss": 3.420276129118299,
+ "tokens_seen": 2294808576
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024629087261785355,
+ "loss": 1.2523,
+ "theoretical_loss": 3.4202682695398536,
+ "tokens_seen": 2294874112
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024627482447342025,
+ "loss": 1.1966,
+ "theoretical_loss": 3.4202604102486993,
+ "tokens_seen": 2294939648
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024625877632898695,
+ "loss": 1.2442,
+ "theoretical_loss": 3.4202525512448174,
+ "tokens_seen": 2295005184
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024624272818455365,
+ "loss": 1.299,
+ "theoretical_loss": 3.4202446925281893,
+ "tokens_seen": 2295070720
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024622668004012035,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4202368340987963,
+ "tokens_seen": 2295136256
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024621063189568705,
+ "loss": 1.1848,
+ "theoretical_loss": 3.42022897595662,
+ "tokens_seen": 2295201792
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024619458375125375,
+ "loss": 1.2692,
+ "theoretical_loss": 3.420221118101641,
+ "tokens_seen": 2295267328
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024617853560682045,
+ "loss": 1.2605,
+ "theoretical_loss": 3.420213260533841,
+ "tokens_seen": 2295332864
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024616248746238715,
+ "loss": 1.2786,
+ "theoretical_loss": 3.420205403253202,
+ "tokens_seen": 2295398400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024614643931795385,
+ "loss": 1.1801,
+ "theoretical_loss": 3.420197546259704,
+ "tokens_seen": 2295463936
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024613039117352055,
+ "loss": 1.3359,
+ "theoretical_loss": 3.4201896895533292,
+ "tokens_seen": 2295529472
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024611434302908725,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4201818331340585,
+ "tokens_seen": 2295595008
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024609829488465395,
+ "loss": 1.2212,
+ "theoretical_loss": 3.4201739770018733,
+ "tokens_seen": 2295660544
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024608224674022066,
+ "loss": 1.2301,
+ "theoretical_loss": 3.4201661211567553,
+ "tokens_seen": 2295726080
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024606619859578736,
+ "loss": 1.188,
+ "theoretical_loss": 3.420158265598685,
+ "tokens_seen": 2295791616
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024605015045135406,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4201504103276448,
+ "tokens_seen": 2295857152
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024603410230692076,
+ "loss": 1.1569,
+ "theoretical_loss": 3.420142555343615,
+ "tokens_seen": 2295922688
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024601805416248746,
+ "loss": 1.1837,
+ "theoretical_loss": 3.420134700646577,
+ "tokens_seen": 2295988224
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024600200601805416,
+ "loss": 1.2489,
+ "theoretical_loss": 3.4201268462365135,
+ "tokens_seen": 2296053760
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024598595787362086,
+ "loss": 1.2533,
+ "theoretical_loss": 3.4201189921134043,
+ "tokens_seen": 2296119296
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024596990972918756,
+ "loss": 1.2695,
+ "theoretical_loss": 3.420111138277231,
+ "tokens_seen": 2296184832
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024595386158475426,
+ "loss": 1.4606,
+ "theoretical_loss": 3.420103284727975,
+ "tokens_seen": 2296250368
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024593781344032096,
+ "loss": 1.1763,
+ "theoretical_loss": 3.4200954314656182,
+ "tokens_seen": 2296315904
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024592176529588766,
+ "loss": 1.2866,
+ "theoretical_loss": 3.4200875784901417,
+ "tokens_seen": 2296381440
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024590571715145436,
+ "loss": 1.1622,
+ "theoretical_loss": 3.420079725801526,
+ "tokens_seen": 2296446976
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024588966900702106,
+ "loss": 1.231,
+ "theoretical_loss": 3.4200718733997535,
+ "tokens_seen": 2296512512
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024587362086258776,
+ "loss": 1.1985,
+ "theoretical_loss": 3.420064021284805,
+ "tokens_seen": 2296578048
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024585757271815446,
+ "loss": 1.3075,
+ "theoretical_loss": 3.420056169456662,
+ "tokens_seen": 2296643584
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024584152457372116,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4200483179153056,
+ "tokens_seen": 2296709120
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024582547642928787,
+ "loss": 1.246,
+ "theoretical_loss": 3.4200404666607174,
+ "tokens_seen": 2296774656
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024580942828485457,
+ "loss": 1.2715,
+ "theoretical_loss": 3.420032615692879,
+ "tokens_seen": 2296840192
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024579338014042127,
+ "loss": 1.2971,
+ "theoretical_loss": 3.420024765011771,
+ "tokens_seen": 2296905728
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024577733199598797,
+ "loss": 1.3177,
+ "theoretical_loss": 3.4200169146173756,
+ "tokens_seen": 2296971264
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 768307,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.105657935142517,
+ "objective/train/theoretical_loss": 3.4200090645096735,
+ "objective/train/tokens_used": 2317496800,
+ "theoretical_loss": 3.4200090645096735,
+ "tokens_seen": 2297036800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024576128385155467,
+ "loss": 1.1518,
+ "theoretical_loss": 3.4200090645096735,
+ "tokens_seen": 2297036800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024574523570712137,
+ "loss": 1.2841,
+ "theoretical_loss": 3.420001214688646,
+ "tokens_seen": 2297102336
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024572918756268807,
+ "loss": 1.3217,
+ "theoretical_loss": 3.4199933651542755,
+ "tokens_seen": 2297167872
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024571313941825477,
+ "loss": 1.296,
+ "theoretical_loss": 3.419985515906542,
+ "tokens_seen": 2297233408
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024569709127382147,
+ "loss": 1.1431,
+ "theoretical_loss": 3.419977666945427,
+ "tokens_seen": 2297298944
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024568104312938817,
+ "loss": 1.2402,
+ "theoretical_loss": 3.4199698182709133,
+ "tokens_seen": 2297364480
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024566499498495487,
+ "loss": 1.2115,
+ "theoretical_loss": 3.419961969882981,
+ "tokens_seen": 2297430016
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024564894684052157,
+ "loss": 1.2586,
+ "theoretical_loss": 3.419954121781611,
+ "tokens_seen": 2297495552
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002456328986960883,
+ "loss": 1.2367,
+ "theoretical_loss": 3.419946273966786,
+ "tokens_seen": 2297561088
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000245616850551655,
+ "loss": 1.2727,
+ "theoretical_loss": 3.419938426438487,
+ "tokens_seen": 2297626624
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002456008024072217,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4199305791966945,
+ "tokens_seen": 2297692160
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455847542627884,
+ "loss": 1.2832,
+ "theoretical_loss": 3.4199227322413908,
+ "tokens_seen": 2297757696
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455687061183551,
+ "loss": 1.1593,
+ "theoretical_loss": 3.419914885572557,
+ "tokens_seen": 2297823232
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455526579739218,
+ "loss": 1.3186,
+ "theoretical_loss": 3.419907039190174,
+ "tokens_seen": 2297888768
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455366098294885,
+ "loss": 1.2784,
+ "theoretical_loss": 3.419899193094224,
+ "tokens_seen": 2297954304
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455205616850552,
+ "loss": 1.2032,
+ "theoretical_loss": 3.419891347284688,
+ "tokens_seen": 2298019840
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002455045135406219,
+ "loss": 1.1354,
+ "theoretical_loss": 3.419883501761547,
+ "tokens_seen": 2298085376
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454884653961886,
+ "loss": 1.2374,
+ "theoretical_loss": 3.419875656524783,
+ "tokens_seen": 2298150912
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454724172517553,
+ "loss": 1.2615,
+ "theoretical_loss": 3.419867811574377,
+ "tokens_seen": 2298216448
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000245456369107322,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4198599669103107,
+ "tokens_seen": 2298281984
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454403209628887,
+ "loss": 1.2169,
+ "theoretical_loss": 3.419852122532565,
+ "tokens_seen": 2298347520
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454242728184554,
+ "loss": 1.2835,
+ "theoretical_loss": 3.4198442784411216,
+ "tokens_seen": 2298413056
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454082246740221,
+ "loss": 1.2993,
+ "theoretical_loss": 3.419836434635962,
+ "tokens_seen": 2298478592
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002454082246740221,
+ "loss": 1.352,
+ "theoretical_loss": 3.419828591117067,
+ "tokens_seen": 2298544128
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453921765295888,
+ "loss": 1.273,
+ "theoretical_loss": 3.4198207478844194,
+ "tokens_seen": 2298609664
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453761283851555,
+ "loss": 1.3216,
+ "theoretical_loss": 3.4198129049379986,
+ "tokens_seen": 2298675200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453600802407222,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4198050622777876,
+ "tokens_seen": 2298740736
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453440320962889,
+ "loss": 1.3101,
+ "theoretical_loss": 3.419797219903767,
+ "tokens_seen": 2298806272
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453279839518556,
+ "loss": 1.327,
+ "theoretical_loss": 3.4197893778159187,
+ "tokens_seen": 2298871808
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002453119358074223,
+ "loss": 1.2516,
+ "theoretical_loss": 3.4197815360142236,
+ "tokens_seen": 2298937344
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000245295887662989,
+ "loss": 1.3472,
+ "theoretical_loss": 3.419773694498663,
+ "tokens_seen": 2299002880
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002452798395185557,
+ "loss": 1.2326,
+ "theoretical_loss": 3.419765853269219,
+ "tokens_seen": 2299068416
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002452637913741224,
+ "loss": 1.2125,
+ "theoretical_loss": 3.4197580123258726,
+ "tokens_seen": 2299133952
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002452477432296891,
+ "loss": 1.3399,
+ "theoretical_loss": 3.419750171668605,
+ "tokens_seen": 2299199488
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002452316950852558,
+ "loss": 1.1849,
+ "theoretical_loss": 3.419742331297398,
+ "tokens_seen": 2299265024
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002452156469408225,
+ "loss": 1.2078,
+ "theoretical_loss": 3.4197344912122327,
+ "tokens_seen": 2299330560
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451995987963892,
+ "loss": 1.2277,
+ "theoretical_loss": 3.419726651413091,
+ "tokens_seen": 2299396096
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451835506519559,
+ "loss": 1.2681,
+ "theoretical_loss": 3.419718811899954,
+ "tokens_seen": 2299461632
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451675025075226,
+ "loss": 1.1143,
+ "theoretical_loss": 3.4197109726728026,
+ "tokens_seen": 2299527168
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451514543630893,
+ "loss": 1.2242,
+ "theoretical_loss": 3.419703133731619,
+ "tokens_seen": 2299592704
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000245135406218656,
+ "loss": 1.1756,
+ "theoretical_loss": 3.419695295076384,
+ "tokens_seen": 2299658240
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451193580742227,
+ "loss": 1.2637,
+ "theoretical_loss": 3.41968745670708,
+ "tokens_seen": 2299723776
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002451033099297894,
+ "loss": 1.2711,
+ "theoretical_loss": 3.419679618623687,
+ "tokens_seen": 2299789312
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450872617853561,
+ "loss": 1.1857,
+ "theoretical_loss": 3.419671780826188,
+ "tokens_seen": 2299854848
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450712136409228,
+ "loss": 1.3093,
+ "theoretical_loss": 3.419663943314563,
+ "tokens_seen": 2299920384
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450551654964895,
+ "loss": 1.2069,
+ "theoretical_loss": 3.419656106088794,
+ "tokens_seen": 2299985920
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450391173520562,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4196482691488628,
+ "tokens_seen": 2300051456
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450230692076229,
+ "loss": 1.3024,
+ "theoretical_loss": 3.4196404324947505,
+ "tokens_seen": 2300116992
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002450070210631896,
+ "loss": 1.195,
+ "theoretical_loss": 3.419632596126438,
+ "tokens_seen": 2300182528
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002449909729187563,
+ "loss": 1.1644,
+ "theoretical_loss": 3.4196247600439076,
+ "tokens_seen": 2300248064
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 769491,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4527536630630493,
+ "objective/train/theoretical_loss": 3.4196169242471406,
+ "objective/train/tokens_used": 2320773600,
+ "theoretical_loss": 3.4196169242471406,
+ "tokens_seen": 2300313600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000244974924774323,
+ "loss": 1.3522,
+ "theoretical_loss": 3.4196169242471406,
+ "tokens_seen": 2300313600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002449588766298897,
+ "loss": 1.2813,
+ "theoretical_loss": 3.4196090887361184,
+ "tokens_seen": 2300379136
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002449428284854564,
+ "loss": 1.4101,
+ "theoretical_loss": 3.419601253510822,
+ "tokens_seen": 2300444672
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002449267803410231,
+ "loss": 1.175,
+ "theoretical_loss": 3.419593418571233,
+ "tokens_seen": 2300510208
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002449107321965898,
+ "loss": 1.2367,
+ "theoretical_loss": 3.4195855839173332,
+ "tokens_seen": 2300575744
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002448946840521565,
+ "loss": 1.1824,
+ "theoretical_loss": 3.4195777495491035,
+ "tokens_seen": 2300641280
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024487863590772315,
+ "loss": 1.3752,
+ "theoretical_loss": 3.419569915466526,
+ "tokens_seen": 2300706816
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024486258776328985,
+ "loss": 1.2257,
+ "theoretical_loss": 3.4195620816695818,
+ "tokens_seen": 2300772352
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024484653961885655,
+ "loss": 1.3311,
+ "theoretical_loss": 3.419554248158252,
+ "tokens_seen": 2300837888
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024483049147442325,
+ "loss": 1.2443,
+ "theoretical_loss": 3.419546414932519,
+ "tokens_seen": 2300903424
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024481444332998995,
+ "loss": 1.3097,
+ "theoretical_loss": 3.419538581992363,
+ "tokens_seen": 2300968960
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024479839518555665,
+ "loss": 1.1115,
+ "theoretical_loss": 3.419530749337767,
+ "tokens_seen": 2301034496
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024478234704112335,
+ "loss": 1.2281,
+ "theoretical_loss": 3.4195229169687105,
+ "tokens_seen": 2301100032
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024476629889669005,
+ "loss": 1.3452,
+ "theoretical_loss": 3.4195150848851767,
+ "tokens_seen": 2301165568
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024475025075225675,
+ "loss": 1.2994,
+ "theoretical_loss": 3.419507253087146,
+ "tokens_seen": 2301231104
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024473420260782345,
+ "loss": 1.2345,
+ "theoretical_loss": 3.4194994215746006,
+ "tokens_seen": 2301296640
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024471815446339016,
+ "loss": 1.3042,
+ "theoretical_loss": 3.419491590347522,
+ "tokens_seen": 2301362176
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024470210631895686,
+ "loss": 1.1382,
+ "theoretical_loss": 3.41948375940589,
+ "tokens_seen": 2301427712
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024468605817452356,
+ "loss": 1.3672,
+ "theoretical_loss": 3.4194759287496885,
+ "tokens_seen": 2301493248
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024467001003009026,
+ "loss": 1.1663,
+ "theoretical_loss": 3.4194680983788976,
+ "tokens_seen": 2301558784
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024465396188565696,
+ "loss": 1.2034,
+ "theoretical_loss": 3.419460268293499,
+ "tokens_seen": 2301624320
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024463791374122366,
+ "loss": 1.2607,
+ "theoretical_loss": 3.4194524384934737,
+ "tokens_seen": 2301689856
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024462186559679036,
+ "loss": 1.2202,
+ "theoretical_loss": 3.419444608978804,
+ "tokens_seen": 2301755392
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024460581745235706,
+ "loss": 1.1679,
+ "theoretical_loss": 3.4194367797494714,
+ "tokens_seen": 2301820928
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024458976930792376,
+ "loss": 1.2208,
+ "theoretical_loss": 3.4194289508054565,
+ "tokens_seen": 2301886464
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024457372116349046,
+ "loss": 1.2726,
+ "theoretical_loss": 3.4194211221467414,
+ "tokens_seen": 2301952000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024455767301905716,
+ "loss": 1.2151,
+ "theoretical_loss": 3.419413293773307,
+ "tokens_seen": 2302017536
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024454162487462386,
+ "loss": 1.3508,
+ "theoretical_loss": 3.419405465685136,
+ "tokens_seen": 2302083072
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024452557673019056,
+ "loss": 1.2986,
+ "theoretical_loss": 3.4193976378822084,
+ "tokens_seen": 2302148608
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024450952858575726,
+ "loss": 1.135,
+ "theoretical_loss": 3.419389810364507,
+ "tokens_seen": 2302214144
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024449348044132396,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4193819831320127,
+ "tokens_seen": 2302279680
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024447743229689066,
+ "loss": 1.1845,
+ "theoretical_loss": 3.4193741561847064,
+ "tokens_seen": 2302345216
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024446138415245737,
+ "loss": 1.2369,
+ "theoretical_loss": 3.4193663295225707,
+ "tokens_seen": 2302410752
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024444533600802407,
+ "loss": 1.193,
+ "theoretical_loss": 3.4193585031455864,
+ "tokens_seen": 2302476288
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002444292878635908,
+ "loss": 1.267,
+ "theoretical_loss": 3.4193506770537354,
+ "tokens_seen": 2302541824
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002444132397191575,
+ "loss": 1.1866,
+ "theoretical_loss": 3.4193428512469985,
+ "tokens_seen": 2302607360
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443971915747242,
+ "loss": 1.2464,
+ "theoretical_loss": 3.419335025725358,
+ "tokens_seen": 2302672896
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443811434302909,
+ "loss": 1.1643,
+ "theoretical_loss": 3.4193272004887953,
+ "tokens_seen": 2302738432
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443650952858576,
+ "loss": 1.3318,
+ "theoretical_loss": 3.419319375537291,
+ "tokens_seen": 2302803968
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443490471414243,
+ "loss": 1.1503,
+ "theoretical_loss": 3.4193115508708276,
+ "tokens_seen": 2302869504
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000244332998996991,
+ "loss": 1.2373,
+ "theoretical_loss": 3.419303726489386,
+ "tokens_seen": 2302935040
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443169508525577,
+ "loss": 1.3668,
+ "theoretical_loss": 3.419295902392949,
+ "tokens_seen": 2303000576
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002443009027081244,
+ "loss": 1.169,
+ "theoretical_loss": 3.419288078581496,
+ "tokens_seen": 2303066112
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442848545636911,
+ "loss": 1.1522,
+ "theoretical_loss": 3.4192802550550097,
+ "tokens_seen": 2303131648
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442688064192578,
+ "loss": 1.329,
+ "theoretical_loss": 3.419272431813472,
+ "tokens_seen": 2303197184
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442527582748245,
+ "loss": 1.2005,
+ "theoretical_loss": 3.4192646088568637,
+ "tokens_seen": 2303262720
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442367101303912,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4192567861851666,
+ "tokens_seen": 2303328256
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442206619859579,
+ "loss": 1.2947,
+ "theoretical_loss": 3.419248963798362,
+ "tokens_seen": 2303393792
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002442046138415246,
+ "loss": 1.2765,
+ "theoretical_loss": 3.419241141696432,
+ "tokens_seen": 2303459328
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002441885656970913,
+ "loss": 1.2166,
+ "theoretical_loss": 3.419233319879357,
+ "tokens_seen": 2303524864
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 770522,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3516998291015625,
+ "objective/train/theoretical_loss": 3.41922549834712,
+ "objective/train/tokens_used": 2324050400,
+ "theoretical_loss": 3.41922549834712,
+ "tokens_seen": 2303590400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000244172517552658,
+ "loss": 1.3416,
+ "theoretical_loss": 3.41922549834712,
+ "tokens_seen": 2303590400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002441564694082247,
+ "loss": 1.2954,
+ "theoretical_loss": 3.4192176770997014,
+ "tokens_seen": 2303655936
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002441404212637914,
+ "loss": 1.2578,
+ "theoretical_loss": 3.4192098561370825,
+ "tokens_seen": 2303721472
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002441243731193581,
+ "loss": 1.1514,
+ "theoretical_loss": 3.4192020354592465,
+ "tokens_seen": 2303787008
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002441083249749248,
+ "loss": 1.1638,
+ "theoretical_loss": 3.419194215066173,
+ "tokens_seen": 2303852544
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002440922768304915,
+ "loss": 1.2296,
+ "theoretical_loss": 3.4191863949578445,
+ "tokens_seen": 2303918080
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002440762286860582,
+ "loss": 1.1848,
+ "theoretical_loss": 3.4191785751342425,
+ "tokens_seen": 2303983616
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024406018054162488,
+ "loss": 1.2919,
+ "theoretical_loss": 3.4191707555953483,
+ "tokens_seen": 2304049152
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024404413239719158,
+ "loss": 1.263,
+ "theoretical_loss": 3.419162936341144,
+ "tokens_seen": 2304114688
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024402808425275828,
+ "loss": 1.3238,
+ "theoretical_loss": 3.4191551173716106,
+ "tokens_seen": 2304180224
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024401203610832498,
+ "loss": 1.333,
+ "theoretical_loss": 3.4191472986867293,
+ "tokens_seen": 2304245760
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024399598796389168,
+ "loss": 1.2272,
+ "theoretical_loss": 3.4191394802864825,
+ "tokens_seen": 2304311296
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024397993981945838,
+ "loss": 1.2379,
+ "theoretical_loss": 3.419131662170851,
+ "tokens_seen": 2304376832
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024396389167502508,
+ "loss": 1.2953,
+ "theoretical_loss": 3.419123844339817,
+ "tokens_seen": 2304442368
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024394784353059179,
+ "loss": 1.2376,
+ "theoretical_loss": 3.4191160267933616,
+ "tokens_seen": 2304507904
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024393179538615849,
+ "loss": 1.1884,
+ "theoretical_loss": 3.4191082095314664,
+ "tokens_seen": 2304573440
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002439157472417252,
+ "loss": 1.1712,
+ "theoretical_loss": 3.419100392554113,
+ "tokens_seen": 2304638976
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438996990972919,
+ "loss": 1.3699,
+ "theoretical_loss": 3.419092575861283,
+ "tokens_seen": 2304704512
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438836509528586,
+ "loss": 1.2194,
+ "theoretical_loss": 3.419084759452958,
+ "tokens_seen": 2304770048
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438676028084253,
+ "loss": 1.1735,
+ "theoretical_loss": 3.4190769433291197,
+ "tokens_seen": 2304835584
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000243851554663992,
+ "loss": 1.2454,
+ "theoretical_loss": 3.419069127489749,
+ "tokens_seen": 2304901120
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438355065195587,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4190613119348283,
+ "tokens_seen": 2304966656
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438194583751254,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4190534966643384,
+ "tokens_seen": 2305032192
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002438034102306921,
+ "loss": 1.3197,
+ "theoretical_loss": 3.4190456816782615,
+ "tokens_seen": 2305097728
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437873620862588,
+ "loss": 1.1784,
+ "theoretical_loss": 3.419037866976579,
+ "tokens_seen": 2305163264
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437713139418255,
+ "loss": 1.3009,
+ "theoretical_loss": 3.419030052559272,
+ "tokens_seen": 2305228800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437552657973922,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4190222384263222,
+ "tokens_seen": 2305294336
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437392176529589,
+ "loss": 1.2733,
+ "theoretical_loss": 3.419014424577712,
+ "tokens_seen": 2305359872
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437231695085256,
+ "loss": 1.2621,
+ "theoretical_loss": 3.4190066110134216,
+ "tokens_seen": 2305425408
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002437071213640923,
+ "loss": 1.1991,
+ "theoretical_loss": 3.418998797733434,
+ "tokens_seen": 2305490944
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000243691073219659,
+ "loss": 1.1619,
+ "theoretical_loss": 3.41899098473773,
+ "tokens_seen": 2305556480
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024367502507522567,
+ "loss": 1.2237,
+ "theoretical_loss": 3.418983172026291,
+ "tokens_seen": 2305622016
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024365897693079237,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4189753595990995,
+ "tokens_seen": 2305687552
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024364292878635907,
+ "loss": 1.2892,
+ "theoretical_loss": 3.418967547456136,
+ "tokens_seen": 2305753088
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024362688064192577,
+ "loss": 1.1772,
+ "theoretical_loss": 3.418959735597382,
+ "tokens_seen": 2305818624
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024361083249749247,
+ "loss": 1.2196,
+ "theoretical_loss": 3.4189519240228203,
+ "tokens_seen": 2305884160
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024359478435305917,
+ "loss": 1.1519,
+ "theoretical_loss": 3.4189441127324316,
+ "tokens_seen": 2305949696
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024357873620862587,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4189363017261973,
+ "tokens_seen": 2306015232
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024356268806419257,
+ "loss": 1.1952,
+ "theoretical_loss": 3.4189284910041,
+ "tokens_seen": 2306080768
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024354663991975927,
+ "loss": 1.28,
+ "theoretical_loss": 3.4189206805661203,
+ "tokens_seen": 2306146304
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024353059177532597,
+ "loss": 1.3052,
+ "theoretical_loss": 3.4189128704122402,
+ "tokens_seen": 2306211840
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024351454363089268,
+ "loss": 1.0519,
+ "theoretical_loss": 3.4189050605424414,
+ "tokens_seen": 2306277376
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024349849548645938,
+ "loss": 1.215,
+ "theoretical_loss": 3.418897250956705,
+ "tokens_seen": 2306342912
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024348244734202608,
+ "loss": 1.2355,
+ "theoretical_loss": 3.418889441655013,
+ "tokens_seen": 2306408448
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024346639919759278,
+ "loss": 1.3166,
+ "theoretical_loss": 3.4188816326373472,
+ "tokens_seen": 2306473984
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024345035105315948,
+ "loss": 1.3523,
+ "theoretical_loss": 3.418873823903689,
+ "tokens_seen": 2306539520
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024343430290872618,
+ "loss": 1.1571,
+ "theoretical_loss": 3.41886601545402,
+ "tokens_seen": 2306605056
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024341825476429288,
+ "loss": 1.2541,
+ "theoretical_loss": 3.418858207288321,
+ "tokens_seen": 2306670592
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024340220661985958,
+ "loss": 1.2924,
+ "theoretical_loss": 3.4188503994065744,
+ "tokens_seen": 2306736128
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024338615847542628,
+ "loss": 1.1861,
+ "theoretical_loss": 3.4188425918087626,
+ "tokens_seen": 2306801664
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 771663,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1629533767700195,
+ "objective/train/theoretical_loss": 3.4188347844948654,
+ "objective/train/tokens_used": 2327327200,
+ "theoretical_loss": 3.4188347844948654,
+ "tokens_seen": 2306867200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024337011033099298,
+ "loss": 1.1111,
+ "theoretical_loss": 3.4188347844948654,
+ "tokens_seen": 2306867200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024335406218655968,
+ "loss": 1.3076,
+ "theoretical_loss": 3.418826977464866,
+ "tokens_seen": 2306932736
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024333801404212638,
+ "loss": 1.2254,
+ "theoretical_loss": 3.418819170718745,
+ "tokens_seen": 2306998272
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024332196589769308,
+ "loss": 1.1884,
+ "theoretical_loss": 3.418811364256485,
+ "tokens_seen": 2307063808
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024330591775325978,
+ "loss": 1.316,
+ "theoretical_loss": 3.4188035580780665,
+ "tokens_seen": 2307129344
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024328986960882646,
+ "loss": 1.3396,
+ "theoretical_loss": 3.4187957521834713,
+ "tokens_seen": 2307194880
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024327382146439316,
+ "loss": 1.2228,
+ "theoretical_loss": 3.418787946572682,
+ "tokens_seen": 2307260416
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024325777331995986,
+ "loss": 1.1799,
+ "theoretical_loss": 3.418780141245679,
+ "tokens_seen": 2307325952
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002432417251755266,
+ "loss": 1.2019,
+ "theoretical_loss": 3.418772336202445,
+ "tokens_seen": 2307391488
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024322567703109331,
+ "loss": 1.204,
+ "theoretical_loss": 3.418764531442961,
+ "tokens_seen": 2307457024
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024320962888666001,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4187567269672083,
+ "tokens_seen": 2307522560
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024319358074222672,
+ "loss": 1.1461,
+ "theoretical_loss": 3.4187489227751695,
+ "tokens_seen": 2307588096
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024317753259779342,
+ "loss": 1.1785,
+ "theoretical_loss": 3.4187411188668255,
+ "tokens_seen": 2307653632
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024316148445336012,
+ "loss": 1.2037,
+ "theoretical_loss": 3.418733315242158,
+ "tokens_seen": 2307719168
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024314543630892682,
+ "loss": 1.3124,
+ "theoretical_loss": 3.418725511901149,
+ "tokens_seen": 2307784704
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024312938816449352,
+ "loss": 1.2337,
+ "theoretical_loss": 3.4187177088437797,
+ "tokens_seen": 2307850240
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024311334002006022,
+ "loss": 1.1818,
+ "theoretical_loss": 3.4187099060700317,
+ "tokens_seen": 2307915776
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024309729187562692,
+ "loss": 1.313,
+ "theoretical_loss": 3.418702103579887,
+ "tokens_seen": 2307981312
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024308124373119362,
+ "loss": 1.217,
+ "theoretical_loss": 3.4186943013733275,
+ "tokens_seen": 2308046848
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024306519558676032,
+ "loss": 1.2899,
+ "theoretical_loss": 3.418686499450334,
+ "tokens_seen": 2308112384
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024304914744232702,
+ "loss": 1.2581,
+ "theoretical_loss": 3.418678697810889,
+ "tokens_seen": 2308177920
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002430330992978937,
+ "loss": 1.2201,
+ "theoretical_loss": 3.418670896454973,
+ "tokens_seen": 2308243456
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002430170511534604,
+ "loss": 1.2428,
+ "theoretical_loss": 3.418663095382569,
+ "tokens_seen": 2308308992
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002430010030090271,
+ "loss": 1.2345,
+ "theoretical_loss": 3.418655294593658,
+ "tokens_seen": 2308374528
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429849548645938,
+ "loss": 1.1857,
+ "theoretical_loss": 3.4186474940882214,
+ "tokens_seen": 2308440064
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429689067201605,
+ "loss": 1.1638,
+ "theoretical_loss": 3.418639693866241,
+ "tokens_seen": 2308505600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429528585757272,
+ "loss": 1.1704,
+ "theoretical_loss": 3.418631893927699,
+ "tokens_seen": 2308571136
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429368104312939,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4186240942725767,
+ "tokens_seen": 2308636672
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429207622868606,
+ "loss": 1.1853,
+ "theoretical_loss": 3.418616294900855,
+ "tokens_seen": 2308702208
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002429047141424273,
+ "loss": 1.1513,
+ "theoretical_loss": 3.418608495812517,
+ "tokens_seen": 2308767744
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000242888665997994,
+ "loss": 1.1231,
+ "theoretical_loss": 3.4186006970075433,
+ "tokens_seen": 2308833280
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002428726178535607,
+ "loss": 1.2092,
+ "theoretical_loss": 3.4185928984859157,
+ "tokens_seen": 2308898816
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002428565697091274,
+ "loss": 1.2945,
+ "theoretical_loss": 3.418585100247616,
+ "tokens_seen": 2308964352
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002428405215646941,
+ "loss": 1.252,
+ "theoretical_loss": 3.4185773022926265,
+ "tokens_seen": 2309029888
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002428244734202608,
+ "loss": 1.2378,
+ "theoretical_loss": 3.418569504620928,
+ "tokens_seen": 2309095424
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002428084252758275,
+ "loss": 1.188,
+ "theoretical_loss": 3.4185617072325023,
+ "tokens_seen": 2309160960
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002427923771313942,
+ "loss": 1.2108,
+ "theoretical_loss": 3.418553910127331,
+ "tokens_seen": 2309226496
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002427763289869609,
+ "loss": 1.3484,
+ "theoretical_loss": 3.4185461133053963,
+ "tokens_seen": 2309292032
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002427602808425276,
+ "loss": 1.3047,
+ "theoretical_loss": 3.4185383167666794,
+ "tokens_seen": 2309357568
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002427442326980943,
+ "loss": 1.2167,
+ "theoretical_loss": 3.418530520511162,
+ "tokens_seen": 2309423104
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000242728184553661,
+ "loss": 1.268,
+ "theoretical_loss": 3.418522724538826,
+ "tokens_seen": 2309488640
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002427121364092277,
+ "loss": 1.3635,
+ "theoretical_loss": 3.4185149288496532,
+ "tokens_seen": 2309554176
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002426960882647944,
+ "loss": 1.2457,
+ "theoretical_loss": 3.4185071334436246,
+ "tokens_seen": 2309619712
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002426800401203611,
+ "loss": 1.1987,
+ "theoretical_loss": 3.4184993383207227,
+ "tokens_seen": 2309685248
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002426639919759278,
+ "loss": 1.3117,
+ "theoretical_loss": 3.4184915434809287,
+ "tokens_seen": 2309750784
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024264794383149448,
+ "loss": 1.1847,
+ "theoretical_loss": 3.4184837489242246,
+ "tokens_seen": 2309816320
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024263189568706118,
+ "loss": 1.1902,
+ "theoretical_loss": 3.4184759546505914,
+ "tokens_seen": 2309881856
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024261584754262788,
+ "loss": 1.1751,
+ "theoretical_loss": 3.418468160660012,
+ "tokens_seen": 2309947392
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024259979939819458,
+ "loss": 1.3326,
+ "theoretical_loss": 3.4184603669524667,
+ "tokens_seen": 2310012928
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024258375125376129,
+ "loss": 1.2949,
+ "theoretical_loss": 3.4184525735279383,
+ "tokens_seen": 2310078464
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 773000,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2139942646026611,
+ "objective/train/theoretical_loss": 3.418444780386408,
+ "objective/train/tokens_used": 2330604000,
+ "theoretical_loss": 3.418444780386408,
+ "tokens_seen": 2310144000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024256770310932799,
+ "loss": 1.1897,
+ "theoretical_loss": 3.418444780386408,
+ "tokens_seen": 2310144000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002425516549648947,
+ "loss": 1.2346,
+ "theoretical_loss": 3.4184369875278575,
+ "tokens_seen": 2310209536
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002425356068204614,
+ "loss": 1.2794,
+ "theoretical_loss": 3.4184291949522683,
+ "tokens_seen": 2310275072
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002425195586760281,
+ "loss": 1.2384,
+ "theoretical_loss": 3.418421402659623,
+ "tokens_seen": 2310340608
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002425035105315948,
+ "loss": 1.1777,
+ "theoretical_loss": 3.4184136106499023,
+ "tokens_seen": 2310406144
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002424874623871615,
+ "loss": 1.3348,
+ "theoretical_loss": 3.418405818923088,
+ "tokens_seen": 2310471680
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002424714142427282,
+ "loss": 1.2096,
+ "theoretical_loss": 3.4183980274791623,
+ "tokens_seen": 2310537216
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002424553660982949,
+ "loss": 1.2266,
+ "theoretical_loss": 3.418390236318107,
+ "tokens_seen": 2310602752
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002424393179538616,
+ "loss": 1.224,
+ "theoretical_loss": 3.418382445439903,
+ "tokens_seen": 2310668288
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002424232698094283,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4183746548445324,
+ "tokens_seen": 2310733824
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000242407221664995,
+ "loss": 1.2155,
+ "theoretical_loss": 3.4183668645319774,
+ "tokens_seen": 2310799360
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423911735205617,
+ "loss": 1.3593,
+ "theoretical_loss": 3.418359074502219,
+ "tokens_seen": 2310864896
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423751253761284,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4183512847552393,
+ "tokens_seen": 2310930432
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423590772316951,
+ "loss": 1.2543,
+ "theoretical_loss": 3.41834349529102,
+ "tokens_seen": 2310995968
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423430290872618,
+ "loss": 1.2913,
+ "theoretical_loss": 3.4183357061095427,
+ "tokens_seen": 2311061504
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423269809428285,
+ "loss": 1.2047,
+ "theoretical_loss": 3.418327917210789,
+ "tokens_seen": 2311127040
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002423109327983952,
+ "loss": 1.1791,
+ "theoretical_loss": 3.4183201285947415,
+ "tokens_seen": 2311192576
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002422948846539619,
+ "loss": 1.2353,
+ "theoretical_loss": 3.4183123402613806,
+ "tokens_seen": 2311258112
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002422788365095286,
+ "loss": 1.2173,
+ "theoretical_loss": 3.4183045522106887,
+ "tokens_seen": 2311323648
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024226278836509527,
+ "loss": 1.2546,
+ "theoretical_loss": 3.4182967644426476,
+ "tokens_seen": 2311389184
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024224674022066197,
+ "loss": 1.1064,
+ "theoretical_loss": 3.418288976957239,
+ "tokens_seen": 2311454720
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024223069207622867,
+ "loss": 1.217,
+ "theoretical_loss": 3.4182811897544445,
+ "tokens_seen": 2311520256
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024221464393179537,
+ "loss": 1.23,
+ "theoretical_loss": 3.418273402834246,
+ "tokens_seen": 2311585792
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024219859578736207,
+ "loss": 1.2094,
+ "theoretical_loss": 3.4182656161966247,
+ "tokens_seen": 2311651328
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024218254764292877,
+ "loss": 1.361,
+ "theoretical_loss": 3.418257829841563,
+ "tokens_seen": 2311716864
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024216649949849547,
+ "loss": 1.2967,
+ "theoretical_loss": 3.418250043769042,
+ "tokens_seen": 2311782400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024215045135406218,
+ "loss": 1.2401,
+ "theoretical_loss": 3.418242257979044,
+ "tokens_seen": 2311847936
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024213440320962888,
+ "loss": 1.2797,
+ "theoretical_loss": 3.4182344724715508,
+ "tokens_seen": 2311913472
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024211835506519558,
+ "loss": 1.1904,
+ "theoretical_loss": 3.418226687246544,
+ "tokens_seen": 2311979008
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024210230692076228,
+ "loss": 1.1804,
+ "theoretical_loss": 3.418218902304005,
+ "tokens_seen": 2312044544
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024208625877632898,
+ "loss": 1.2188,
+ "theoretical_loss": 3.4182111176439154,
+ "tokens_seen": 2312110080
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024207021063189568,
+ "loss": 1.307,
+ "theoretical_loss": 3.418203333266258,
+ "tokens_seen": 2312175616
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024205416248746238,
+ "loss": 1.2398,
+ "theoretical_loss": 3.4181955491710134,
+ "tokens_seen": 2312241152
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024203811434302913,
+ "loss": 1.2119,
+ "theoretical_loss": 3.418187765358164,
+ "tokens_seen": 2312306688
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002420220661985958,
+ "loss": 1.1762,
+ "theoretical_loss": 3.4181799818276914,
+ "tokens_seen": 2312372224
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002420060180541625,
+ "loss": 1.2232,
+ "theoretical_loss": 3.418172198579577,
+ "tokens_seen": 2312437760
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002419899699097292,
+ "loss": 1.2855,
+ "theoretical_loss": 3.4181644156138034,
+ "tokens_seen": 2312503296
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002419739217652959,
+ "loss": 1.243,
+ "theoretical_loss": 3.4181566329303514,
+ "tokens_seen": 2312568832
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002419578736208626,
+ "loss": 1.16,
+ "theoretical_loss": 3.4181488505292035,
+ "tokens_seen": 2312634368
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002419418254764293,
+ "loss": 1.2766,
+ "theoretical_loss": 3.418141068410341,
+ "tokens_seen": 2312699904
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000241925777331996,
+ "loss": 1.0887,
+ "theoretical_loss": 3.4181332865737457,
+ "tokens_seen": 2312765440
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002419097291875627,
+ "loss": 1.2448,
+ "theoretical_loss": 3.4181255050194,
+ "tokens_seen": 2312830976
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002418936810431294,
+ "loss": 1.1304,
+ "theoretical_loss": 3.418117723747285,
+ "tokens_seen": 2312896512
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002418776328986961,
+ "loss": 1.1748,
+ "theoretical_loss": 3.418109942757382,
+ "tokens_seen": 2312962048
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024186158475426281,
+ "loss": 1.1968,
+ "theoretical_loss": 3.418102162049674,
+ "tokens_seen": 2313027584
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024184553660982951,
+ "loss": 1.191,
+ "theoretical_loss": 3.418094381624142,
+ "tokens_seen": 2313093120
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024182948846539622,
+ "loss": 1.2321,
+ "theoretical_loss": 3.418086601480768,
+ "tokens_seen": 2313158656
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024181344032096292,
+ "loss": 1.2134,
+ "theoretical_loss": 3.4180788216195332,
+ "tokens_seen": 2313224192
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024179739217652962,
+ "loss": 1.2907,
+ "theoretical_loss": 3.4180710420404203,
+ "tokens_seen": 2313289728
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024178134403209632,
+ "loss": 1.2111,
+ "theoretical_loss": 3.4180632627434107,
+ "tokens_seen": 2313355264
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 773923,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1678115129470825,
+ "objective/train/theoretical_loss": 3.418055483728486,
+ "objective/train/tokens_used": 2333880800,
+ "theoretical_loss": 3.418055483728486,
+ "tokens_seen": 2313420800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024176529588766302,
+ "loss": 1.2176,
+ "theoretical_loss": 3.418055483728486,
+ "tokens_seen": 2313420800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024174924774322972,
+ "loss": 1.3111,
+ "theoretical_loss": 3.418047704995628,
+ "tokens_seen": 2313486336
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024173319959879642,
+ "loss": 1.208,
+ "theoretical_loss": 3.418039926544819,
+ "tokens_seen": 2313551872
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024171715145436312,
+ "loss": 1.2033,
+ "theoretical_loss": 3.41803214837604,
+ "tokens_seen": 2313617408
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024170110330992982,
+ "loss": 1.2241,
+ "theoretical_loss": 3.4180243704892734,
+ "tokens_seen": 2313682944
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024168505516549652,
+ "loss": 1.2007,
+ "theoretical_loss": 3.4180165928845008,
+ "tokens_seen": 2313748480
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024166900702106322,
+ "loss": 1.2143,
+ "theoretical_loss": 3.418008815561704,
+ "tokens_seen": 2313814016
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024165295887662992,
+ "loss": 1.1417,
+ "theoretical_loss": 3.418001038520864,
+ "tokens_seen": 2313879552
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002416369107321966,
+ "loss": 1.2981,
+ "theoretical_loss": 3.417993261761964,
+ "tokens_seen": 2313945088
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002416208625877633,
+ "loss": 1.1643,
+ "theoretical_loss": 3.4179854852849854,
+ "tokens_seen": 2314010624
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024160481444333,
+ "loss": 1.2872,
+ "theoretical_loss": 3.417977709089909,
+ "tokens_seen": 2314076160
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415887662988967,
+ "loss": 1.1273,
+ "theoretical_loss": 3.417969933176718,
+ "tokens_seen": 2314141696
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415727181544634,
+ "loss": 1.2216,
+ "theoretical_loss": 3.417962157545393,
+ "tokens_seen": 2314207232
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415566700100301,
+ "loss": 1.179,
+ "theoretical_loss": 3.4179543821959166,
+ "tokens_seen": 2314272768
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415406218655968,
+ "loss": 1.2121,
+ "theoretical_loss": 3.41794660712827,
+ "tokens_seen": 2314338304
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415245737211635,
+ "loss": 1.1965,
+ "theoretical_loss": 3.4179388323424353,
+ "tokens_seen": 2314403840
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002415085255767302,
+ "loss": 1.2813,
+ "theoretical_loss": 3.417931057838395,
+ "tokens_seen": 2314469376
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002414924774322969,
+ "loss": 1.2938,
+ "theoretical_loss": 3.4179232836161297,
+ "tokens_seen": 2314534912
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002414764292878636,
+ "loss": 1.1113,
+ "theoretical_loss": 3.4179155096756215,
+ "tokens_seen": 2314600448
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002414603811434303,
+ "loss": 1.1911,
+ "theoretical_loss": 3.417907736016853,
+ "tokens_seen": 2314665984
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000241444332998997,
+ "loss": 1.2883,
+ "theoretical_loss": 3.417899962639805,
+ "tokens_seen": 2314731520
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002414282848545637,
+ "loss": 1.3057,
+ "theoretical_loss": 3.4178921895444603,
+ "tokens_seen": 2314797056
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002414122367101304,
+ "loss": 1.2054,
+ "theoretical_loss": 3.4178844167308,
+ "tokens_seen": 2314862592
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413961885656971,
+ "loss": 1.3421,
+ "theoretical_loss": 3.417876644198806,
+ "tokens_seen": 2314928128
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413801404212638,
+ "loss": 1.1826,
+ "theoretical_loss": 3.41786887194846,
+ "tokens_seen": 2314993664
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413640922768305,
+ "loss": 1.2321,
+ "theoretical_loss": 3.4178610999797447,
+ "tokens_seen": 2315059200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413480441323972,
+ "loss": 1.2188,
+ "theoretical_loss": 3.4178533282926407,
+ "tokens_seen": 2315124736
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413319959879639,
+ "loss": 1.286,
+ "theoretical_loss": 3.417845556887131,
+ "tokens_seen": 2315190272
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002413159478435306,
+ "loss": 1.171,
+ "theoretical_loss": 3.4178377857631963,
+ "tokens_seen": 2315255808
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002412998996990973,
+ "loss": 1.2522,
+ "theoretical_loss": 3.4178300149208187,
+ "tokens_seen": 2315321344
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000241283851554664,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4178222443599813,
+ "tokens_seen": 2315386880
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002412678034102307,
+ "loss": 1.3311,
+ "theoretical_loss": 3.417814474080664,
+ "tokens_seen": 2315452416
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024125175526579738,
+ "loss": 1.2337,
+ "theoretical_loss": 3.41780670408285,
+ "tokens_seen": 2315517952
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024123570712136408,
+ "loss": 1.325,
+ "theoretical_loss": 3.4177989343665205,
+ "tokens_seen": 2315583488
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024121965897693078,
+ "loss": 1.2764,
+ "theoretical_loss": 3.417791164931658,
+ "tokens_seen": 2315649024
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024120361083249749,
+ "loss": 1.1479,
+ "theoretical_loss": 3.417783395778243,
+ "tokens_seen": 2315714560
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024118756268806419,
+ "loss": 1.1879,
+ "theoretical_loss": 3.4177756269062587,
+ "tokens_seen": 2315780096
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002411715145436309,
+ "loss": 1.1476,
+ "theoretical_loss": 3.4177678583156865,
+ "tokens_seen": 2315845632
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002411554663991976,
+ "loss": 1.1463,
+ "theoretical_loss": 3.417760090006508,
+ "tokens_seen": 2315911168
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002411394182547643,
+ "loss": 1.206,
+ "theoretical_loss": 3.4177523219787056,
+ "tokens_seen": 2315976704
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000241123370110331,
+ "loss": 1.1195,
+ "theoretical_loss": 3.4177445542322604,
+ "tokens_seen": 2316042240
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002411073219658977,
+ "loss": 1.2057,
+ "theoretical_loss": 3.4177367867671546,
+ "tokens_seen": 2316107776
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410912738214644,
+ "loss": 1.2296,
+ "theoretical_loss": 3.4177290195833705,
+ "tokens_seen": 2316173312
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410752256770311,
+ "loss": 1.1976,
+ "theoretical_loss": 3.417721252680889,
+ "tokens_seen": 2316238848
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410591775325978,
+ "loss": 1.3124,
+ "theoretical_loss": 3.417713486059693,
+ "tokens_seen": 2316304384
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410431293881645,
+ "loss": 1.1993,
+ "theoretical_loss": 3.4177057197197636,
+ "tokens_seen": 2316369920
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410270812437312,
+ "loss": 1.2105,
+ "theoretical_loss": 3.417697953661083,
+ "tokens_seen": 2316435456
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002410110330992979,
+ "loss": 1.303,
+ "theoretical_loss": 3.4176901878836325,
+ "tokens_seen": 2316500992
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002409949849548646,
+ "loss": 1.1791,
+ "theoretical_loss": 3.417682422387395,
+ "tokens_seen": 2316566528
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002409789368104313,
+ "loss": 1.1918,
+ "theoretical_loss": 3.417674657172351,
+ "tokens_seen": 2316632064
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 774990,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1873466968536377,
+ "objective/train/theoretical_loss": 3.417666892238484,
+ "objective/train/tokens_used": 2337157600,
+ "theoretical_loss": 3.417666892238484,
+ "tokens_seen": 2316697600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000240962888665998,
+ "loss": 1.2267,
+ "theoretical_loss": 3.417666892238484,
+ "tokens_seen": 2316697600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002409468405215647,
+ "loss": 1.1805,
+ "theoretical_loss": 3.4176591275857744,
+ "tokens_seen": 2316763136
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002409307923771314,
+ "loss": 1.3056,
+ "theoretical_loss": 3.417651363214205,
+ "tokens_seen": 2316828672
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002409147442326981,
+ "loss": 1.2232,
+ "theoretical_loss": 3.4176435991237577,
+ "tokens_seen": 2316894208
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002408986960882648,
+ "loss": 1.053,
+ "theoretical_loss": 3.4176358353144134,
+ "tokens_seen": 2316959744
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002408826479438315,
+ "loss": 1.2256,
+ "theoretical_loss": 3.4176280717861545,
+ "tokens_seen": 2317025280
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024086659979939817,
+ "loss": 1.167,
+ "theoretical_loss": 3.4176203085389636,
+ "tokens_seen": 2317090816
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024085055165496487,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4176125455728212,
+ "tokens_seen": 2317156352
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024083450351053163,
+ "loss": 1.1633,
+ "theoretical_loss": 3.41760478288771,
+ "tokens_seen": 2317221888
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024081845536609833,
+ "loss": 1.2474,
+ "theoretical_loss": 3.4175970204836124,
+ "tokens_seen": 2317287424
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024080240722166503,
+ "loss": 1.2162,
+ "theoretical_loss": 3.4175892583605094,
+ "tokens_seen": 2317352960
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024078635907723173,
+ "loss": 1.2649,
+ "theoretical_loss": 3.417581496518383,
+ "tokens_seen": 2317418496
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024077031093279843,
+ "loss": 1.2525,
+ "theoretical_loss": 3.417573734957215,
+ "tokens_seen": 2317484032
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024075426278836513,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4175659736769877,
+ "tokens_seen": 2317549568
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024073821464393183,
+ "loss": 1.174,
+ "theoretical_loss": 3.417558212677683,
+ "tokens_seen": 2317615104
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024072216649949853,
+ "loss": 1.2988,
+ "theoretical_loss": 3.4175504519592828,
+ "tokens_seen": 2317680640
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024070611835506523,
+ "loss": 1.2362,
+ "theoretical_loss": 3.417542691521768,
+ "tokens_seen": 2317746176
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024069007021063193,
+ "loss": 1.2468,
+ "theoretical_loss": 3.4175349313651218,
+ "tokens_seen": 2317811712
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024067402206619863,
+ "loss": 1.2884,
+ "theoretical_loss": 3.4175271714893256,
+ "tokens_seen": 2317877248
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024065797392176533,
+ "loss": 1.244,
+ "theoretical_loss": 3.4175194118943613,
+ "tokens_seen": 2317942784
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024064192577733203,
+ "loss": 1.3247,
+ "theoretical_loss": 3.4175116525802105,
+ "tokens_seen": 2318008320
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002406258776328987,
+ "loss": 1.1485,
+ "theoretical_loss": 3.4175038935468556,
+ "tokens_seen": 2318073856
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002406098294884654,
+ "loss": 1.1154,
+ "theoretical_loss": 3.417496134794278,
+ "tokens_seen": 2318139392
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405937813440321,
+ "loss": 1.2538,
+ "theoretical_loss": 3.41748837632246,
+ "tokens_seen": 2318204928
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405777331995988,
+ "loss": 1.2356,
+ "theoretical_loss": 3.4174806181313833,
+ "tokens_seen": 2318270464
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405616850551655,
+ "loss": 1.1618,
+ "theoretical_loss": 3.41747286022103,
+ "tokens_seen": 2318336000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405456369107322,
+ "loss": 1.2046,
+ "theoretical_loss": 3.4174651025913816,
+ "tokens_seen": 2318401536
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405295887662989,
+ "loss": 1.1598,
+ "theoretical_loss": 3.4174573452424206,
+ "tokens_seen": 2318467072
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002405135406218656,
+ "loss": 1.218,
+ "theoretical_loss": 3.4174495881741285,
+ "tokens_seen": 2318532608
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002404974924774323,
+ "loss": 1.207,
+ "theoretical_loss": 3.4174418313864874,
+ "tokens_seen": 2318598144
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024048144433299901,
+ "loss": 1.2691,
+ "theoretical_loss": 3.4174340748794787,
+ "tokens_seen": 2318663680
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024046539618856571,
+ "loss": 1.2642,
+ "theoretical_loss": 3.417426318653085,
+ "tokens_seen": 2318729216
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024044934804413242,
+ "loss": 1.1647,
+ "theoretical_loss": 3.4174185627072884,
+ "tokens_seen": 2318794752
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024043329989969912,
+ "loss": 1.1665,
+ "theoretical_loss": 3.41741080704207,
+ "tokens_seen": 2318860288
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024041725175526582,
+ "loss": 1.2644,
+ "theoretical_loss": 3.417403051657412,
+ "tokens_seen": 2318925824
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024040120361083252,
+ "loss": 1.2616,
+ "theoretical_loss": 3.4173952965532965,
+ "tokens_seen": 2318991360
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024038515546639922,
+ "loss": 1.3224,
+ "theoretical_loss": 3.4173875417297053,
+ "tokens_seen": 2319056896
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024036910732196592,
+ "loss": 1.2479,
+ "theoretical_loss": 3.4173797871866203,
+ "tokens_seen": 2319122432
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024035305917753262,
+ "loss": 1.2857,
+ "theoretical_loss": 3.417372032924024,
+ "tokens_seen": 2319187968
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024033701103309932,
+ "loss": 1.191,
+ "theoretical_loss": 3.417364278941897,
+ "tokens_seen": 2319253504
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024032096288866602,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4173565252402223,
+ "tokens_seen": 2319319040
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024030491474423272,
+ "loss": 1.3087,
+ "theoretical_loss": 3.4173487718189817,
+ "tokens_seen": 2319384576
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024028886659979942,
+ "loss": 1.2734,
+ "theoretical_loss": 3.417341018678157,
+ "tokens_seen": 2319450112
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024027281845536612,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4173332658177302,
+ "tokens_seen": 2319515648
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024025677031093282,
+ "loss": 1.2498,
+ "theoretical_loss": 3.4173255132376834,
+ "tokens_seen": 2319581184
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002402407221664995,
+ "loss": 1.2062,
+ "theoretical_loss": 3.417317760937998,
+ "tokens_seen": 2319646720
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002402246740220662,
+ "loss": 1.1652,
+ "theoretical_loss": 3.4173100089186565,
+ "tokens_seen": 2319712256
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002402086258776329,
+ "loss": 1.2413,
+ "theoretical_loss": 3.4173022571796405,
+ "tokens_seen": 2319777792
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002401925777331996,
+ "loss": 1.1894,
+ "theoretical_loss": 3.417294505720932,
+ "tokens_seen": 2319843328
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002401765295887663,
+ "loss": 1.1982,
+ "theoretical_loss": 3.417286754542513,
+ "tokens_seen": 2319908864
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 775797,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4274017810821533,
+ "objective/train/theoretical_loss": 3.4172790036443654,
+ "objective/train/tokens_used": 2340434400,
+ "theoretical_loss": 3.4172790036443654,
+ "tokens_seen": 2319974400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000240160481444333,
+ "loss": 1.2221,
+ "theoretical_loss": 3.4172790036443654,
+ "tokens_seen": 2319974400
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002401444332998997,
+ "loss": 1.1468,
+ "theoretical_loss": 3.4172712530264713,
+ "tokens_seen": 2320039936
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002401283851554664,
+ "loss": 1.2992,
+ "theoretical_loss": 3.4172635026888125,
+ "tokens_seen": 2320105472
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002401123370110331,
+ "loss": 1.271,
+ "theoretical_loss": 3.4172557526313714,
+ "tokens_seen": 2320171008
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400962888665998,
+ "loss": 1.2403,
+ "theoretical_loss": 3.417248002854129,
+ "tokens_seen": 2320236544
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400802407221665,
+ "loss": 1.2454,
+ "theoretical_loss": 3.4172402533570683,
+ "tokens_seen": 2320302080
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400641925777332,
+ "loss": 1.2685,
+ "theoretical_loss": 3.4172325041401703,
+ "tokens_seen": 2320367616
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400481444332999,
+ "loss": 1.23,
+ "theoretical_loss": 3.4172247552034176,
+ "tokens_seen": 2320433152
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400320962888666,
+ "loss": 1.2113,
+ "theoretical_loss": 3.417217006546792,
+ "tokens_seen": 2320498688
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002400160481444333,
+ "loss": 1.2328,
+ "theoretical_loss": 3.4172092581702755,
+ "tokens_seen": 2320564224
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00024,
+ "loss": 1.2716,
+ "theoretical_loss": 3.4172015100738498,
+ "tokens_seen": 2320629760
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399839518555667,
+ "loss": 1.1896,
+ "theoretical_loss": 3.4171937622574973,
+ "tokens_seen": 2320695296
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399679037111334,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4171860147211994,
+ "tokens_seen": 2320760832
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399518555667001,
+ "loss": 1.2511,
+ "theoretical_loss": 3.417178267464939,
+ "tokens_seen": 2320826368
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399358074222668,
+ "loss": 1.1029,
+ "theoretical_loss": 3.417170520488697,
+ "tokens_seen": 2320891904
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399197592778335,
+ "loss": 1.2411,
+ "theoretical_loss": 3.417162773792456,
+ "tokens_seen": 2320957440
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002399037111334002,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4171550273761984,
+ "tokens_seen": 2321022976
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002398876629889669,
+ "loss": 1.2865,
+ "theoretical_loss": 3.4171472812399046,
+ "tokens_seen": 2321088512
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002398716148445336,
+ "loss": 1.1996,
+ "theoretical_loss": 3.4171395353835585,
+ "tokens_seen": 2321154048
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023985556670010028,
+ "loss": 1.2874,
+ "theoretical_loss": 3.4171317898071405,
+ "tokens_seen": 2321219584
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023983951855566699,
+ "loss": 1.2396,
+ "theoretical_loss": 3.4171240445106337,
+ "tokens_seen": 2321285120
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023982347041123369,
+ "loss": 1.2183,
+ "theoretical_loss": 3.417116299494019,
+ "tokens_seen": 2321350656
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002398074222668004,
+ "loss": 1.2177,
+ "theoretical_loss": 3.4171085547572795,
+ "tokens_seen": 2321416192
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397913741223671,
+ "loss": 1.2373,
+ "theoretical_loss": 3.4171008103003966,
+ "tokens_seen": 2321481728
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397753259779338,
+ "loss": 1.2279,
+ "theoretical_loss": 3.4170930661233525,
+ "tokens_seen": 2321547264
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397592778335005,
+ "loss": 1.2978,
+ "theoretical_loss": 3.417085322226129,
+ "tokens_seen": 2321612800
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397432296890672,
+ "loss": 1.2702,
+ "theoretical_loss": 3.417077578608708,
+ "tokens_seen": 2321678336
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397271815446339,
+ "loss": 1.2528,
+ "theoretical_loss": 3.4170698352710716,
+ "tokens_seen": 2321743872
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002397111334002006,
+ "loss": 1.2094,
+ "theoretical_loss": 3.4170620922132025,
+ "tokens_seen": 2321809408
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002396950852557673,
+ "loss": 1.1891,
+ "theoretical_loss": 3.417054349435081,
+ "tokens_seen": 2321874944
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000239679037111334,
+ "loss": 1.2374,
+ "theoretical_loss": 3.417046606936691,
+ "tokens_seen": 2321940480
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002396629889669007,
+ "loss": 1.2481,
+ "theoretical_loss": 3.4170388647180134,
+ "tokens_seen": 2322006016
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002396469408224674,
+ "loss": 1.2104,
+ "theoretical_loss": 3.41703112277903,
+ "tokens_seen": 2322071552
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023963089267803415,
+ "loss": 1.336,
+ "theoretical_loss": 3.417023381119724,
+ "tokens_seen": 2322137088
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023961484453360085,
+ "loss": 1.2554,
+ "theoretical_loss": 3.4170156397400766,
+ "tokens_seen": 2322202624
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023959879638916752,
+ "loss": 1.2703,
+ "theoretical_loss": 3.4170078986400694,
+ "tokens_seen": 2322268160
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023958274824473422,
+ "loss": 1.2716,
+ "theoretical_loss": 3.417000157819685,
+ "tokens_seen": 2322333696
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023956670010030092,
+ "loss": 1.2542,
+ "theoretical_loss": 3.4169924172789052,
+ "tokens_seen": 2322399232
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023955065195586762,
+ "loss": 1.2212,
+ "theoretical_loss": 3.4169846770177124,
+ "tokens_seen": 2322464768
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023953460381143432,
+ "loss": 1.2614,
+ "theoretical_loss": 3.416976937036088,
+ "tokens_seen": 2322530304
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023951855566700103,
+ "loss": 1.3248,
+ "theoretical_loss": 3.4169691973340144,
+ "tokens_seen": 2322595840
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023950250752256773,
+ "loss": 1.2007,
+ "theoretical_loss": 3.4169614579114733,
+ "tokens_seen": 2322661376
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023948645937813443,
+ "loss": 1.2513,
+ "theoretical_loss": 3.4169537187684473,
+ "tokens_seen": 2322726912
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023947041123370113,
+ "loss": 1.2448,
+ "theoretical_loss": 3.4169459799049178,
+ "tokens_seen": 2322792448
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023945436308926783,
+ "loss": 1.1328,
+ "theoretical_loss": 3.4169382413208673,
+ "tokens_seen": 2322857984
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023943831494483453,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4169305030162773,
+ "tokens_seen": 2322923520
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023942226680040123,
+ "loss": 1.2887,
+ "theoretical_loss": 3.4169227649911305,
+ "tokens_seen": 2322989056
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023940621865596793,
+ "loss": 1.2077,
+ "theoretical_loss": 3.4169150272454085,
+ "tokens_seen": 2323054592
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023939017051153463,
+ "loss": 1.26,
+ "theoretical_loss": 3.4169072897790933,
+ "tokens_seen": 2323120128
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023937412236710133,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4168995525921666,
+ "tokens_seen": 2323185664
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 777104,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3013246059417725,
+ "objective/train/theoretical_loss": 3.4168918156846115,
+ "objective/train/tokens_used": 2343711200,
+ "theoretical_loss": 3.4168918156846115,
+ "tokens_seen": 2323251200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023935807422266803,
+ "loss": 1.1993,
+ "theoretical_loss": 3.4168918156846115,
+ "tokens_seen": 2323251200
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023934202607823473,
+ "loss": 1.2379,
+ "theoretical_loss": 3.416884079056409,
+ "tokens_seen": 2323316736
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023932597793380143,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4168763427075417,
+ "tokens_seen": 2323382272
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023930992978936813,
+ "loss": 1.3569,
+ "theoretical_loss": 3.4168686066379914,
+ "tokens_seen": 2323447808
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023929388164493483,
+ "loss": 1.2539,
+ "theoretical_loss": 3.41686087084774,
+ "tokens_seen": 2323513344
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023927783350050153,
+ "loss": 1.2552,
+ "theoretical_loss": 3.41685313533677,
+ "tokens_seen": 2323578880
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023926178535606824,
+ "loss": 1.215,
+ "theoretical_loss": 3.4168454001050628,
+ "tokens_seen": 2323644416
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023924573721163494,
+ "loss": 1.1481,
+ "theoretical_loss": 3.4168376651526007,
+ "tokens_seen": 2323709952
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023922968906720164,
+ "loss": 1.0823,
+ "theoretical_loss": 3.4168299304793663,
+ "tokens_seen": 2323775488
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002392136409227683,
+ "loss": 1.2402,
+ "theoretical_loss": 3.416822196085341,
+ "tokens_seen": 2323841024
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000239197592778335,
+ "loss": 1.2029,
+ "theoretical_loss": 3.416814461970507,
+ "tokens_seen": 2323906560
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002391815446339017,
+ "loss": 1.239,
+ "theoretical_loss": 3.416806728134847,
+ "tokens_seen": 2323972096
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002391654964894684,
+ "loss": 1.1811,
+ "theoretical_loss": 3.4167989945783415,
+ "tokens_seen": 2324037632
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002391494483450351,
+ "loss": 1.2025,
+ "theoretical_loss": 3.4167912613009737,
+ "tokens_seen": 2324103168
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002391334002006018,
+ "loss": 1.1869,
+ "theoretical_loss": 3.4167835283027257,
+ "tokens_seen": 2324168704
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023911735205616851,
+ "loss": 1.2109,
+ "theoretical_loss": 3.4167757955835794,
+ "tokens_seen": 2324234240
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023910130391173521,
+ "loss": 1.2326,
+ "theoretical_loss": 3.4167680631435164,
+ "tokens_seen": 2324299776
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023908525576730192,
+ "loss": 1.3234,
+ "theoretical_loss": 3.4167603309825196,
+ "tokens_seen": 2324365312
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023906920762286862,
+ "loss": 1.2298,
+ "theoretical_loss": 3.41675259910057,
+ "tokens_seen": 2324430848
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023905315947843532,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4167448674976506,
+ "tokens_seen": 2324496384
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023903711133400202,
+ "loss": 1.2117,
+ "theoretical_loss": 3.416737136173743,
+ "tokens_seen": 2324561920
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023902106318956872,
+ "loss": 1.3468,
+ "theoretical_loss": 3.4167294051288293,
+ "tokens_seen": 2324627456
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023900501504513542,
+ "loss": 1.2653,
+ "theoretical_loss": 3.416721674362892,
+ "tokens_seen": 2324692992
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023898896690070212,
+ "loss": 1.2799,
+ "theoretical_loss": 3.416713943875912,
+ "tokens_seen": 2324758528
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023897291875626882,
+ "loss": 1.1944,
+ "theoretical_loss": 3.416706213667873,
+ "tokens_seen": 2324824064
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023895687061183552,
+ "loss": 1.2337,
+ "theoretical_loss": 3.416698483738756,
+ "tokens_seen": 2324889600
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023894082246740222,
+ "loss": 1.2246,
+ "theoretical_loss": 3.416690754088543,
+ "tokens_seen": 2324955136
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023892477432296892,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4166830247172166,
+ "tokens_seen": 2325020672
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023890872617853562,
+ "loss": 1.2872,
+ "theoretical_loss": 3.416675295624759,
+ "tokens_seen": 2325086208
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023889267803410232,
+ "loss": 1.2534,
+ "theoretical_loss": 3.4166675668111517,
+ "tokens_seen": 2325151744
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023887662988966902,
+ "loss": 1.2224,
+ "theoretical_loss": 3.4166598382763773,
+ "tokens_seen": 2325217280
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023886058174523572,
+ "loss": 1.1143,
+ "theoretical_loss": 3.4166521100204172,
+ "tokens_seen": 2325282816
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.00023884453360080242,
+ "loss": 1.2768,
+ "theoretical_loss": 3.4166443820432537,
+ "tokens_seen": 2325348352
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002388284854563691,
+ "loss": 1.158,
+ "theoretical_loss": 3.4166366543448694,
+ "tokens_seen": 2325413888
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002388124373119358,
+ "loss": 1.0745,
+ "theoretical_loss": 3.416628926925246,
+ "tokens_seen": 2325479424
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387963891675025,
+ "loss": 1.153,
+ "theoretical_loss": 3.416621199784366,
+ "tokens_seen": 2325544960
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387803410230692,
+ "loss": 1.3482,
+ "theoretical_loss": 3.416613472922211,
+ "tokens_seen": 2325610496
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387642928786359,
+ "loss": 1.2649,
+ "theoretical_loss": 3.4166057463387633,
+ "tokens_seen": 2325676032
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387482447342026,
+ "loss": 1.2798,
+ "theoretical_loss": 3.4165980200340047,
+ "tokens_seen": 2325741568
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387321965897693,
+ "loss": 1.2071,
+ "theoretical_loss": 3.4165902940079174,
+ "tokens_seen": 2325807104
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.000238716148445336,
+ "loss": 1.1782,
+ "theoretical_loss": 3.416582568260484,
+ "tokens_seen": 2325872640
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002387001003009027,
+ "loss": 1.2734,
+ "theoretical_loss": 3.416574842791686,
+ "tokens_seen": 2325938176
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386840521564694,
+ "loss": 1.2,
+ "theoretical_loss": 3.416567117601506,
+ "tokens_seen": 2326003712
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386680040120361,
+ "loss": 1.1891,
+ "theoretical_loss": 3.4165593926899254,
+ "tokens_seen": 2326069248
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386519558676028,
+ "loss": 1.249,
+ "theoretical_loss": 3.4165516680569272,
+ "tokens_seen": 2326134784
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386359077231695,
+ "loss": 1.2563,
+ "theoretical_loss": 3.4165439437024925,
+ "tokens_seen": 2326200320
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386198595787362,
+ "loss": 1.4344,
+ "theoretical_loss": 3.4165362196266043,
+ "tokens_seen": 2326265856
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002386038114343029,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4165284958292443,
+ "tokens_seen": 2326331392
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002385877632898696,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4165207723103945,
+ "tokens_seen": 2326396928
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 0.0002385717151454363,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4165130490700375,
+ "tokens_seen": 2326462464
+ },
+ {
+ "epoch": 0.7,
+ "objective/train/docs_used": 777937,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2629358768463135,
+ "objective/train/theoretical_loss": 3.4165053261081546,
+ "objective/train/tokens_used": 2346988000,
+ "theoretical_loss": 3.4165053261081546,
+ "tokens_seen": 2326528000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.000238555667001003,
+ "loss": 1.2397,
+ "theoretical_loss": 3.4165053261081546,
+ "tokens_seen": 2326528000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002385396188565697,
+ "loss": 1.1785,
+ "theoretical_loss": 3.416497603424729,
+ "tokens_seen": 2326593536
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002385235707121364,
+ "loss": 1.2778,
+ "theoretical_loss": 3.416489881019742,
+ "tokens_seen": 2326659072
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002385075225677031,
+ "loss": 1.2964,
+ "theoretical_loss": 3.4164821588931757,
+ "tokens_seen": 2326724608
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002384914744232698,
+ "loss": 1.2518,
+ "theoretical_loss": 3.4164744370450126,
+ "tokens_seen": 2326790144
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002384754262788365,
+ "loss": 1.2052,
+ "theoretical_loss": 3.416466715475235,
+ "tokens_seen": 2326855680
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002384593781344032,
+ "loss": 1.2613,
+ "theoretical_loss": 3.416458994183824,
+ "tokens_seen": 2326921216
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023844332998996989,
+ "loss": 1.2501,
+ "theoretical_loss": 3.416451273170763,
+ "tokens_seen": 2326986752
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023842728184553664,
+ "loss": 1.272,
+ "theoretical_loss": 3.4164435524360335,
+ "tokens_seen": 2327052288
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023841123370110334,
+ "loss": 1.2377,
+ "theoretical_loss": 3.4164358319796175,
+ "tokens_seen": 2327117824
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023839518555667004,
+ "loss": 1.2464,
+ "theoretical_loss": 3.4164281118014976,
+ "tokens_seen": 2327183360
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023837913741223674,
+ "loss": 1.2954,
+ "theoretical_loss": 3.4164203919016556,
+ "tokens_seen": 2327248896
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023836308926780344,
+ "loss": 1.2255,
+ "theoretical_loss": 3.416412672280073,
+ "tokens_seen": 2327314432
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023834704112337014,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4164049529367335,
+ "tokens_seen": 2327379968
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023833099297893684,
+ "loss": 1.1865,
+ "theoretical_loss": 3.416397233871618,
+ "tokens_seen": 2327445504
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023831494483450355,
+ "loss": 1.1867,
+ "theoretical_loss": 3.416389515084709,
+ "tokens_seen": 2327511040
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023829889669007025,
+ "loss": 1.2868,
+ "theoretical_loss": 3.4163817965759886,
+ "tokens_seen": 2327576576
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023828284854563695,
+ "loss": 1.2104,
+ "theoretical_loss": 3.416374078345439,
+ "tokens_seen": 2327642112
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023826680040120365,
+ "loss": 1.2571,
+ "theoretical_loss": 3.4163663603930425,
+ "tokens_seen": 2327707648
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023825075225677035,
+ "loss": 1.2077,
+ "theoretical_loss": 3.4163586427187806,
+ "tokens_seen": 2327773184
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023823470411233705,
+ "loss": 1.1783,
+ "theoretical_loss": 3.4163509253226363,
+ "tokens_seen": 2327838720
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023821865596790375,
+ "loss": 1.1714,
+ "theoretical_loss": 3.4163432082045913,
+ "tokens_seen": 2327904256
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023820260782347042,
+ "loss": 1.224,
+ "theoretical_loss": 3.4163354913646273,
+ "tokens_seen": 2327969792
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023818655967903712,
+ "loss": 1.3062,
+ "theoretical_loss": 3.4163277748027276,
+ "tokens_seen": 2328035328
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023817051153460382,
+ "loss": 1.3114,
+ "theoretical_loss": 3.4163200585188735,
+ "tokens_seen": 2328100864
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023815446339017052,
+ "loss": 1.2085,
+ "theoretical_loss": 3.4163123425130473,
+ "tokens_seen": 2328166400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023813841524573723,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4163046267852315,
+ "tokens_seen": 2328231936
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023812236710130393,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4162969113354076,
+ "tokens_seen": 2328297472
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023810631895687063,
+ "loss": 1.2111,
+ "theoretical_loss": 3.416289196163558,
+ "tokens_seen": 2328363008
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023809027081243733,
+ "loss": 1.243,
+ "theoretical_loss": 3.4162814812696656,
+ "tokens_seen": 2328428544
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023807422266800403,
+ "loss": 1.1425,
+ "theoretical_loss": 3.416273766653712,
+ "tokens_seen": 2328494080
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023805817452357073,
+ "loss": 1.1676,
+ "theoretical_loss": 3.4162660523156787,
+ "tokens_seen": 2328559616
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023804212637913743,
+ "loss": 1.254,
+ "theoretical_loss": 3.416258338255549,
+ "tokens_seen": 2328625152
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023802607823470413,
+ "loss": 1.1355,
+ "theoretical_loss": 3.416250624473304,
+ "tokens_seen": 2328690688
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023801003009027083,
+ "loss": 1.3077,
+ "theoretical_loss": 3.416242910968927,
+ "tokens_seen": 2328756224
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023799398194583753,
+ "loss": 1.2697,
+ "theoretical_loss": 3.4162351977423993,
+ "tokens_seen": 2328821760
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023797793380140423,
+ "loss": 1.2941,
+ "theoretical_loss": 3.4162274847937035,
+ "tokens_seen": 2328887296
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023796188565697093,
+ "loss": 1.245,
+ "theoretical_loss": 3.4162197721228216,
+ "tokens_seen": 2328952832
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023794583751253763,
+ "loss": 1.1329,
+ "theoretical_loss": 3.4162120597297356,
+ "tokens_seen": 2329018368
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023792978936810433,
+ "loss": 1.258,
+ "theoretical_loss": 3.4162043476144284,
+ "tokens_seen": 2329083904
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023791374122367103,
+ "loss": 1.1703,
+ "theoretical_loss": 3.4161966357768816,
+ "tokens_seen": 2329149440
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023789769307923773,
+ "loss": 1.2445,
+ "theoretical_loss": 3.416188924217077,
+ "tokens_seen": 2329214976
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023788164493480444,
+ "loss": 1.3041,
+ "theoretical_loss": 3.4161812129349975,
+ "tokens_seen": 2329280512
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023786559679037114,
+ "loss": 1.2789,
+ "theoretical_loss": 3.416173501930625,
+ "tokens_seen": 2329346048
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023784954864593784,
+ "loss": 1.2527,
+ "theoretical_loss": 3.416165791203942,
+ "tokens_seen": 2329411584
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023783350050150454,
+ "loss": 1.2361,
+ "theoretical_loss": 3.41615808075493,
+ "tokens_seen": 2329477120
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002378174523570712,
+ "loss": 1.1774,
+ "theoretical_loss": 3.416150370583572,
+ "tokens_seen": 2329542656
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002378014042126379,
+ "loss": 1.1653,
+ "theoretical_loss": 3.4161426606898493,
+ "tokens_seen": 2329608192
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002377853560682046,
+ "loss": 1.2219,
+ "theoretical_loss": 3.416134951073745,
+ "tokens_seen": 2329673728
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002377693079237713,
+ "loss": 1.3072,
+ "theoretical_loss": 3.416127241735241,
+ "tokens_seen": 2329739264
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 779113,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2098369598388672,
+ "objective/train/theoretical_loss": 3.416119532674319,
+ "objective/train/tokens_used": 2350264800,
+ "theoretical_loss": 3.416119532674319,
+ "tokens_seen": 2329804800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023775325977933801,
+ "loss": 1.2314,
+ "theoretical_loss": 3.416119532674319,
+ "tokens_seen": 2329804800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023773721163490471,
+ "loss": 1.1416,
+ "theoretical_loss": 3.4161118238909616,
+ "tokens_seen": 2329870336
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023772116349047141,
+ "loss": 1.2683,
+ "theoretical_loss": 3.416104115385151,
+ "tokens_seen": 2329935872
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023770511534603812,
+ "loss": 1.1155,
+ "theoretical_loss": 3.4160964071568696,
+ "tokens_seen": 2330001408
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023768906720160482,
+ "loss": 1.2445,
+ "theoretical_loss": 3.416088699206099,
+ "tokens_seen": 2330066944
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023767301905717152,
+ "loss": 1.1785,
+ "theoretical_loss": 3.416080991532822,
+ "tokens_seen": 2330132480
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023765697091273822,
+ "loss": 1.2796,
+ "theoretical_loss": 3.4160732841370205,
+ "tokens_seen": 2330198016
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023764092276830492,
+ "loss": 1.2182,
+ "theoretical_loss": 3.416065577018677,
+ "tokens_seen": 2330263552
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023762487462387162,
+ "loss": 1.2349,
+ "theoretical_loss": 3.416057870177773,
+ "tokens_seen": 2330329088
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023760882647943832,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4160501636142913,
+ "tokens_seen": 2330394624
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023759277833500502,
+ "loss": 1.2843,
+ "theoretical_loss": 3.4160424573282144,
+ "tokens_seen": 2330460160
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023757673019057172,
+ "loss": 1.259,
+ "theoretical_loss": 3.4160347513195237,
+ "tokens_seen": 2330525696
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023756068204613842,
+ "loss": 1.1507,
+ "theoretical_loss": 3.4160270455882022,
+ "tokens_seen": 2330591232
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023754463390170512,
+ "loss": 1.1515,
+ "theoretical_loss": 3.416019340134232,
+ "tokens_seen": 2330656768
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023752858575727182,
+ "loss": 1.2914,
+ "theoretical_loss": 3.4160116349575946,
+ "tokens_seen": 2330722304
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023751253761283852,
+ "loss": 1.2567,
+ "theoretical_loss": 3.416003930058273,
+ "tokens_seen": 2330787840
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023749648946840522,
+ "loss": 1.2134,
+ "theoretical_loss": 3.415996225436249,
+ "tokens_seen": 2330853376
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023748044132397192,
+ "loss": 1.1854,
+ "theoretical_loss": 3.4159885210915046,
+ "tokens_seen": 2330918912
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023746439317953863,
+ "loss": 1.1824,
+ "theoretical_loss": 3.415980817024023,
+ "tokens_seen": 2330984448
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023744834503510533,
+ "loss": 1.1746,
+ "theoretical_loss": 3.415973113233785,
+ "tokens_seen": 2331049984
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.000237432296890672,
+ "loss": 1.3284,
+ "theoretical_loss": 3.4159654097207746,
+ "tokens_seen": 2331115520
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002374162487462387,
+ "loss": 1.197,
+ "theoretical_loss": 3.4159577064849724,
+ "tokens_seen": 2331181056
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002374002006018054,
+ "loss": 1.2251,
+ "theoretical_loss": 3.415950003526362,
+ "tokens_seen": 2331246592
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373841524573721,
+ "loss": 1.1673,
+ "theoretical_loss": 3.415942300844924,
+ "tokens_seen": 2331312128
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373681043129388,
+ "loss": 1.2353,
+ "theoretical_loss": 3.415934598440642,
+ "tokens_seen": 2331377664
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373520561685055,
+ "loss": 1.1379,
+ "theoretical_loss": 3.415926896313498,
+ "tokens_seen": 2331443200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373360080240722,
+ "loss": 1.2357,
+ "theoretical_loss": 3.4159191944634735,
+ "tokens_seen": 2331508736
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373199598796389,
+ "loss": 1.2638,
+ "theoretical_loss": 3.415911492890552,
+ "tokens_seen": 2331574272
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002373039117352056,
+ "loss": 1.1949,
+ "theoretical_loss": 3.4159037915947144,
+ "tokens_seen": 2331639808
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002372878635907723,
+ "loss": 1.1427,
+ "theoretical_loss": 3.4158960905759437,
+ "tokens_seen": 2331705344
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.000237271815446339,
+ "loss": 1.2115,
+ "theoretical_loss": 3.4158883898342225,
+ "tokens_seen": 2331770880
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002372557673019057,
+ "loss": 1.3252,
+ "theoretical_loss": 3.4158806893695317,
+ "tokens_seen": 2331836416
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002372397191574724,
+ "loss": 1.1431,
+ "theoretical_loss": 3.415872989181855,
+ "tokens_seen": 2331901952
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023722367101303916,
+ "loss": 1.313,
+ "theoretical_loss": 3.415865289271174,
+ "tokens_seen": 2331967488
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023720762286860586,
+ "loss": 1.195,
+ "theoretical_loss": 3.4158575896374708,
+ "tokens_seen": 2332033024
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023719157472417256,
+ "loss": 1.1401,
+ "theoretical_loss": 3.415849890280728,
+ "tokens_seen": 2332098560
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023717552657973924,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4158421912009276,
+ "tokens_seen": 2332164096
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023715947843530594,
+ "loss": 1.1656,
+ "theoretical_loss": 3.4158344923980524,
+ "tokens_seen": 2332229632
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023714343029087264,
+ "loss": 1.2996,
+ "theoretical_loss": 3.4158267938720837,
+ "tokens_seen": 2332295168
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023712738214643934,
+ "loss": 1.2282,
+ "theoretical_loss": 3.4158190956230046,
+ "tokens_seen": 2332360704
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023711133400200604,
+ "loss": 1.2035,
+ "theoretical_loss": 3.415811397650797,
+ "tokens_seen": 2332426240
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023709528585757274,
+ "loss": 1.2083,
+ "theoretical_loss": 3.4158036999554433,
+ "tokens_seen": 2332491776
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023707923771313944,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4157960025369256,
+ "tokens_seen": 2332557312
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023706318956870614,
+ "loss": 1.0885,
+ "theoretical_loss": 3.415788305395226,
+ "tokens_seen": 2332622848
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023704714142427284,
+ "loss": 1.2815,
+ "theoretical_loss": 3.4157806085303273,
+ "tokens_seen": 2332688384
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023703109327983954,
+ "loss": 1.0539,
+ "theoretical_loss": 3.415772911942211,
+ "tokens_seen": 2332753920
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023701504513540624,
+ "loss": 1.2252,
+ "theoretical_loss": 3.41576521563086,
+ "tokens_seen": 2332819456
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023699899699097294,
+ "loss": 1.2041,
+ "theoretical_loss": 3.415757519596257,
+ "tokens_seen": 2332884992
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023698294884653964,
+ "loss": 1.1514,
+ "theoretical_loss": 3.415749823838383,
+ "tokens_seen": 2332950528
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023696690070210634,
+ "loss": 1.3344,
+ "theoretical_loss": 3.4157421283572216,
+ "tokens_seen": 2333016064
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 779969,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.34850013256073,
+ "objective/train/theoretical_loss": 3.415734433152754,
+ "objective/train/tokens_used": 2353541600,
+ "theoretical_loss": 3.415734433152754,
+ "tokens_seen": 2333081600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023695085255767305,
+ "loss": 1.1848,
+ "theoretical_loss": 3.415734433152754,
+ "tokens_seen": 2333081600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023693480441323975,
+ "loss": 1.2165,
+ "theoretical_loss": 3.4157267382249636,
+ "tokens_seen": 2333147136
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023691875626880645,
+ "loss": 1.141,
+ "theoretical_loss": 3.4157190435738314,
+ "tokens_seen": 2333212672
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023690270812437315,
+ "loss": 1.289,
+ "theoretical_loss": 3.4157113491993405,
+ "tokens_seen": 2333278208
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023688665997993985,
+ "loss": 1.1714,
+ "theoretical_loss": 3.415703655101473,
+ "tokens_seen": 2333343744
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023687061183550655,
+ "loss": 1.1581,
+ "theoretical_loss": 3.415695961280211,
+ "tokens_seen": 2333409280
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023685456369107325,
+ "loss": 1.1589,
+ "theoretical_loss": 3.415688267735537,
+ "tokens_seen": 2333474816
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023683851554663995,
+ "loss": 1.319,
+ "theoretical_loss": 3.4156805744674337,
+ "tokens_seen": 2333540352
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023682246740220665,
+ "loss": 1.224,
+ "theoretical_loss": 3.4156728814758823,
+ "tokens_seen": 2333605888
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023680641925777335,
+ "loss": 1.2391,
+ "theoretical_loss": 3.415665188760866,
+ "tokens_seen": 2333671424
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023679037111334002,
+ "loss": 1.2196,
+ "theoretical_loss": 3.4156574963223667,
+ "tokens_seen": 2333736960
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023677432296890673,
+ "loss": 1.2348,
+ "theoretical_loss": 3.415649804160367,
+ "tokens_seen": 2333802496
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023675827482447343,
+ "loss": 1.2407,
+ "theoretical_loss": 3.415642112274849,
+ "tokens_seen": 2333868032
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023674222668004013,
+ "loss": 1.3505,
+ "theoretical_loss": 3.4156344206657945,
+ "tokens_seen": 2333933568
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023672617853560683,
+ "loss": 1.1753,
+ "theoretical_loss": 3.4156267293331872,
+ "tokens_seen": 2333999104
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023671013039117353,
+ "loss": 1.2209,
+ "theoretical_loss": 3.4156190382770077,
+ "tokens_seen": 2334064640
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023669408224674023,
+ "loss": 1.2361,
+ "theoretical_loss": 3.4156113474972396,
+ "tokens_seen": 2334130176
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023667803410230693,
+ "loss": 1.1704,
+ "theoretical_loss": 3.4156036569938646,
+ "tokens_seen": 2334195712
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023666198595787363,
+ "loss": 1.1813,
+ "theoretical_loss": 3.4155959667668654,
+ "tokens_seen": 2334261248
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023664593781344033,
+ "loss": 1.2015,
+ "theoretical_loss": 3.415588276816224,
+ "tokens_seen": 2334326784
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023662988966900703,
+ "loss": 1.2214,
+ "theoretical_loss": 3.415580587141922,
+ "tokens_seen": 2334392320
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023661384152457373,
+ "loss": 1.2365,
+ "theoretical_loss": 3.4155728977439432,
+ "tokens_seen": 2334457856
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023659779338014043,
+ "loss": 1.2475,
+ "theoretical_loss": 3.415565208622269,
+ "tokens_seen": 2334523392
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023658174523570713,
+ "loss": 1.1534,
+ "theoretical_loss": 3.415557519776882,
+ "tokens_seen": 2334588928
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023656569709127383,
+ "loss": 1.276,
+ "theoretical_loss": 3.4155498312077643,
+ "tokens_seen": 2334654464
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023654964894684053,
+ "loss": 1.1967,
+ "theoretical_loss": 3.4155421429148984,
+ "tokens_seen": 2334720000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023653360080240723,
+ "loss": 1.1333,
+ "theoretical_loss": 3.4155344548982667,
+ "tokens_seen": 2334785536
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023651755265797394,
+ "loss": 1.2213,
+ "theoretical_loss": 3.415526767157851,
+ "tokens_seen": 2334851072
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023650150451354064,
+ "loss": 1.2263,
+ "theoretical_loss": 3.415519079693634,
+ "tokens_seen": 2334916608
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023648545636910734,
+ "loss": 1.2838,
+ "theoretical_loss": 3.415511392505598,
+ "tokens_seen": 2334982144
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023646940822467404,
+ "loss": 1.2802,
+ "theoretical_loss": 3.415503705593726,
+ "tokens_seen": 2335047680
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023645336008024074,
+ "loss": 1.2428,
+ "theoretical_loss": 3.415496018957999,
+ "tokens_seen": 2335113216
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023643731193580744,
+ "loss": 1.2283,
+ "theoretical_loss": 3.4154883325984002,
+ "tokens_seen": 2335178752
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023642126379137414,
+ "loss": 1.2336,
+ "theoretical_loss": 3.415480646514912,
+ "tokens_seen": 2335244288
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002364052156469408,
+ "loss": 1.3312,
+ "theoretical_loss": 3.415472960707516,
+ "tokens_seen": 2335309824
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002363891675025075,
+ "loss": 1.2251,
+ "theoretical_loss": 3.4154652751761954,
+ "tokens_seen": 2335375360
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023637311935807421,
+ "loss": 1.2917,
+ "theoretical_loss": 3.415457589920932,
+ "tokens_seen": 2335440896
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023635707121364091,
+ "loss": 1.1916,
+ "theoretical_loss": 3.415449904941708,
+ "tokens_seen": 2335506432
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023634102306920762,
+ "loss": 1.1994,
+ "theoretical_loss": 3.4154422202385066,
+ "tokens_seen": 2335571968
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023632497492477432,
+ "loss": 1.1953,
+ "theoretical_loss": 3.4154345358113094,
+ "tokens_seen": 2335637504
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023630892678034102,
+ "loss": 1.2497,
+ "theoretical_loss": 3.4154268516600985,
+ "tokens_seen": 2335703040
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023629287863590772,
+ "loss": 1.1955,
+ "theoretical_loss": 3.415419167784857,
+ "tokens_seen": 2335768576
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023627683049147442,
+ "loss": 1.2891,
+ "theoretical_loss": 3.415411484185567,
+ "tokens_seen": 2335834112
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023626078234704112,
+ "loss": 1.2492,
+ "theoretical_loss": 3.4154038008622103,
+ "tokens_seen": 2335899648
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023624473420260782,
+ "loss": 1.1559,
+ "theoretical_loss": 3.41539611781477,
+ "tokens_seen": 2335965184
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023622868605817452,
+ "loss": 1.2286,
+ "theoretical_loss": 3.4153884350432278,
+ "tokens_seen": 2336030720
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023621263791374122,
+ "loss": 1.2079,
+ "theoretical_loss": 3.415380752547567,
+ "tokens_seen": 2336096256
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023619658976930792,
+ "loss": 1.2148,
+ "theoretical_loss": 3.415373070327769,
+ "tokens_seen": 2336161792
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023618054162487462,
+ "loss": 1.2845,
+ "theoretical_loss": 3.415365388383816,
+ "tokens_seen": 2336227328
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023616449348044132,
+ "loss": 1.2171,
+ "theoretical_loss": 3.415357706715692,
+ "tokens_seen": 2336292864
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 781226,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1148186922073364,
+ "objective/train/theoretical_loss": 3.4153500253233773,
+ "objective/train/tokens_used": 2356818400,
+ "theoretical_loss": 3.4153500253233773,
+ "tokens_seen": 2336358400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023614844533600802,
+ "loss": 1.1695,
+ "theoretical_loss": 3.4153500253233773,
+ "tokens_seen": 2336358400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023613239719157472,
+ "loss": 1.3498,
+ "theoretical_loss": 3.415342344206856,
+ "tokens_seen": 2336423936
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023611634904714142,
+ "loss": 1.2823,
+ "theoretical_loss": 3.415334663366109,
+ "tokens_seen": 2336489472
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023610030090270812,
+ "loss": 1.0882,
+ "theoretical_loss": 3.415326982801119,
+ "tokens_seen": 2336555008
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023608425275827483,
+ "loss": 1.2933,
+ "theoretical_loss": 3.415319302511869,
+ "tokens_seen": 2336620544
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023606820461384153,
+ "loss": 1.1677,
+ "theoretical_loss": 3.4153116224983417,
+ "tokens_seen": 2336686080
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023605215646940823,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4153039427605183,
+ "tokens_seen": 2336751616
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023603610832497493,
+ "loss": 1.1604,
+ "theoretical_loss": 3.4152962632983814,
+ "tokens_seen": 2336817152
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023602006018054166,
+ "loss": 1.2628,
+ "theoretical_loss": 3.4152885841119143,
+ "tokens_seen": 2336882688
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023600401203610836,
+ "loss": 1.1698,
+ "theoretical_loss": 3.415280905201098,
+ "tokens_seen": 2336948224
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023598796389167506,
+ "loss": 1.0888,
+ "theoretical_loss": 3.4152732265659163,
+ "tokens_seen": 2337013760
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023597191574724176,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4152655482063503,
+ "tokens_seen": 2337079296
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023595586760280846,
+ "loss": 1.383,
+ "theoretical_loss": 3.4152578701223835,
+ "tokens_seen": 2337144832
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023593981945837516,
+ "loss": 1.2602,
+ "theoretical_loss": 3.415250192313997,
+ "tokens_seen": 2337210368
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023592377131394186,
+ "loss": 1.1818,
+ "theoretical_loss": 3.4152425147811747,
+ "tokens_seen": 2337275904
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023590772316950856,
+ "loss": 1.1751,
+ "theoretical_loss": 3.415234837523898,
+ "tokens_seen": 2337341440
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023589167502507526,
+ "loss": 1.1655,
+ "theoretical_loss": 3.415227160542149,
+ "tokens_seen": 2337406976
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023587562688064196,
+ "loss": 1.2185,
+ "theoretical_loss": 3.4152194838359113,
+ "tokens_seen": 2337472512
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023585957873620866,
+ "loss": 1.1316,
+ "theoretical_loss": 3.415211807405166,
+ "tokens_seen": 2337538048
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023584353059177536,
+ "loss": 1.2174,
+ "theoretical_loss": 3.4152041312498964,
+ "tokens_seen": 2337603584
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023582748244734206,
+ "loss": 1.2716,
+ "theoretical_loss": 3.4151964553700846,
+ "tokens_seen": 2337669120
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023581143430290876,
+ "loss": 1.1816,
+ "theoretical_loss": 3.4151887797657126,
+ "tokens_seen": 2337734656
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023579538615847546,
+ "loss": 1.1557,
+ "theoretical_loss": 3.4151811044367633,
+ "tokens_seen": 2337800192
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023577933801404214,
+ "loss": 1.1639,
+ "theoretical_loss": 3.415173429383219,
+ "tokens_seen": 2337865728
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023576328986960884,
+ "loss": 1.2042,
+ "theoretical_loss": 3.415165754605062,
+ "tokens_seen": 2337931264
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023574724172517554,
+ "loss": 1.1896,
+ "theoretical_loss": 3.415158080102275,
+ "tokens_seen": 2337996800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023573119358074224,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4151504058748396,
+ "tokens_seen": 2338062336
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023571514543630894,
+ "loss": 1.208,
+ "theoretical_loss": 3.4151427319227388,
+ "tokens_seen": 2338127872
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023569909729187564,
+ "loss": 1.1695,
+ "theoretical_loss": 3.4151350582459554,
+ "tokens_seen": 2338193408
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023568304914744234,
+ "loss": 1.3035,
+ "theoretical_loss": 3.415127384844471,
+ "tokens_seen": 2338258944
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023566700100300904,
+ "loss": 1.2315,
+ "theoretical_loss": 3.4151197117182686,
+ "tokens_seen": 2338324480
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023565095285857574,
+ "loss": 1.2985,
+ "theoretical_loss": 3.41511203886733,
+ "tokens_seen": 2338390016
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023563490471414244,
+ "loss": 1.2176,
+ "theoretical_loss": 3.4151043662916383,
+ "tokens_seen": 2338455552
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023561885656970914,
+ "loss": 1.1845,
+ "theoretical_loss": 3.4150966939911753,
+ "tokens_seen": 2338521088
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023560280842527584,
+ "loss": 1.2156,
+ "theoretical_loss": 3.415089021965924,
+ "tokens_seen": 2338586624
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023558676028084255,
+ "loss": 1.2396,
+ "theoretical_loss": 3.4150813502158663,
+ "tokens_seen": 2338652160
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023557071213640925,
+ "loss": 1.2638,
+ "theoretical_loss": 3.4150736787409848,
+ "tokens_seen": 2338717696
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023555466399197595,
+ "loss": 1.1818,
+ "theoretical_loss": 3.415066007541262,
+ "tokens_seen": 2338783232
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023553861584754265,
+ "loss": 1.3052,
+ "theoretical_loss": 3.4150583366166805,
+ "tokens_seen": 2338848768
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023552256770310935,
+ "loss": 1.2165,
+ "theoretical_loss": 3.4150506659672226,
+ "tokens_seen": 2338914304
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023550651955867605,
+ "loss": 1.2539,
+ "theoretical_loss": 3.41504299559287,
+ "tokens_seen": 2338979840
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023549047141424275,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4150353254936063,
+ "tokens_seen": 2339045376
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023547442326980945,
+ "loss": 1.1701,
+ "theoretical_loss": 3.4150276556694132,
+ "tokens_seen": 2339110912
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023545837512537615,
+ "loss": 1.1945,
+ "theoretical_loss": 3.4150199861202735,
+ "tokens_seen": 2339176448
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023544232698094285,
+ "loss": 1.1932,
+ "theoretical_loss": 3.415012316846169,
+ "tokens_seen": 2339241984
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023542627883650955,
+ "loss": 1.2808,
+ "theoretical_loss": 3.415004647847083,
+ "tokens_seen": 2339307520
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023541023069207625,
+ "loss": 1.0831,
+ "theoretical_loss": 3.4149969791229973,
+ "tokens_seen": 2339373056
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023539418254764293,
+ "loss": 1.1572,
+ "theoretical_loss": 3.414989310673894,
+ "tokens_seen": 2339438592
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023537813440320963,
+ "loss": 1.1823,
+ "theoretical_loss": 3.4149816424997566,
+ "tokens_seen": 2339504128
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023536208625877633,
+ "loss": 1.0633,
+ "theoretical_loss": 3.414973974600567,
+ "tokens_seen": 2339569664
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 782115,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.165141224861145,
+ "objective/train/theoretical_loss": 3.414966306976308,
+ "objective/train/tokens_used": 2360095200,
+ "theoretical_loss": 3.414966306976308,
+ "tokens_seen": 2339635200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023534603811434303,
+ "loss": 1.1655,
+ "theoretical_loss": 3.414966306976308,
+ "tokens_seen": 2339635200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023532998996990973,
+ "loss": 1.2571,
+ "theoretical_loss": 3.414958639626961,
+ "tokens_seen": 2339700736
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023531394182547643,
+ "loss": 1.2492,
+ "theoretical_loss": 3.41495097255251,
+ "tokens_seen": 2339766272
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023529789368104313,
+ "loss": 1.0723,
+ "theoretical_loss": 3.414943305752936,
+ "tokens_seen": 2339831808
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023528184553660983,
+ "loss": 1.1744,
+ "theoretical_loss": 3.4149356392282217,
+ "tokens_seen": 2339897344
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023526579739217653,
+ "loss": 1.212,
+ "theoretical_loss": 3.4149279729783504,
+ "tokens_seen": 2339962880
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023524974924774323,
+ "loss": 1.173,
+ "theoretical_loss": 3.414920307003304,
+ "tokens_seen": 2340028416
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023523370110330993,
+ "loss": 1.2268,
+ "theoretical_loss": 3.4149126413030646,
+ "tokens_seen": 2340093952
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023521765295887663,
+ "loss": 1.1959,
+ "theoretical_loss": 3.4149049758776155,
+ "tokens_seen": 2340159488
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023520160481444333,
+ "loss": 1.1722,
+ "theoretical_loss": 3.4148973107269383,
+ "tokens_seen": 2340225024
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023518555667001003,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4148896458510163,
+ "tokens_seen": 2340290560
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023516950852557673,
+ "loss": 1.2382,
+ "theoretical_loss": 3.414881981249831,
+ "tokens_seen": 2340356096
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023515346038114344,
+ "loss": 1.2398,
+ "theoretical_loss": 3.4148743169233655,
+ "tokens_seen": 2340421632
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023513741223671014,
+ "loss": 1.1895,
+ "theoretical_loss": 3.414866652871602,
+ "tokens_seen": 2340487168
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023512136409227684,
+ "loss": 1.2001,
+ "theoretical_loss": 3.4148589890945233,
+ "tokens_seen": 2340552704
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023510531594784354,
+ "loss": 1.1978,
+ "theoretical_loss": 3.414851325592112,
+ "tokens_seen": 2340618240
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023508926780341024,
+ "loss": 1.1235,
+ "theoretical_loss": 3.4148436623643494,
+ "tokens_seen": 2340683776
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023507321965897694,
+ "loss": 1.2128,
+ "theoretical_loss": 3.4148359994112196,
+ "tokens_seen": 2340749312
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023505717151454364,
+ "loss": 1.1256,
+ "theoretical_loss": 3.414828336732704,
+ "tokens_seen": 2340814848
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023504112337011034,
+ "loss": 1.2779,
+ "theoretical_loss": 3.414820674328785,
+ "tokens_seen": 2340880384
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023502507522567704,
+ "loss": 1.208,
+ "theoretical_loss": 3.4148130121994456,
+ "tokens_seen": 2340945920
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023500902708124371,
+ "loss": 1.2995,
+ "theoretical_loss": 3.4148053503446683,
+ "tokens_seen": 2341011456
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023499297893681041,
+ "loss": 1.2244,
+ "theoretical_loss": 3.414797688764435,
+ "tokens_seen": 2341076992
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023497693079237711,
+ "loss": 1.1991,
+ "theoretical_loss": 3.414790027458728,
+ "tokens_seen": 2341142528
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023496088264794382,
+ "loss": 1.1807,
+ "theoretical_loss": 3.414782366427531,
+ "tokens_seen": 2341208064
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023494483450351052,
+ "loss": 1.2335,
+ "theoretical_loss": 3.414774705670826,
+ "tokens_seen": 2341273600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023492878635907722,
+ "loss": 1.3009,
+ "theoretical_loss": 3.414767045188595,
+ "tokens_seen": 2341339136
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023491273821464392,
+ "loss": 1.1905,
+ "theoretical_loss": 3.4147593849808207,
+ "tokens_seen": 2341404672
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023489669007021062,
+ "loss": 1.2399,
+ "theoretical_loss": 3.4147517250474855,
+ "tokens_seen": 2341470208
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023488064192577732,
+ "loss": 1.2147,
+ "theoretical_loss": 3.414744065388572,
+ "tokens_seen": 2341535744
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023486459378134402,
+ "loss": 1.284,
+ "theoretical_loss": 3.414736406004063,
+ "tokens_seen": 2341601280
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023484854563691072,
+ "loss": 1.0707,
+ "theoretical_loss": 3.4147287468939407,
+ "tokens_seen": 2341666816
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023483249749247747,
+ "loss": 1.2434,
+ "theoretical_loss": 3.414721088058187,
+ "tokens_seen": 2341732352
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023481644934804418,
+ "loss": 1.2338,
+ "theoretical_loss": 3.4147134294967856,
+ "tokens_seen": 2341797888
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023480040120361088,
+ "loss": 1.207,
+ "theoretical_loss": 3.414705771209718,
+ "tokens_seen": 2341863424
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023478435305917758,
+ "loss": 1.168,
+ "theoretical_loss": 3.414698113196967,
+ "tokens_seen": 2341928960
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023476830491474425,
+ "loss": 1.3546,
+ "theoretical_loss": 3.4146904554585156,
+ "tokens_seen": 2341994496
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023475225677031095,
+ "loss": 1.3034,
+ "theoretical_loss": 3.4146827979943457,
+ "tokens_seen": 2342060032
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023473620862587765,
+ "loss": 1.244,
+ "theoretical_loss": 3.41467514080444,
+ "tokens_seen": 2342125568
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023472016048144435,
+ "loss": 1.2391,
+ "theoretical_loss": 3.4146674838887807,
+ "tokens_seen": 2342191104
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023470411233701105,
+ "loss": 1.3049,
+ "theoretical_loss": 3.414659827247351,
+ "tokens_seen": 2342256640
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023468806419257775,
+ "loss": 1.1441,
+ "theoretical_loss": 3.4146521708801325,
+ "tokens_seen": 2342322176
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023467201604814445,
+ "loss": 1.3079,
+ "theoretical_loss": 3.4146445147871085,
+ "tokens_seen": 2342387712
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023465596790371115,
+ "loss": 1.3107,
+ "theoretical_loss": 3.414636858968261,
+ "tokens_seen": 2342453248
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023463991975927786,
+ "loss": 1.2952,
+ "theoretical_loss": 3.414629203423573,
+ "tokens_seen": 2342518784
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023462387161484456,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4146215481530264,
+ "tokens_seen": 2342584320
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023460782347041126,
+ "loss": 1.2027,
+ "theoretical_loss": 3.4146138931566044,
+ "tokens_seen": 2342649856
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023459177532597796,
+ "loss": 1.1749,
+ "theoretical_loss": 3.4146062384342892,
+ "tokens_seen": 2342715392
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023457572718154466,
+ "loss": 1.1367,
+ "theoretical_loss": 3.414598583986063,
+ "tokens_seen": 2342780928
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023455967903711136,
+ "loss": 1.1252,
+ "theoretical_loss": 3.4145909298119084,
+ "tokens_seen": 2342846464
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 783307,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.293138861656189,
+ "objective/train/theoretical_loss": 3.4145832759118084,
+ "objective/train/tokens_used": 2363372000,
+ "theoretical_loss": 3.4145832759118084,
+ "tokens_seen": 2342912000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023454363089267806,
+ "loss": 1.2362,
+ "theoretical_loss": 3.4145832759118084,
+ "tokens_seen": 2342912000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023452758274824476,
+ "loss": 1.1641,
+ "theoretical_loss": 3.4145756222857453,
+ "tokens_seen": 2342977536
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023451153460381146,
+ "loss": 1.1295,
+ "theoretical_loss": 3.414567968933701,
+ "tokens_seen": 2343043072
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023449548645937816,
+ "loss": 1.3184,
+ "theoretical_loss": 3.4145603158556592,
+ "tokens_seen": 2343108608
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023447943831494486,
+ "loss": 1.1716,
+ "theoretical_loss": 3.4145526630516017,
+ "tokens_seen": 2343174144
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023446339017051156,
+ "loss": 1.2537,
+ "theoretical_loss": 3.414545010521511,
+ "tokens_seen": 2343239680
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023444734202607826,
+ "loss": 1.2144,
+ "theoretical_loss": 3.41453735826537,
+ "tokens_seen": 2343305216
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023443129388164496,
+ "loss": 1.1962,
+ "theoretical_loss": 3.4145297062831608,
+ "tokens_seen": 2343370752
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023441524573721166,
+ "loss": 1.2483,
+ "theoretical_loss": 3.4145220545748662,
+ "tokens_seen": 2343436288
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023439919759277836,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4145144031404686,
+ "tokens_seen": 2343501824
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023438314944834504,
+ "loss": 1.1629,
+ "theoretical_loss": 3.4145067519799506,
+ "tokens_seen": 2343567360
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023436710130391174,
+ "loss": 1.2492,
+ "theoretical_loss": 3.414499101093295,
+ "tokens_seen": 2343632896
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023435105315947844,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4144914504804835,
+ "tokens_seen": 2343698432
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023433500501504514,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4144838001414994,
+ "tokens_seen": 2343763968
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023431895687061184,
+ "loss": 1.2527,
+ "theoretical_loss": 3.4144761500763257,
+ "tokens_seen": 2343829504
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023430290872617854,
+ "loss": 1.1994,
+ "theoretical_loss": 3.4144685002849435,
+ "tokens_seen": 2343895040
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023428686058174524,
+ "loss": 1.1923,
+ "theoretical_loss": 3.4144608507673366,
+ "tokens_seen": 2343960576
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023427081243731194,
+ "loss": 1.2197,
+ "theoretical_loss": 3.414453201523487,
+ "tokens_seen": 2344026112
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023425476429287864,
+ "loss": 1.2069,
+ "theoretical_loss": 3.4144455525533774,
+ "tokens_seen": 2344091648
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023423871614844534,
+ "loss": 1.1677,
+ "theoretical_loss": 3.4144379038569905,
+ "tokens_seen": 2344157184
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023422266800401204,
+ "loss": 1.3352,
+ "theoretical_loss": 3.4144302554343082,
+ "tokens_seen": 2344222720
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023420661985957875,
+ "loss": 1.1749,
+ "theoretical_loss": 3.4144226072853137,
+ "tokens_seen": 2344288256
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023419057171514545,
+ "loss": 1.2767,
+ "theoretical_loss": 3.4144149594099895,
+ "tokens_seen": 2344353792
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023417452357071215,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4144073118083176,
+ "tokens_seen": 2344419328
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023415847542627885,
+ "loss": 1.2943,
+ "theoretical_loss": 3.414399664480282,
+ "tokens_seen": 2344484864
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023414242728184555,
+ "loss": 1.2046,
+ "theoretical_loss": 3.414392017425863,
+ "tokens_seen": 2344550400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023412637913741225,
+ "loss": 1.2131,
+ "theoretical_loss": 3.414384370645045,
+ "tokens_seen": 2344615936
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023411033099297895,
+ "loss": 1.1986,
+ "theoretical_loss": 3.41437672413781,
+ "tokens_seen": 2344681472
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023409428284854565,
+ "loss": 1.2144,
+ "theoretical_loss": 3.4143690779041407,
+ "tokens_seen": 2344747008
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023407823470411235,
+ "loss": 1.2912,
+ "theoretical_loss": 3.414361431944019,
+ "tokens_seen": 2344812544
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023406218655967905,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4143537862574282,
+ "tokens_seen": 2344878080
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023404613841524575,
+ "loss": 1.1999,
+ "theoretical_loss": 3.4143461408443505,
+ "tokens_seen": 2344943616
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023403009027081245,
+ "loss": 1.2522,
+ "theoretical_loss": 3.414338495704769,
+ "tokens_seen": 2345009152
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023401404212637915,
+ "loss": 1.2103,
+ "theoretical_loss": 3.414330850838666,
+ "tokens_seen": 2345074688
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023399799398194583,
+ "loss": 1.1813,
+ "theoretical_loss": 3.4143232062460234,
+ "tokens_seen": 2345140224
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023398194583751253,
+ "loss": 1.2397,
+ "theoretical_loss": 3.4143155619268244,
+ "tokens_seen": 2345205760
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023396589769307923,
+ "loss": 1.2384,
+ "theoretical_loss": 3.414307917881052,
+ "tokens_seen": 2345271296
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023394984954864593,
+ "loss": 1.2147,
+ "theoretical_loss": 3.4143002741086876,
+ "tokens_seen": 2345336832
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023393380140421263,
+ "loss": 1.306,
+ "theoretical_loss": 3.414292630609715,
+ "tokens_seen": 2345402368
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023391775325977933,
+ "loss": 1.1879,
+ "theoretical_loss": 3.414284987384116,
+ "tokens_seen": 2345467904
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023390170511534603,
+ "loss": 1.2373,
+ "theoretical_loss": 3.414277344431874,
+ "tokens_seen": 2345533440
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023388565697091273,
+ "loss": 1.2236,
+ "theoretical_loss": 3.4142697017529704,
+ "tokens_seen": 2345598976
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023386960882647943,
+ "loss": 1.2648,
+ "theoretical_loss": 3.4142620593473887,
+ "tokens_seen": 2345664512
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023385356068204613,
+ "loss": 1.1546,
+ "theoretical_loss": 3.414254417215111,
+ "tokens_seen": 2345730048
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023383751253761283,
+ "loss": 1.2093,
+ "theoretical_loss": 3.4142467753561205,
+ "tokens_seen": 2345795584
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023382146439317953,
+ "loss": 1.2563,
+ "theoretical_loss": 3.414239133770399,
+ "tokens_seen": 2345861120
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023380541624874623,
+ "loss": 1.2515,
+ "theoretical_loss": 3.4142314924579296,
+ "tokens_seen": 2345926656
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023378936810431293,
+ "loss": 1.1926,
+ "theoretical_loss": 3.414223851418695,
+ "tokens_seen": 2345992192
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023377331995987964,
+ "loss": 1.2683,
+ "theoretical_loss": 3.414216210652677,
+ "tokens_seen": 2346057728
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023375727181544634,
+ "loss": 1.2123,
+ "theoretical_loss": 3.4142085701598592,
+ "tokens_seen": 2346123264
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 784199,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3218762874603271,
+ "objective/train/theoretical_loss": 3.414200929940224,
+ "objective/train/tokens_used": 2366648800,
+ "theoretical_loss": 3.414200929940224,
+ "tokens_seen": 2346188800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023374122367101304,
+ "loss": 1.3144,
+ "theoretical_loss": 3.414200929940224,
+ "tokens_seen": 2346188800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023372517552657974,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4141932899937535,
+ "tokens_seen": 2346254336
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023370912738214644,
+ "loss": 1.2883,
+ "theoretical_loss": 3.4141856503204306,
+ "tokens_seen": 2346319872
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023369307923771314,
+ "loss": 1.2402,
+ "theoretical_loss": 3.4141780109202378,
+ "tokens_seen": 2346385408
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023367703109327984,
+ "loss": 1.2103,
+ "theoretical_loss": 3.414170371793158,
+ "tokens_seen": 2346450944
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023366098294884654,
+ "loss": 1.2279,
+ "theoretical_loss": 3.4141627329391735,
+ "tokens_seen": 2346516480
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023364493480441324,
+ "loss": 1.1865,
+ "theoretical_loss": 3.414155094358267,
+ "tokens_seen": 2346582016
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023362888665997997,
+ "loss": 1.2566,
+ "theoretical_loss": 3.4141474560504212,
+ "tokens_seen": 2346647552
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023361283851554667,
+ "loss": 1.2841,
+ "theoretical_loss": 3.4141398180156184,
+ "tokens_seen": 2346713088
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023359679037111337,
+ "loss": 1.3069,
+ "theoretical_loss": 3.4141321802538416,
+ "tokens_seen": 2346778624
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023358074222668007,
+ "loss": 1.2193,
+ "theoretical_loss": 3.4141245427650735,
+ "tokens_seen": 2346844160
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023356469408224677,
+ "loss": 1.2489,
+ "theoretical_loss": 3.414116905549296,
+ "tokens_seen": 2346909696
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023354864593781347,
+ "loss": 1.2378,
+ "theoretical_loss": 3.4141092686064924,
+ "tokens_seen": 2346975232
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023353259779338017,
+ "loss": 1.2603,
+ "theoretical_loss": 3.4141016319366457,
+ "tokens_seen": 2347040768
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023351654964894687,
+ "loss": 1.1539,
+ "theoretical_loss": 3.414093995539737,
+ "tokens_seen": 2347106304
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023350050150451357,
+ "loss": 1.2245,
+ "theoretical_loss": 3.4140863594157507,
+ "tokens_seen": 2347171840
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023348445336008027,
+ "loss": 1.3484,
+ "theoretical_loss": 3.414078723564668,
+ "tokens_seen": 2347237376
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023346840521564697,
+ "loss": 1.1816,
+ "theoretical_loss": 3.4140710879864726,
+ "tokens_seen": 2347302912
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023345235707121368,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4140634526811464,
+ "tokens_seen": 2347368448
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023343630892678038,
+ "loss": 1.2833,
+ "theoretical_loss": 3.4140558176486726,
+ "tokens_seen": 2347433984
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023342026078234708,
+ "loss": 1.2545,
+ "theoretical_loss": 3.414048182889033,
+ "tokens_seen": 2347499520
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023340421263791378,
+ "loss": 1.2099,
+ "theoretical_loss": 3.414040548402211,
+ "tokens_seen": 2347565056
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023338816449348048,
+ "loss": 1.2287,
+ "theoretical_loss": 3.414032914188189,
+ "tokens_seen": 2347630592
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023337211634904718,
+ "loss": 1.3031,
+ "theoretical_loss": 3.4140252802469497,
+ "tokens_seen": 2347696128
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023335606820461385,
+ "loss": 1.3304,
+ "theoretical_loss": 3.4140176465784755,
+ "tokens_seen": 2347761664
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023334002006018055,
+ "loss": 1.1592,
+ "theoretical_loss": 3.4140100131827493,
+ "tokens_seen": 2347827200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023332397191574725,
+ "loss": 1.2097,
+ "theoretical_loss": 3.414002380059754,
+ "tokens_seen": 2347892736
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023330792377131395,
+ "loss": 1.1487,
+ "theoretical_loss": 3.4139947472094714,
+ "tokens_seen": 2347958272
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023329187562688065,
+ "loss": 1.1334,
+ "theoretical_loss": 3.413987114631885,
+ "tokens_seen": 2348023808
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023327582748244736,
+ "loss": 1.1415,
+ "theoretical_loss": 3.413979482326977,
+ "tokens_seen": 2348089344
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023325977933801406,
+ "loss": 1.3036,
+ "theoretical_loss": 3.41397185029473,
+ "tokens_seen": 2348154880
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023324373119358076,
+ "loss": 1.2213,
+ "theoretical_loss": 3.413964218535127,
+ "tokens_seen": 2348220416
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023322768304914746,
+ "loss": 1.2219,
+ "theoretical_loss": 3.41395658704815,
+ "tokens_seen": 2348285952
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023321163490471416,
+ "loss": 1.236,
+ "theoretical_loss": 3.4139489558337823,
+ "tokens_seen": 2348351488
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023319558676028086,
+ "loss": 1.1862,
+ "theoretical_loss": 3.4139413248920065,
+ "tokens_seen": 2348417024
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023317953861584756,
+ "loss": 1.2423,
+ "theoretical_loss": 3.4139336942228047,
+ "tokens_seen": 2348482560
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023316349047141426,
+ "loss": 1.2459,
+ "theoretical_loss": 3.4139260638261604,
+ "tokens_seen": 2348548096
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023314744232698096,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4139184337020554,
+ "tokens_seen": 2348613632
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023313139418254766,
+ "loss": 1.1317,
+ "theoretical_loss": 3.4139108038504733,
+ "tokens_seen": 2348679168
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023311534603811436,
+ "loss": 1.2063,
+ "theoretical_loss": 3.413903174271396,
+ "tokens_seen": 2348744704
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023309929789368106,
+ "loss": 1.1988,
+ "theoretical_loss": 3.4138955449648063,
+ "tokens_seen": 2348810240
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023308324974924776,
+ "loss": 1.1637,
+ "theoretical_loss": 3.4138879159306867,
+ "tokens_seen": 2348875776
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023306720160481446,
+ "loss": 1.2081,
+ "theoretical_loss": 3.413880287169021,
+ "tokens_seen": 2348941312
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023305115346038116,
+ "loss": 1.1337,
+ "theoretical_loss": 3.4138726586797903,
+ "tokens_seen": 2349006848
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023303510531594786,
+ "loss": 1.2688,
+ "theoretical_loss": 3.4138650304629783,
+ "tokens_seen": 2349072384
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023301905717151457,
+ "loss": 1.1101,
+ "theoretical_loss": 3.413857402518567,
+ "tokens_seen": 2349137920
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023300300902708127,
+ "loss": 1.1289,
+ "theoretical_loss": 3.4138497748465397,
+ "tokens_seen": 2349203456
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023298696088264797,
+ "loss": 1.2333,
+ "theoretical_loss": 3.4138421474468785,
+ "tokens_seen": 2349268992
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023297091273821464,
+ "loss": 1.2767,
+ "theoretical_loss": 3.413834520319567,
+ "tokens_seen": 2349334528
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023295486459378134,
+ "loss": 1.2191,
+ "theoretical_loss": 3.4138268934645866,
+ "tokens_seen": 2349400064
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 785442,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0939662456512451,
+ "objective/train/theoretical_loss": 3.413819266881921,
+ "objective/train/tokens_used": 2369925600,
+ "theoretical_loss": 3.413819266881921,
+ "tokens_seen": 2349465600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023293881644934804,
+ "loss": 1.1555,
+ "theoretical_loss": 3.413819266881921,
+ "tokens_seen": 2349465600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023292276830491474,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4138116405715526,
+ "tokens_seen": 2349531136
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023290672016048144,
+ "loss": 1.2048,
+ "theoretical_loss": 3.4138040145334636,
+ "tokens_seen": 2349596672
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023289067201604814,
+ "loss": 1.3099,
+ "theoretical_loss": 3.4137963887676372,
+ "tokens_seen": 2349662208
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023287462387161484,
+ "loss": 1.2072,
+ "theoretical_loss": 3.4137887632740562,
+ "tokens_seen": 2349727744
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023285857572718154,
+ "loss": 1.2773,
+ "theoretical_loss": 3.413781138052703,
+ "tokens_seen": 2349793280
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023284252758274825,
+ "loss": 1.2624,
+ "theoretical_loss": 3.41377351310356,
+ "tokens_seen": 2349858816
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023282647943831495,
+ "loss": 1.1926,
+ "theoretical_loss": 3.4137658884266107,
+ "tokens_seen": 2349924352
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023281043129388165,
+ "loss": 1.2331,
+ "theoretical_loss": 3.413758264021837,
+ "tokens_seen": 2349989888
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023279438314944835,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4137506398892223,
+ "tokens_seen": 2350055424
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023277833500501505,
+ "loss": 1.1192,
+ "theoretical_loss": 3.4137430160287483,
+ "tokens_seen": 2350120960
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023276228686058175,
+ "loss": 1.1886,
+ "theoretical_loss": 3.413735392440399,
+ "tokens_seen": 2350186496
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023274623871614845,
+ "loss": 1.1554,
+ "theoretical_loss": 3.413727769124156,
+ "tokens_seen": 2350252032
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023273019057171515,
+ "loss": 1.2255,
+ "theoretical_loss": 3.413720146080003,
+ "tokens_seen": 2350317568
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023271414242728185,
+ "loss": 1.2293,
+ "theoretical_loss": 3.4137125233079213,
+ "tokens_seen": 2350383104
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023269809428284855,
+ "loss": 1.3214,
+ "theoretical_loss": 3.4137049008078946,
+ "tokens_seen": 2350448640
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023268204613841525,
+ "loss": 1.2422,
+ "theoretical_loss": 3.413697278579906,
+ "tokens_seen": 2350514176
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023266599799398195,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4136896566239368,
+ "tokens_seen": 2350579712
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023264994984954865,
+ "loss": 1.2372,
+ "theoretical_loss": 3.413682034939971,
+ "tokens_seen": 2350645248
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023263390170511535,
+ "loss": 1.1928,
+ "theoretical_loss": 3.413674413527991,
+ "tokens_seen": 2350710784
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023261785356068205,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4136667923879793,
+ "tokens_seen": 2350776320
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023260180541624875,
+ "loss": 1.2787,
+ "theoretical_loss": 3.4136591715199183,
+ "tokens_seen": 2350841856
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023258575727181543,
+ "loss": 1.1926,
+ "theoretical_loss": 3.4136515509237912,
+ "tokens_seen": 2350907392
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023256970912738213,
+ "loss": 1.2183,
+ "theoretical_loss": 3.413643930599581,
+ "tokens_seen": 2350972928
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023255366098294883,
+ "loss": 1.3204,
+ "theoretical_loss": 3.4136363105472696,
+ "tokens_seen": 2351038464
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023253761283851553,
+ "loss": 1.2239,
+ "theoretical_loss": 3.4136286907668403,
+ "tokens_seen": 2351104000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023252156469408223,
+ "loss": 1.2035,
+ "theoretical_loss": 3.4136210712582757,
+ "tokens_seen": 2351169536
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023250551654964893,
+ "loss": 1.2894,
+ "theoretical_loss": 3.4136134520215586,
+ "tokens_seen": 2351235072
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023248946840521563,
+ "loss": 1.1739,
+ "theoretical_loss": 3.413605833056671,
+ "tokens_seen": 2351300608
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023247342026078233,
+ "loss": 1.2308,
+ "theoretical_loss": 3.4135982143635974,
+ "tokens_seen": 2351366144
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023245737211634903,
+ "loss": 1.3007,
+ "theoretical_loss": 3.4135905959423187,
+ "tokens_seen": 2351431680
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023244132397191573,
+ "loss": 1.2635,
+ "theoretical_loss": 3.413582977792818,
+ "tokens_seen": 2351497216
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002324252758274825,
+ "loss": 1.2194,
+ "theoretical_loss": 3.413575359915079,
+ "tokens_seen": 2351562752
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002324092276830492,
+ "loss": 1.193,
+ "theoretical_loss": 3.413567742309083,
+ "tokens_seen": 2351628288
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002323931795386159,
+ "loss": 1.2193,
+ "theoretical_loss": 3.413560124974814,
+ "tokens_seen": 2351693824
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002323771313941826,
+ "loss": 1.2819,
+ "theoretical_loss": 3.413552507912254,
+ "tokens_seen": 2351759360
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002323610832497493,
+ "loss": 1.2957,
+ "theoretical_loss": 3.4135448911213864,
+ "tokens_seen": 2351824896
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023234503510531596,
+ "loss": 1.2032,
+ "theoretical_loss": 3.4135372746021932,
+ "tokens_seen": 2351890432
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023232898696088267,
+ "loss": 1.121,
+ "theoretical_loss": 3.4135296583546575,
+ "tokens_seen": 2351955968
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023231293881644937,
+ "loss": 1.2737,
+ "theoretical_loss": 3.413522042378762,
+ "tokens_seen": 2352021504
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023229689067201607,
+ "loss": 1.2542,
+ "theoretical_loss": 3.4135144266744892,
+ "tokens_seen": 2352087040
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023228084252758277,
+ "loss": 1.2555,
+ "theoretical_loss": 3.413506811241822,
+ "tokens_seen": 2352152576
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023226479438314947,
+ "loss": 1.2045,
+ "theoretical_loss": 3.4134991960807435,
+ "tokens_seen": 2352218112
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023224874623871617,
+ "loss": 1.1975,
+ "theoretical_loss": 3.413491581191236,
+ "tokens_seen": 2352283648
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023223269809428287,
+ "loss": 1.3423,
+ "theoretical_loss": 3.413483966573283,
+ "tokens_seen": 2352349184
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023221664994984957,
+ "loss": 1.2084,
+ "theoretical_loss": 3.413476352226866,
+ "tokens_seen": 2352414720
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023220060180541627,
+ "loss": 1.245,
+ "theoretical_loss": 3.413468738151969,
+ "tokens_seen": 2352480256
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023218455366098297,
+ "loss": 1.1807,
+ "theoretical_loss": 3.4134611243485735,
+ "tokens_seen": 2352545792
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023216850551654967,
+ "loss": 1.2,
+ "theoretical_loss": 3.4134535108166633,
+ "tokens_seen": 2352611328
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023215245737211637,
+ "loss": 1.185,
+ "theoretical_loss": 3.4134458975562207,
+ "tokens_seen": 2352676864
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 786316,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2228635549545288,
+ "objective/train/theoretical_loss": 3.4134382845672286,
+ "objective/train/tokens_used": 2373202400,
+ "theoretical_loss": 3.4134382845672286,
+ "tokens_seen": 2352742400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023213640922768307,
+ "loss": 1.2712,
+ "theoretical_loss": 3.4134382845672286,
+ "tokens_seen": 2352742400
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023212036108324977,
+ "loss": 1.2324,
+ "theoretical_loss": 3.4134306718496696,
+ "tokens_seen": 2352807936
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023210431293881647,
+ "loss": 1.3016,
+ "theoretical_loss": 3.4134230594035264,
+ "tokens_seen": 2352873472
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023208826479438317,
+ "loss": 1.2406,
+ "theoretical_loss": 3.4134154472287825,
+ "tokens_seen": 2352939008
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023207221664994988,
+ "loss": 1.1648,
+ "theoretical_loss": 3.41340783532542,
+ "tokens_seen": 2353004544
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023205616850551658,
+ "loss": 1.2235,
+ "theoretical_loss": 3.4134002236934213,
+ "tokens_seen": 2353070080
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023204012036108328,
+ "loss": 1.2541,
+ "theoretical_loss": 3.41339261233277,
+ "tokens_seen": 2353135616
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023202407221664998,
+ "loss": 1.1816,
+ "theoretical_loss": 3.413385001243448,
+ "tokens_seen": 2353201152
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023200802407221668,
+ "loss": 1.1229,
+ "theoretical_loss": 3.413377390425439,
+ "tokens_seen": 2353266688
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023199197592778338,
+ "loss": 1.1939,
+ "theoretical_loss": 3.4133697798787255,
+ "tokens_seen": 2353332224
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023197592778335008,
+ "loss": 1.1541,
+ "theoretical_loss": 3.41336216960329,
+ "tokens_seen": 2353397760
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023195987963891675,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4133545595991155,
+ "tokens_seen": 2353463296
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023194383149448345,
+ "loss": 1.1853,
+ "theoretical_loss": 3.413346949866184,
+ "tokens_seen": 2353528832
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023192778335005015,
+ "loss": 1.2286,
+ "theoretical_loss": 3.41333934040448,
+ "tokens_seen": 2353594368
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023191173520561685,
+ "loss": 1.2181,
+ "theoretical_loss": 3.413331731213985,
+ "tokens_seen": 2353659904
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023189568706118356,
+ "loss": 1.2701,
+ "theoretical_loss": 3.4133241222946813,
+ "tokens_seen": 2353725440
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023187963891675026,
+ "loss": 1.2919,
+ "theoretical_loss": 3.413316513646553,
+ "tokens_seen": 2353790976
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023186359077231696,
+ "loss": 1.2744,
+ "theoretical_loss": 3.4133089052695818,
+ "tokens_seen": 2353856512
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023184754262788366,
+ "loss": 1.1734,
+ "theoretical_loss": 3.4133012971637515,
+ "tokens_seen": 2353922048
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023183149448345036,
+ "loss": 1.3188,
+ "theoretical_loss": 3.413293689329044,
+ "tokens_seen": 2353987584
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023181544633901706,
+ "loss": 1.2333,
+ "theoretical_loss": 3.4132860817654427,
+ "tokens_seen": 2354053120
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023179939819458376,
+ "loss": 1.2989,
+ "theoretical_loss": 3.4132784744729303,
+ "tokens_seen": 2354118656
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023178335005015046,
+ "loss": 1.2331,
+ "theoretical_loss": 3.413270867451489,
+ "tokens_seen": 2354184192
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023176730190571716,
+ "loss": 1.2665,
+ "theoretical_loss": 3.413263260701102,
+ "tokens_seen": 2354249728
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023175125376128386,
+ "loss": 1.2293,
+ "theoretical_loss": 3.4132556542217527,
+ "tokens_seen": 2354315264
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023173520561685056,
+ "loss": 1.3399,
+ "theoretical_loss": 3.4132480480134233,
+ "tokens_seen": 2354380800
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023171915747241726,
+ "loss": 1.1807,
+ "theoretical_loss": 3.4132404420760962,
+ "tokens_seen": 2354446336
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023170310932798396,
+ "loss": 1.2153,
+ "theoretical_loss": 3.4132328364097546,
+ "tokens_seen": 2354511872
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023168706118355066,
+ "loss": 1.1957,
+ "theoretical_loss": 3.413225231014382,
+ "tokens_seen": 2354577408
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023167101303911736,
+ "loss": 1.2111,
+ "theoretical_loss": 3.41321762588996,
+ "tokens_seen": 2354642944
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023165496489468407,
+ "loss": 1.3073,
+ "theoretical_loss": 3.413210021036472,
+ "tokens_seen": 2354708480
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023163891675025077,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4132024164539008,
+ "tokens_seen": 2354774016
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023162286860581747,
+ "loss": 1.2869,
+ "theoretical_loss": 3.413194812142229,
+ "tokens_seen": 2354839552
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023160682046138417,
+ "loss": 1.2907,
+ "theoretical_loss": 3.41318720810144,
+ "tokens_seen": 2354905088
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023159077231695087,
+ "loss": 1.1976,
+ "theoretical_loss": 3.413179604331516,
+ "tokens_seen": 2354970624
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023157472417251754,
+ "loss": 1.1686,
+ "theoretical_loss": 3.4131720008324398,
+ "tokens_seen": 2355036160
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023155867602808424,
+ "loss": 1.3465,
+ "theoretical_loss": 3.4131643976041945,
+ "tokens_seen": 2355101696
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023154262788365094,
+ "loss": 1.2006,
+ "theoretical_loss": 3.413156794646763,
+ "tokens_seen": 2355167232
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023152657973921764,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4131491919601276,
+ "tokens_seen": 2355232768
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023151053159478434,
+ "loss": 1.3158,
+ "theoretical_loss": 3.413141589544272,
+ "tokens_seen": 2355298304
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023149448345035104,
+ "loss": 1.2637,
+ "theoretical_loss": 3.413133987399178,
+ "tokens_seen": 2355363840
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023147843530591774,
+ "loss": 1.1707,
+ "theoretical_loss": 3.4131263855248295,
+ "tokens_seen": 2355429376
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023146238716148445,
+ "loss": 1.1645,
+ "theoretical_loss": 3.4131187839212083,
+ "tokens_seen": 2355494912
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023144633901705115,
+ "loss": 1.178,
+ "theoretical_loss": 3.4131111825882976,
+ "tokens_seen": 2355560448
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023143029087261785,
+ "loss": 1.231,
+ "theoretical_loss": 3.4131035815260806,
+ "tokens_seen": 2355625984
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023141424272818455,
+ "loss": 1.1212,
+ "theoretical_loss": 3.4130959807345396,
+ "tokens_seen": 2355691520
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023139819458375125,
+ "loss": 1.2088,
+ "theoretical_loss": 3.413088380213658,
+ "tokens_seen": 2355757056
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023138214643931795,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4130807799634177,
+ "tokens_seen": 2355822592
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023136609829488465,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4130731799838028,
+ "tokens_seen": 2355888128
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023135005015045135,
+ "loss": 1.1628,
+ "theoretical_loss": 3.413065580274795,
+ "tokens_seen": 2355953664
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 787703,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3746349811553955,
+ "objective/train/theoretical_loss": 3.4130579808363777,
+ "objective/train/tokens_used": 2376479200,
+ "theoretical_loss": 3.4130579808363777,
+ "tokens_seen": 2356019200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023133400200601805,
+ "loss": 1.3168,
+ "theoretical_loss": 3.4130579808363777,
+ "tokens_seen": 2356019200
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023131795386158475,
+ "loss": 1.2377,
+ "theoretical_loss": 3.413050381668534,
+ "tokens_seen": 2356084736
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023130190571715145,
+ "loss": 1.1867,
+ "theoretical_loss": 3.413042782771246,
+ "tokens_seen": 2356150272
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023128585757271815,
+ "loss": 1.2846,
+ "theoretical_loss": 3.413035184144497,
+ "tokens_seen": 2356215808
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023126980942828485,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4130275857882695,
+ "tokens_seen": 2356281344
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023125376128385155,
+ "loss": 1.2563,
+ "theoretical_loss": 3.4130199877025467,
+ "tokens_seen": 2356346880
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023123771313941825,
+ "loss": 1.231,
+ "theoretical_loss": 3.413012389887312,
+ "tokens_seen": 2356412416
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023122166499498498,
+ "loss": 1.3347,
+ "theoretical_loss": 3.413004792342547,
+ "tokens_seen": 2356477952
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023120561685055168,
+ "loss": 1.2702,
+ "theoretical_loss": 3.412997195068235,
+ "tokens_seen": 2356543488
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023118956870611838,
+ "loss": 1.2081,
+ "theoretical_loss": 3.4129895980643594,
+ "tokens_seen": 2356609024
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023117352056168508,
+ "loss": 1.135,
+ "theoretical_loss": 3.412982001330903,
+ "tokens_seen": 2356674560
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023115747241725178,
+ "loss": 1.263,
+ "theoretical_loss": 3.4129744048678474,
+ "tokens_seen": 2356740096
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023114142427281849,
+ "loss": 1.2018,
+ "theoretical_loss": 3.412966808675177,
+ "tokens_seen": 2356805632
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023112537612838519,
+ "loss": 1.2313,
+ "theoretical_loss": 3.412959212752874,
+ "tokens_seen": 2356871168
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002311093279839519,
+ "loss": 1.221,
+ "theoretical_loss": 3.412951617100921,
+ "tokens_seen": 2356936704
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002310932798395186,
+ "loss": 1.3483,
+ "theoretical_loss": 3.4129440217193014,
+ "tokens_seen": 2357002240
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002310772316950853,
+ "loss": 1.3294,
+ "theoretical_loss": 3.412936426607998,
+ "tokens_seen": 2357067776
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.000231061183550652,
+ "loss": 1.2395,
+ "theoretical_loss": 3.412928831766993,
+ "tokens_seen": 2357133312
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002310451354062187,
+ "loss": 1.3221,
+ "theoretical_loss": 3.41292123719627,
+ "tokens_seen": 2357198848
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002310290872617854,
+ "loss": 1.1959,
+ "theoretical_loss": 3.412913642895812,
+ "tokens_seen": 2357264384
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002310130391173521,
+ "loss": 1.2143,
+ "theoretical_loss": 3.412906048865601,
+ "tokens_seen": 2357329920
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002309969909729188,
+ "loss": 1.3165,
+ "theoretical_loss": 3.41289845510562,
+ "tokens_seen": 2357395456
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002309809428284855,
+ "loss": 1.224,
+ "theoretical_loss": 3.4128908616158524,
+ "tokens_seen": 2357460992
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002309648946840522,
+ "loss": 1.1545,
+ "theoretical_loss": 3.4128832683962815,
+ "tokens_seen": 2357526528
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.0002309488465396189,
+ "loss": 1.3043,
+ "theoretical_loss": 3.412875675446889,
+ "tokens_seen": 2357592064
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023093279839518557,
+ "loss": 1.2761,
+ "theoretical_loss": 3.412868082767659,
+ "tokens_seen": 2357657600
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023091675025075227,
+ "loss": 1.2987,
+ "theoretical_loss": 3.4128604903585726,
+ "tokens_seen": 2357723136
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023090070210631897,
+ "loss": 1.3419,
+ "theoretical_loss": 3.412852898219615,
+ "tokens_seen": 2357788672
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023088465396188567,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4128453063507673,
+ "tokens_seen": 2357854208
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023086860581745237,
+ "loss": 1.2646,
+ "theoretical_loss": 3.412837714752013,
+ "tokens_seen": 2357919744
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023085255767301907,
+ "loss": 1.2016,
+ "theoretical_loss": 3.412830123423335,
+ "tokens_seen": 2357985280
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023083650952858577,
+ "loss": 1.1522,
+ "theoretical_loss": 3.412822532364716,
+ "tokens_seen": 2358050816
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023082046138415247,
+ "loss": 1.2405,
+ "theoretical_loss": 3.4128149415761393,
+ "tokens_seen": 2358116352
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023080441323971917,
+ "loss": 1.2846,
+ "theoretical_loss": 3.4128073510575874,
+ "tokens_seen": 2358181888
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023078836509528587,
+ "loss": 1.257,
+ "theoretical_loss": 3.4127997608090435,
+ "tokens_seen": 2358247424
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023077231695085257,
+ "loss": 1.2674,
+ "theoretical_loss": 3.41279217083049,
+ "tokens_seen": 2358312960
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023075626880641927,
+ "loss": 1.3299,
+ "theoretical_loss": 3.41278458112191,
+ "tokens_seen": 2358378496
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023074022066198597,
+ "loss": 1.3131,
+ "theoretical_loss": 3.4127769916832866,
+ "tokens_seen": 2358444032
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023072417251755267,
+ "loss": 1.1599,
+ "theoretical_loss": 3.412769402514603,
+ "tokens_seen": 2358509568
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023070812437311938,
+ "loss": 1.2204,
+ "theoretical_loss": 3.412761813615841,
+ "tokens_seen": 2358575104
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023069207622868608,
+ "loss": 1.3053,
+ "theoretical_loss": 3.4127542249869847,
+ "tokens_seen": 2358640640
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023067602808425278,
+ "loss": 1.1934,
+ "theoretical_loss": 3.4127466366280164,
+ "tokens_seen": 2358706176
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023065997993981948,
+ "loss": 1.2287,
+ "theoretical_loss": 3.412739048538919,
+ "tokens_seen": 2358771712
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023064393179538618,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4127314607196753,
+ "tokens_seen": 2358837248
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023062788365095288,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4127238731702687,
+ "tokens_seen": 2358902784
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023061183550651958,
+ "loss": 1.147,
+ "theoretical_loss": 3.412716285890682,
+ "tokens_seen": 2358968320
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023059578736208628,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4127086988808975,
+ "tokens_seen": 2359033856
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023057973921765298,
+ "loss": 1.2203,
+ "theoretical_loss": 3.4127011121408986,
+ "tokens_seen": 2359099392
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023056369107321968,
+ "loss": 1.1661,
+ "theoretical_loss": 3.412693525670668,
+ "tokens_seen": 2359164928
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023054764292878635,
+ "loss": 1.1706,
+ "theoretical_loss": 3.412685939470189,
+ "tokens_seen": 2359230464
+ },
+ {
+ "epoch": 0.71,
+ "objective/train/docs_used": 788845,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.213777780532837,
+ "objective/train/theoretical_loss": 3.412678353539444,
+ "objective/train/tokens_used": 2379756000,
+ "theoretical_loss": 3.412678353539444,
+ "tokens_seen": 2359296000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023053159478435306,
+ "loss": 1.1956,
+ "theoretical_loss": 3.412678353539444,
+ "tokens_seen": 2359296000
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023051554663991976,
+ "loss": 1.1663,
+ "theoretical_loss": 3.4126707678784167,
+ "tokens_seen": 2359361536
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023049949849548646,
+ "loss": 1.2537,
+ "theoretical_loss": 3.412663182487089,
+ "tokens_seen": 2359427072
+ },
+ {
+ "epoch": 0.71,
+ "learning_rate": 0.00023048345035105316,
+ "loss": 1.2581,
+ "theoretical_loss": 3.4126555973654447,
+ "tokens_seen": 2359492608
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023046740220661986,
+ "loss": 1.2683,
+ "theoretical_loss": 3.412648012513466,
+ "tokens_seen": 2359558144
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023045135406218656,
+ "loss": 1.2868,
+ "theoretical_loss": 3.4126404279311364,
+ "tokens_seen": 2359623680
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023043530591775326,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4126328436184385,
+ "tokens_seen": 2359689216
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023041925777331996,
+ "loss": 1.1353,
+ "theoretical_loss": 3.4126252595753552,
+ "tokens_seen": 2359754752
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023040320962888666,
+ "loss": 1.1895,
+ "theoretical_loss": 3.41261767580187,
+ "tokens_seen": 2359820288
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023038716148445336,
+ "loss": 1.1818,
+ "theoretical_loss": 3.412610092297965,
+ "tokens_seen": 2359885824
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023037111334002006,
+ "loss": 1.2427,
+ "theoretical_loss": 3.412602509063624,
+ "tokens_seen": 2359951360
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023035506519558676,
+ "loss": 1.1537,
+ "theoretical_loss": 3.412594926098829,
+ "tokens_seen": 2360016896
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023033901705115346,
+ "loss": 1.1345,
+ "theoretical_loss": 3.412587343403563,
+ "tokens_seen": 2360082432
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023032296890672016,
+ "loss": 1.1786,
+ "theoretical_loss": 3.4125797609778097,
+ "tokens_seen": 2360147968
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023030692076228686,
+ "loss": 1.3433,
+ "theoretical_loss": 3.412572178821552,
+ "tokens_seen": 2360213504
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023029087261785356,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4125645969347724,
+ "tokens_seen": 2360279040
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023027482447342027,
+ "loss": 1.0895,
+ "theoretical_loss": 3.4125570153174536,
+ "tokens_seen": 2360344576
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023025877632898697,
+ "loss": 1.3115,
+ "theoretical_loss": 3.4125494339695788,
+ "tokens_seen": 2360410112
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023024272818455367,
+ "loss": 1.2873,
+ "theoretical_loss": 3.412541852891131,
+ "tokens_seen": 2360475648
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023022668004012037,
+ "loss": 1.3395,
+ "theoretical_loss": 3.4125342720820937,
+ "tokens_seen": 2360541184
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023021063189568707,
+ "loss": 1.1301,
+ "theoretical_loss": 3.412526691542449,
+ "tokens_seen": 2360606720
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023019458375125377,
+ "loss": 1.197,
+ "theoretical_loss": 3.41251911127218,
+ "tokens_seen": 2360672256
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023017853560682047,
+ "loss": 1.223,
+ "theoretical_loss": 3.41251153127127,
+ "tokens_seen": 2360737792
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023016248746238714,
+ "loss": 1.2352,
+ "theoretical_loss": 3.4125039515397018,
+ "tokens_seen": 2360803328
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023014643931795384,
+ "loss": 1.1167,
+ "theoretical_loss": 3.412496372077458,
+ "tokens_seen": 2360868864
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023013039117352054,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4124887928845222,
+ "tokens_seen": 2360934400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023011434302908724,
+ "loss": 1.3013,
+ "theoretical_loss": 3.412481213960877,
+ "tokens_seen": 2360999936
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023009829488465395,
+ "loss": 1.282,
+ "theoretical_loss": 3.412473635306505,
+ "tokens_seen": 2361065472
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023008224674022065,
+ "loss": 1.2048,
+ "theoretical_loss": 3.41246605692139,
+ "tokens_seen": 2361131008
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023006619859578735,
+ "loss": 1.2246,
+ "theoretical_loss": 3.4124584788055143,
+ "tokens_seen": 2361196544
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023005015045135405,
+ "loss": 1.2444,
+ "theoretical_loss": 3.412450900958861,
+ "tokens_seen": 2361262080
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00023003410230692075,
+ "loss": 1.3942,
+ "theoretical_loss": 3.4124433233814133,
+ "tokens_seen": 2361327616
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002300180541624875,
+ "loss": 1.1909,
+ "theoretical_loss": 3.412435746073154,
+ "tokens_seen": 2361393152
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002300020060180542,
+ "loss": 1.2456,
+ "theoretical_loss": 3.4124281690340657,
+ "tokens_seen": 2361458688
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002299859578736209,
+ "loss": 1.2122,
+ "theoretical_loss": 3.412420592264132,
+ "tokens_seen": 2361524224
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002299699097291876,
+ "loss": 1.1769,
+ "theoretical_loss": 3.4124130157633354,
+ "tokens_seen": 2361589760
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002299538615847543,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4124054395316596,
+ "tokens_seen": 2361655296
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000229937813440321,
+ "loss": 1.1807,
+ "theoretical_loss": 3.412397863569087,
+ "tokens_seen": 2361720832
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022992176529588768,
+ "loss": 1.215,
+ "theoretical_loss": 3.4123902878756,
+ "tokens_seen": 2361786368
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022990571715145438,
+ "loss": 1.1909,
+ "theoretical_loss": 3.4123827124511825,
+ "tokens_seen": 2361851904
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022988966900702108,
+ "loss": 1.1894,
+ "theoretical_loss": 3.412375137295817,
+ "tokens_seen": 2361917440
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022987362086258778,
+ "loss": 1.1633,
+ "theoretical_loss": 3.4123675624094867,
+ "tokens_seen": 2361982976
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022985757271815448,
+ "loss": 1.2018,
+ "theoretical_loss": 3.4123599877921746,
+ "tokens_seen": 2362048512
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022984152457372118,
+ "loss": 1.3097,
+ "theoretical_loss": 3.4123524134438634,
+ "tokens_seen": 2362114048
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022982547642928788,
+ "loss": 1.2144,
+ "theoretical_loss": 3.412344839364537,
+ "tokens_seen": 2362179584
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022980942828485458,
+ "loss": 1.1674,
+ "theoretical_loss": 3.412337265554177,
+ "tokens_seen": 2362245120
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022979338014042128,
+ "loss": 1.1621,
+ "theoretical_loss": 3.4123296920127673,
+ "tokens_seen": 2362310656
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022977733199598799,
+ "loss": 1.2634,
+ "theoretical_loss": 3.4123221187402906,
+ "tokens_seen": 2362376192
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022976128385155469,
+ "loss": 1.2931,
+ "theoretical_loss": 3.41231454573673,
+ "tokens_seen": 2362441728
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022974523570712139,
+ "loss": 1.2438,
+ "theoretical_loss": 3.4123069730020683,
+ "tokens_seen": 2362507264
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 789852,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4360320568084717,
+ "objective/train/theoretical_loss": 3.4122994005362886,
+ "objective/train/tokens_used": 2383032800,
+ "theoretical_loss": 3.4122994005362886,
+ "tokens_seen": 2362572800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002297291875626881,
+ "loss": 1.3079,
+ "theoretical_loss": 3.4122994005362886,
+ "tokens_seen": 2362572800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002297131394182548,
+ "loss": 1.1812,
+ "theoretical_loss": 3.412291828339374,
+ "tokens_seen": 2362638336
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296970912738215,
+ "loss": 1.2449,
+ "theoretical_loss": 3.412284256411308,
+ "tokens_seen": 2362703872
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296810431293882,
+ "loss": 1.1647,
+ "theoretical_loss": 3.4122766847520722,
+ "tokens_seen": 2362769408
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296649949849549,
+ "loss": 1.2091,
+ "theoretical_loss": 3.4122691133616505,
+ "tokens_seen": 2362834944
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296489468405216,
+ "loss": 1.2829,
+ "theoretical_loss": 3.412261542240026,
+ "tokens_seen": 2362900480
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296328986960883,
+ "loss": 1.2731,
+ "theoretical_loss": 3.4122539713871816,
+ "tokens_seen": 2362966016
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000229616850551655,
+ "loss": 1.2001,
+ "theoretical_loss": 3.4122464008031,
+ "tokens_seen": 2363031552
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002296008024072217,
+ "loss": 1.2437,
+ "theoretical_loss": 3.412238830487765,
+ "tokens_seen": 2363097088
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002295847542627884,
+ "loss": 1.1921,
+ "theoretical_loss": 3.4122312604411587,
+ "tokens_seen": 2363162624
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002295687061183551,
+ "loss": 1.3145,
+ "theoretical_loss": 3.4122236906632644,
+ "tokens_seen": 2363228160
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002295526579739218,
+ "loss": 1.3032,
+ "theoretical_loss": 3.412216121154065,
+ "tokens_seen": 2363293696
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022953660982948847,
+ "loss": 1.2727,
+ "theoretical_loss": 3.412208551913544,
+ "tokens_seen": 2363359232
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022952056168505517,
+ "loss": 1.3151,
+ "theoretical_loss": 3.4122009829416835,
+ "tokens_seen": 2363424768
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022950451354062187,
+ "loss": 1.2459,
+ "theoretical_loss": 3.412193414238468,
+ "tokens_seen": 2363490304
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022948846539618857,
+ "loss": 1.2052,
+ "theoretical_loss": 3.4121858458038785,
+ "tokens_seen": 2363555840
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022947241725175527,
+ "loss": 1.1967,
+ "theoretical_loss": 3.4121782776378997,
+ "tokens_seen": 2363621376
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022945636910732197,
+ "loss": 1.1985,
+ "theoretical_loss": 3.412170709740514,
+ "tokens_seen": 2363686912
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022944032096288867,
+ "loss": 1.1746,
+ "theoretical_loss": 3.412163142111705,
+ "tokens_seen": 2363752448
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022942427281845537,
+ "loss": 1.1932,
+ "theoretical_loss": 3.4121555747514547,
+ "tokens_seen": 2363817984
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022940822467402207,
+ "loss": 1.2518,
+ "theoretical_loss": 3.412148007659747,
+ "tokens_seen": 2363883520
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022939217652958877,
+ "loss": 1.3093,
+ "theoretical_loss": 3.4121404408365636,
+ "tokens_seen": 2363949056
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022937612838515547,
+ "loss": 1.2148,
+ "theoretical_loss": 3.4121328742818893,
+ "tokens_seen": 2364014592
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022936008024072217,
+ "loss": 1.2189,
+ "theoretical_loss": 3.412125307995706,
+ "tokens_seen": 2364080128
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022934403209628888,
+ "loss": 1.2437,
+ "theoretical_loss": 3.412117741977997,
+ "tokens_seen": 2364145664
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022932798395185558,
+ "loss": 1.2613,
+ "theoretical_loss": 3.4121101762287456,
+ "tokens_seen": 2364211200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022931193580742228,
+ "loss": 1.3464,
+ "theoretical_loss": 3.4121026107479344,
+ "tokens_seen": 2364276736
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022929588766298898,
+ "loss": 1.2408,
+ "theoretical_loss": 3.4120950455355468,
+ "tokens_seen": 2364342272
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022927983951855568,
+ "loss": 1.2704,
+ "theoretical_loss": 3.4120874805915653,
+ "tokens_seen": 2364407808
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022926379137412238,
+ "loss": 1.385,
+ "theoretical_loss": 3.4120799159159736,
+ "tokens_seen": 2364473344
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022924774322968908,
+ "loss": 1.1175,
+ "theoretical_loss": 3.4120723515087548,
+ "tokens_seen": 2364538880
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022923169508525578,
+ "loss": 1.2194,
+ "theoretical_loss": 3.412064787369891,
+ "tokens_seen": 2364604416
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022921564694082248,
+ "loss": 1.2183,
+ "theoretical_loss": 3.412057223499366,
+ "tokens_seen": 2364669952
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022919959879638918,
+ "loss": 1.3321,
+ "theoretical_loss": 3.412049659897163,
+ "tokens_seen": 2364735488
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022918355065195588,
+ "loss": 1.1521,
+ "theoretical_loss": 3.412042096563264,
+ "tokens_seen": 2364801024
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022916750250752258,
+ "loss": 1.2474,
+ "theoretical_loss": 3.4120345334976534,
+ "tokens_seen": 2364866560
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022915145436308926,
+ "loss": 1.1373,
+ "theoretical_loss": 3.412026970700313,
+ "tokens_seen": 2364932096
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022913540621865596,
+ "loss": 1.3293,
+ "theoretical_loss": 3.412019408171227,
+ "tokens_seen": 2364997632
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022911935807422266,
+ "loss": 1.1929,
+ "theoretical_loss": 3.412011845910378,
+ "tokens_seen": 2365063168
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022910330992978936,
+ "loss": 1.1794,
+ "theoretical_loss": 3.4120042839177485,
+ "tokens_seen": 2365128704
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022908726178535606,
+ "loss": 1.1169,
+ "theoretical_loss": 3.4119967221933223,
+ "tokens_seen": 2365194240
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022907121364092276,
+ "loss": 1.1822,
+ "theoretical_loss": 3.411989160737082,
+ "tokens_seen": 2365259776
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022905516549648946,
+ "loss": 1.1998,
+ "theoretical_loss": 3.4119815995490113,
+ "tokens_seen": 2365325312
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022903911735205616,
+ "loss": 1.244,
+ "theoretical_loss": 3.411974038629092,
+ "tokens_seen": 2365390848
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022902306920762286,
+ "loss": 1.1379,
+ "theoretical_loss": 3.411966477977309,
+ "tokens_seen": 2365456384
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022900702106318956,
+ "loss": 1.3107,
+ "theoretical_loss": 3.4119589175936436,
+ "tokens_seen": 2365521920
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022899097291875626,
+ "loss": 1.2924,
+ "theoretical_loss": 3.4119513574780798,
+ "tokens_seen": 2365587456
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022897492477432296,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4119437976306,
+ "tokens_seen": 2365652992
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022895887662988966,
+ "loss": 1.2681,
+ "theoretical_loss": 3.4119362380511884,
+ "tokens_seen": 2365718528
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022894282848545636,
+ "loss": 1.2153,
+ "theoretical_loss": 3.411928678739827,
+ "tokens_seen": 2365784064
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 790849,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1800425052642822,
+ "objective/train/theoretical_loss": 3.4119211196964994,
+ "objective/train/tokens_used": 2386309600,
+ "theoretical_loss": 3.4119211196964994,
+ "tokens_seen": 2365849600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022892678034102306,
+ "loss": 1.2147,
+ "theoretical_loss": 3.4119211196964994,
+ "tokens_seen": 2365849600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022891073219658977,
+ "loss": 1.2565,
+ "theoretical_loss": 3.4119135609211884,
+ "tokens_seen": 2365915136
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022889468405215647,
+ "loss": 1.2411,
+ "theoretical_loss": 3.411906002413877,
+ "tokens_seen": 2365980672
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022887863590772317,
+ "loss": 1.3014,
+ "theoretical_loss": 3.411898444174549,
+ "tokens_seen": 2366046208
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022886258776328987,
+ "loss": 1.2955,
+ "theoretical_loss": 3.411890886203187,
+ "tokens_seen": 2366111744
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022884653961885657,
+ "loss": 1.2318,
+ "theoretical_loss": 3.4118833284997736,
+ "tokens_seen": 2366177280
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022883049147442327,
+ "loss": 1.3338,
+ "theoretical_loss": 3.4118757710642926,
+ "tokens_seen": 2366242816
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022881444332999,
+ "loss": 1.2493,
+ "theoretical_loss": 3.4118682138967262,
+ "tokens_seen": 2366308352
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287983951855567,
+ "loss": 1.1439,
+ "theoretical_loss": 3.411860656997059,
+ "tokens_seen": 2366373888
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287823470411234,
+ "loss": 1.2419,
+ "theoretical_loss": 3.4118531003652723,
+ "tokens_seen": 2366439424
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287662988966901,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4118455440013506,
+ "tokens_seen": 2366504960
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287502507522568,
+ "loss": 1.3039,
+ "theoretical_loss": 3.411837987905276,
+ "tokens_seen": 2366570496
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287342026078235,
+ "loss": 1.259,
+ "theoretical_loss": 3.4118304320770325,
+ "tokens_seen": 2366636032
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287181544633902,
+ "loss": 1.2358,
+ "theoretical_loss": 3.4118228765166023,
+ "tokens_seen": 2366701568
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002287021063189569,
+ "loss": 1.1944,
+ "theoretical_loss": 3.4118153212239695,
+ "tokens_seen": 2366767104
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002286860581745236,
+ "loss": 1.2959,
+ "theoretical_loss": 3.411807766199116,
+ "tokens_seen": 2366832640
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002286700100300903,
+ "loss": 1.3148,
+ "theoretical_loss": 3.4118002114420256,
+ "tokens_seen": 2366898176
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000228653961885657,
+ "loss": 1.2839,
+ "theoretical_loss": 3.4117926569526813,
+ "tokens_seen": 2366963712
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002286379137412237,
+ "loss": 1.1856,
+ "theoretical_loss": 3.4117851027310664,
+ "tokens_seen": 2367029248
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002286218655967904,
+ "loss": 1.3219,
+ "theoretical_loss": 3.4117775487771635,
+ "tokens_seen": 2367094784
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002286058174523571,
+ "loss": 1.1605,
+ "theoretical_loss": 3.4117699950909564,
+ "tokens_seen": 2367160320
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002285897693079238,
+ "loss": 1.2291,
+ "theoretical_loss": 3.4117624416724275,
+ "tokens_seen": 2367225856
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002285737211634905,
+ "loss": 1.1934,
+ "theoretical_loss": 3.41175488852156,
+ "tokens_seen": 2367291392
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002285576730190572,
+ "loss": 1.2652,
+ "theoretical_loss": 3.4117473356383377,
+ "tokens_seen": 2367356928
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002285416248746239,
+ "loss": 1.2991,
+ "theoretical_loss": 3.411739783022743,
+ "tokens_seen": 2367422464
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022852557673019058,
+ "loss": 1.3153,
+ "theoretical_loss": 3.411732230674759,
+ "tokens_seen": 2367488000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022850952858575728,
+ "loss": 1.2223,
+ "theoretical_loss": 3.411724678594369,
+ "tokens_seen": 2367553536
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022849348044132398,
+ "loss": 1.2912,
+ "theoretical_loss": 3.4117171267815563,
+ "tokens_seen": 2367619072
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022847743229689068,
+ "loss": 1.2253,
+ "theoretical_loss": 3.411709575236304,
+ "tokens_seen": 2367684608
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022846138415245738,
+ "loss": 1.1495,
+ "theoretical_loss": 3.411702023958595,
+ "tokens_seen": 2367750144
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022844533600802408,
+ "loss": 1.1605,
+ "theoretical_loss": 3.411694472948412,
+ "tokens_seen": 2367815680
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022842928786359078,
+ "loss": 1.1421,
+ "theoretical_loss": 3.4116869222057393,
+ "tokens_seen": 2367881216
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022841323971915748,
+ "loss": 1.2383,
+ "theoretical_loss": 3.411679371730559,
+ "tokens_seen": 2367946752
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022839719157472419,
+ "loss": 1.315,
+ "theoretical_loss": 3.4116718215228543,
+ "tokens_seen": 2368012288
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022838114343029089,
+ "loss": 1.1597,
+ "theoretical_loss": 3.411664271582609,
+ "tokens_seen": 2368077824
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002283650952858576,
+ "loss": 1.1837,
+ "theoretical_loss": 3.4116567219098055,
+ "tokens_seen": 2368143360
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002283490471414243,
+ "loss": 1.1976,
+ "theoretical_loss": 3.4116491725044273,
+ "tokens_seen": 2368208896
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000228332998996991,
+ "loss": 1.1807,
+ "theoretical_loss": 3.4116416233664575,
+ "tokens_seen": 2368274432
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002283169508525577,
+ "loss": 1.1739,
+ "theoretical_loss": 3.411634074495879,
+ "tokens_seen": 2368339968
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002283009027081244,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4116265258926752,
+ "tokens_seen": 2368405504
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282848545636911,
+ "loss": 1.2574,
+ "theoretical_loss": 3.4116189775568295,
+ "tokens_seen": 2368471040
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282688064192578,
+ "loss": 1.2418,
+ "theoretical_loss": 3.411611429488324,
+ "tokens_seen": 2368536576
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282527582748245,
+ "loss": 1.1883,
+ "theoretical_loss": 3.4116038816871432,
+ "tokens_seen": 2368602112
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282367101303912,
+ "loss": 1.1778,
+ "theoretical_loss": 3.411596334153269,
+ "tokens_seen": 2368667648
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282206619859579,
+ "loss": 1.2314,
+ "theoretical_loss": 3.411588786886685,
+ "tokens_seen": 2368733184
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002282046138415246,
+ "loss": 1.3404,
+ "theoretical_loss": 3.411581239887375,
+ "tokens_seen": 2368798720
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002281885656970913,
+ "loss": 1.2689,
+ "theoretical_loss": 3.411573693155321,
+ "tokens_seen": 2368864256
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000228172517552658,
+ "loss": 1.1791,
+ "theoretical_loss": 3.411566146690507,
+ "tokens_seen": 2368929792
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002281564694082247,
+ "loss": 1.1527,
+ "theoretical_loss": 3.4115586004929157,
+ "tokens_seen": 2368995328
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022814042126379137,
+ "loss": 1.287,
+ "theoretical_loss": 3.4115510545625303,
+ "tokens_seen": 2369060864
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 791985,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1610465049743652,
+ "objective/train/theoretical_loss": 3.411543508899334,
+ "objective/train/tokens_used": 2389586400,
+ "theoretical_loss": 3.411543508899334,
+ "tokens_seen": 2369126400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022812437311935807,
+ "loss": 1.1473,
+ "theoretical_loss": 3.411543508899334,
+ "tokens_seen": 2369126400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022810832497492477,
+ "loss": 1.1854,
+ "theoretical_loss": 3.4115359635033102,
+ "tokens_seen": 2369191936
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022809227683049147,
+ "loss": 1.2386,
+ "theoretical_loss": 3.411528418374442,
+ "tokens_seen": 2369257472
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022807622868605817,
+ "loss": 1.1074,
+ "theoretical_loss": 3.411520873512712,
+ "tokens_seen": 2369323008
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022806018054162487,
+ "loss": 1.2091,
+ "theoretical_loss": 3.4115133289181037,
+ "tokens_seen": 2369388544
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022804413239719157,
+ "loss": 1.1968,
+ "theoretical_loss": 3.4115057845906005,
+ "tokens_seen": 2369454080
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022802808425275827,
+ "loss": 1.2168,
+ "theoretical_loss": 3.411498240530185,
+ "tokens_seen": 2369519616
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022801203610832497,
+ "loss": 1.1901,
+ "theoretical_loss": 3.4114906967368412,
+ "tokens_seen": 2369585152
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022799598796389167,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4114831532105514,
+ "tokens_seen": 2369650688
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022797993981945837,
+ "loss": 1.2362,
+ "theoretical_loss": 3.4114756099512995,
+ "tokens_seen": 2369716224
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022796389167502508,
+ "loss": 1.2446,
+ "theoretical_loss": 3.411468066959068,
+ "tokens_seen": 2369781760
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022794784353059178,
+ "loss": 1.2218,
+ "theoretical_loss": 3.4114605242338403,
+ "tokens_seen": 2369847296
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022793179538615848,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4114529817756,
+ "tokens_seen": 2369912832
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022791574724172518,
+ "loss": 1.2195,
+ "theoretical_loss": 3.4114454395843294,
+ "tokens_seen": 2369978368
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022789969909729188,
+ "loss": 1.1934,
+ "theoretical_loss": 3.4114378976600124,
+ "tokens_seen": 2370043904
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022788365095285858,
+ "loss": 1.1421,
+ "theoretical_loss": 3.411430356002632,
+ "tokens_seen": 2370109440
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022786760280842528,
+ "loss": 1.2698,
+ "theoretical_loss": 3.411422814612171,
+ "tokens_seen": 2370174976
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022785155466399198,
+ "loss": 1.227,
+ "theoretical_loss": 3.411415273488613,
+ "tokens_seen": 2370240512
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022783550651955868,
+ "loss": 1.2715,
+ "theoretical_loss": 3.4114077326319414,
+ "tokens_seen": 2370306048
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022781945837512538,
+ "loss": 1.2606,
+ "theoretical_loss": 3.4114001920421386,
+ "tokens_seen": 2370371584
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022780341023069208,
+ "loss": 1.2148,
+ "theoretical_loss": 3.411392651719188,
+ "tokens_seen": 2370437120
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022778736208625878,
+ "loss": 1.2027,
+ "theoretical_loss": 3.4113851116630736,
+ "tokens_seen": 2370502656
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022777131394182548,
+ "loss": 1.2085,
+ "theoretical_loss": 3.4113775718737775,
+ "tokens_seen": 2370568192
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022775526579739216,
+ "loss": 1.2653,
+ "theoretical_loss": 3.411370032351283,
+ "tokens_seen": 2370633728
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022773921765295886,
+ "loss": 1.2206,
+ "theoretical_loss": 3.411362493095574,
+ "tokens_seen": 2370699264
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022772316950852556,
+ "loss": 1.2067,
+ "theoretical_loss": 3.411354954106634,
+ "tokens_seen": 2370764800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022770712136409226,
+ "loss": 1.1931,
+ "theoretical_loss": 3.4113474153844443,
+ "tokens_seen": 2370830336
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022769107321965896,
+ "loss": 1.3109,
+ "theoretical_loss": 3.4113398769289898,
+ "tokens_seen": 2370895872
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022767502507522566,
+ "loss": 1.2778,
+ "theoretical_loss": 3.4113323387402534,
+ "tokens_seen": 2370961408
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022765897693079236,
+ "loss": 1.2098,
+ "theoretical_loss": 3.411324800818218,
+ "tokens_seen": 2371026944
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022764292878635906,
+ "loss": 1.2,
+ "theoretical_loss": 3.4113172631628665,
+ "tokens_seen": 2371092480
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022762688064192582,
+ "loss": 1.2634,
+ "theoretical_loss": 3.4113097257741827,
+ "tokens_seen": 2371158016
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022761083249749252,
+ "loss": 1.1124,
+ "theoretical_loss": 3.4113021886521495,
+ "tokens_seen": 2371223552
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022759478435305922,
+ "loss": 1.1771,
+ "theoretical_loss": 3.41129465179675,
+ "tokens_seen": 2371289088
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022757873620862592,
+ "loss": 1.2522,
+ "theoretical_loss": 3.411287115207968,
+ "tokens_seen": 2371354624
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022756268806419262,
+ "loss": 1.1815,
+ "theoretical_loss": 3.411279578885786,
+ "tokens_seen": 2371420160
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022754663991975932,
+ "loss": 1.2734,
+ "theoretical_loss": 3.411272042830187,
+ "tokens_seen": 2371485696
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022753059177532602,
+ "loss": 1.2423,
+ "theoretical_loss": 3.411264507041155,
+ "tokens_seen": 2371551232
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022751454363089272,
+ "loss": 1.234,
+ "theoretical_loss": 3.411256971518673,
+ "tokens_seen": 2371616768
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274984954864594,
+ "loss": 1.3039,
+ "theoretical_loss": 3.4112494362627235,
+ "tokens_seen": 2371682304
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274824473420261,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4112419012732906,
+ "tokens_seen": 2371747840
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274663991975928,
+ "loss": 1.2805,
+ "theoretical_loss": 3.4112343665503575,
+ "tokens_seen": 2371813376
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274503510531595,
+ "loss": 1.2336,
+ "theoretical_loss": 3.4112268320939068,
+ "tokens_seen": 2371878912
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274343029087262,
+ "loss": 1.2039,
+ "theoretical_loss": 3.411219297903922,
+ "tokens_seen": 2371944448
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274182547642929,
+ "loss": 1.218,
+ "theoretical_loss": 3.4112117639803863,
+ "tokens_seen": 2372009984
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002274022066198596,
+ "loss": 1.2331,
+ "theoretical_loss": 3.411204230323283,
+ "tokens_seen": 2372075520
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002273861584754263,
+ "loss": 1.2391,
+ "theoretical_loss": 3.411196696932595,
+ "tokens_seen": 2372141056
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000227370110330993,
+ "loss": 1.1919,
+ "theoretical_loss": 3.411189163808306,
+ "tokens_seen": 2372206592
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002273540621865597,
+ "loss": 1.1886,
+ "theoretical_loss": 3.4111816309503986,
+ "tokens_seen": 2372272128
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002273380140421264,
+ "loss": 1.1765,
+ "theoretical_loss": 3.411174098358857,
+ "tokens_seen": 2372337664
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 793283,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2480533123016357,
+ "objective/train/theoretical_loss": 3.4111665660336636,
+ "objective/train/tokens_used": 2392863200,
+ "theoretical_loss": 3.4111665660336636,
+ "tokens_seen": 2372403200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002273219658976931,
+ "loss": 1.207,
+ "theoretical_loss": 3.4111665660336636,
+ "tokens_seen": 2372403200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002273059177532598,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4111590339748017,
+ "tokens_seen": 2372468736
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002272898696088265,
+ "loss": 1.3499,
+ "theoretical_loss": 3.4111515021822547,
+ "tokens_seen": 2372534272
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002272738214643932,
+ "loss": 1.2498,
+ "theoretical_loss": 3.411143970656006,
+ "tokens_seen": 2372599808
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002272577733199599,
+ "loss": 1.1974,
+ "theoretical_loss": 3.4111364393960386,
+ "tokens_seen": 2372665344
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002272417251755266,
+ "loss": 1.2408,
+ "theoretical_loss": 3.4111289084023357,
+ "tokens_seen": 2372730880
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002272256770310933,
+ "loss": 1.2875,
+ "theoretical_loss": 3.4111213776748803,
+ "tokens_seen": 2372796416
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022720962888666,
+ "loss": 1.1829,
+ "theoretical_loss": 3.4111138472136564,
+ "tokens_seen": 2372861952
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002271935807422267,
+ "loss": 1.2387,
+ "theoretical_loss": 3.4111063170186466,
+ "tokens_seen": 2372927488
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002271775325977934,
+ "loss": 1.3391,
+ "theoretical_loss": 3.411098787089834,
+ "tokens_seen": 2372993024
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002271614844533601,
+ "loss": 1.309,
+ "theoretical_loss": 3.411091257427203,
+ "tokens_seen": 2373058560
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002271454363089268,
+ "loss": 1.2755,
+ "theoretical_loss": 3.411083728030735,
+ "tokens_seen": 2373124096
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002271293881644935,
+ "loss": 1.2005,
+ "theoretical_loss": 3.411076198900415,
+ "tokens_seen": 2373189632
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022711334002006018,
+ "loss": 1.2996,
+ "theoretical_loss": 3.411068670036225,
+ "tokens_seen": 2373255168
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022709729187562688,
+ "loss": 1.0836,
+ "theoretical_loss": 3.411061141438149,
+ "tokens_seen": 2373320704
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022708124373119358,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4110536131061693,
+ "tokens_seen": 2373386240
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022706519558676028,
+ "loss": 1.1903,
+ "theoretical_loss": 3.4110460850402706,
+ "tokens_seen": 2373451776
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022704914744232698,
+ "loss": 1.2109,
+ "theoretical_loss": 3.4110385572404347,
+ "tokens_seen": 2373517312
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022703309929789369,
+ "loss": 1.2999,
+ "theoretical_loss": 3.411031029706646,
+ "tokens_seen": 2373582848
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022701705115346039,
+ "loss": 1.2261,
+ "theoretical_loss": 3.411023502438887,
+ "tokens_seen": 2373648384
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022700100300902709,
+ "loss": 1.2605,
+ "theoretical_loss": 3.4110159754371416,
+ "tokens_seen": 2373713920
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269849548645938,
+ "loss": 1.3836,
+ "theoretical_loss": 3.4110084487013923,
+ "tokens_seen": 2373779456
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269689067201605,
+ "loss": 1.2666,
+ "theoretical_loss": 3.4110009222316227,
+ "tokens_seen": 2373844992
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269528585757272,
+ "loss": 1.2384,
+ "theoretical_loss": 3.4109933960278163,
+ "tokens_seen": 2373910528
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269368104312939,
+ "loss": 1.1957,
+ "theoretical_loss": 3.410985870089956,
+ "tokens_seen": 2373976064
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269207622868606,
+ "loss": 1.2826,
+ "theoretical_loss": 3.4109783444180253,
+ "tokens_seen": 2374041600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002269047141424273,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4109708190120074,
+ "tokens_seen": 2374107136
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000226888665997994,
+ "loss": 1.2171,
+ "theoretical_loss": 3.4109632938718852,
+ "tokens_seen": 2374172672
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002268726178535607,
+ "loss": 1.2713,
+ "theoretical_loss": 3.4109557689976424,
+ "tokens_seen": 2374238208
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002268565697091274,
+ "loss": 1.2061,
+ "theoretical_loss": 3.4109482443892625,
+ "tokens_seen": 2374303744
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002268405215646941,
+ "loss": 1.2888,
+ "theoretical_loss": 3.410940720046728,
+ "tokens_seen": 2374369280
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002268244734202608,
+ "loss": 1.2707,
+ "theoretical_loss": 3.410933195970023,
+ "tokens_seen": 2374434816
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002268084252758275,
+ "loss": 1.1276,
+ "theoretical_loss": 3.41092567215913,
+ "tokens_seen": 2374500352
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002267923771313942,
+ "loss": 1.3248,
+ "theoretical_loss": 3.4109181486140328,
+ "tokens_seen": 2374565888
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002267763289869609,
+ "loss": 1.2076,
+ "theoretical_loss": 3.4109106253347146,
+ "tokens_seen": 2374631424
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002267602808425276,
+ "loss": 1.2211,
+ "theoretical_loss": 3.4109031023211585,
+ "tokens_seen": 2374696960
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002267442326980943,
+ "loss": 1.2325,
+ "theoretical_loss": 3.410895579573348,
+ "tokens_seen": 2374762496
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022672818455366097,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4108880570912663,
+ "tokens_seen": 2374828032
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022671213640922767,
+ "loss": 1.2713,
+ "theoretical_loss": 3.410880534874896,
+ "tokens_seen": 2374893568
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022669608826479437,
+ "loss": 1.1759,
+ "theoretical_loss": 3.4108730129242217,
+ "tokens_seen": 2374959104
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022668004012036107,
+ "loss": 1.1801,
+ "theoretical_loss": 3.410865491239226,
+ "tokens_seen": 2375024640
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022666399197592777,
+ "loss": 1.186,
+ "theoretical_loss": 3.410857969819892,
+ "tokens_seen": 2375090176
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022664794383149447,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4108504486662032,
+ "tokens_seen": 2375155712
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022663189568706117,
+ "loss": 1.3107,
+ "theoretical_loss": 3.410842927778143,
+ "tokens_seen": 2375221248
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022661584754262787,
+ "loss": 1.229,
+ "theoretical_loss": 3.410835407155694,
+ "tokens_seen": 2375286784
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022659979939819458,
+ "loss": 1.2506,
+ "theoretical_loss": 3.410827886798841,
+ "tokens_seen": 2375352320
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022658375125376128,
+ "loss": 1.2827,
+ "theoretical_loss": 3.4108203667075654,
+ "tokens_seen": 2375417856
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022656770310932798,
+ "loss": 1.0998,
+ "theoretical_loss": 3.4108128468818517,
+ "tokens_seen": 2375483392
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022655165496489468,
+ "loss": 1.2627,
+ "theoretical_loss": 3.4108053273216834,
+ "tokens_seen": 2375548928
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022653560682046138,
+ "loss": 1.1509,
+ "theoretical_loss": 3.4107978080270427,
+ "tokens_seen": 2375614464
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 794143,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2268905639648438,
+ "objective/train/theoretical_loss": 3.410790288997914,
+ "objective/train/tokens_used": 2396140000,
+ "theoretical_loss": 3.410790288997914,
+ "tokens_seen": 2375680000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022651955867602808,
+ "loss": 1.2947,
+ "theoretical_loss": 3.410790288997914,
+ "tokens_seen": 2375680000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022650351053159478,
+ "loss": 1.2449,
+ "theoretical_loss": 3.4107827702342797,
+ "tokens_seen": 2375745536
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022648746238716148,
+ "loss": 1.2793,
+ "theoretical_loss": 3.4107752517361236,
+ "tokens_seen": 2375811072
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022647141424272818,
+ "loss": 1.2137,
+ "theoretical_loss": 3.4107677335034294,
+ "tokens_seen": 2375876608
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022645536609829488,
+ "loss": 1.2129,
+ "theoretical_loss": 3.4107602155361794,
+ "tokens_seen": 2375942144
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022643931795386158,
+ "loss": 1.1991,
+ "theoretical_loss": 3.410752697834358,
+ "tokens_seen": 2376007680
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002264232698094283,
+ "loss": 1.214,
+ "theoretical_loss": 3.410745180397947,
+ "tokens_seen": 2376073216
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000226407221664995,
+ "loss": 1.2258,
+ "theoretical_loss": 3.4107376632269313,
+ "tokens_seen": 2376138752
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002263911735205617,
+ "loss": 1.1537,
+ "theoretical_loss": 3.410730146321294,
+ "tokens_seen": 2376204288
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002263751253761284,
+ "loss": 1.2582,
+ "theoretical_loss": 3.4107226296810174,
+ "tokens_seen": 2376269824
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002263590772316951,
+ "loss": 1.1709,
+ "theoretical_loss": 3.410715113306085,
+ "tokens_seen": 2376335360
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002263430290872618,
+ "loss": 1.2334,
+ "theoretical_loss": 3.410707597196481,
+ "tokens_seen": 2376400896
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002263269809428285,
+ "loss": 1.2121,
+ "theoretical_loss": 3.4107000813521884,
+ "tokens_seen": 2376466432
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022631093279839521,
+ "loss": 1.3528,
+ "theoretical_loss": 3.41069256577319,
+ "tokens_seen": 2376531968
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022629488465396191,
+ "loss": 1.2494,
+ "theoretical_loss": 3.41068505045947,
+ "tokens_seen": 2376597504
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022627883650952861,
+ "loss": 1.2996,
+ "theoretical_loss": 3.410677535411011,
+ "tokens_seen": 2376663040
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022626278836509532,
+ "loss": 1.197,
+ "theoretical_loss": 3.410670020627796,
+ "tokens_seen": 2376728576
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022624674022066202,
+ "loss": 1.1699,
+ "theoretical_loss": 3.4106625061098095,
+ "tokens_seen": 2376794112
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022623069207622872,
+ "loss": 1.1868,
+ "theoretical_loss": 3.4106549918570335,
+ "tokens_seen": 2376859648
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022621464393179542,
+ "loss": 1.2734,
+ "theoretical_loss": 3.4106474778694524,
+ "tokens_seen": 2376925184
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022619859578736212,
+ "loss": 1.2605,
+ "theoretical_loss": 3.410639964147049,
+ "tokens_seen": 2376990720
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022618254764292882,
+ "loss": 1.2087,
+ "theoretical_loss": 3.410632450689807,
+ "tokens_seen": 2377056256
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022616649949849552,
+ "loss": 1.1526,
+ "theoretical_loss": 3.410624937497709,
+ "tokens_seen": 2377121792
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022615045135406222,
+ "loss": 1.209,
+ "theoretical_loss": 3.4106174245707397,
+ "tokens_seen": 2377187328
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022613440320962892,
+ "loss": 1.3085,
+ "theoretical_loss": 3.410609911908881,
+ "tokens_seen": 2377252864
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022611835506519562,
+ "loss": 1.3448,
+ "theoretical_loss": 3.4106023995121166,
+ "tokens_seen": 2377318400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002261023069207623,
+ "loss": 1.2383,
+ "theoretical_loss": 3.41059488738043,
+ "tokens_seen": 2377383936
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000226086258776329,
+ "loss": 1.3249,
+ "theoretical_loss": 3.410587375513805,
+ "tokens_seen": 2377449472
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002260702106318957,
+ "loss": 1.1943,
+ "theoretical_loss": 3.4105798639122242,
+ "tokens_seen": 2377515008
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002260541624874624,
+ "loss": 1.171,
+ "theoretical_loss": 3.410572352575671,
+ "tokens_seen": 2377580544
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002260381143430291,
+ "loss": 1.2519,
+ "theoretical_loss": 3.4105648415041294,
+ "tokens_seen": 2377646080
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002260220661985958,
+ "loss": 1.245,
+ "theoretical_loss": 3.4105573306975825,
+ "tokens_seen": 2377711616
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002260060180541625,
+ "loss": 1.142,
+ "theoretical_loss": 3.410549820156013,
+ "tokens_seen": 2377777152
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002259899699097292,
+ "loss": 1.3106,
+ "theoretical_loss": 3.410542309879405,
+ "tokens_seen": 2377842688
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002259739217652959,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4105347998677416,
+ "tokens_seen": 2377908224
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002259578736208626,
+ "loss": 1.1594,
+ "theoretical_loss": 3.4105272901210064,
+ "tokens_seen": 2377973760
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002259418254764293,
+ "loss": 1.2976,
+ "theoretical_loss": 3.410519780639182,
+ "tokens_seen": 2378039296
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000225925777331996,
+ "loss": 1.2219,
+ "theoretical_loss": 3.4105122714222524,
+ "tokens_seen": 2378104832
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002259097291875627,
+ "loss": 1.2078,
+ "theoretical_loss": 3.410504762470201,
+ "tokens_seen": 2378170368
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258936810431294,
+ "loss": 1.1857,
+ "theoretical_loss": 3.410497253783011,
+ "tokens_seen": 2378235904
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258776328986961,
+ "loss": 1.171,
+ "theoretical_loss": 3.4104897453606653,
+ "tokens_seen": 2378301440
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258615847542628,
+ "loss": 1.2654,
+ "theoretical_loss": 3.410482237203148,
+ "tokens_seen": 2378366976
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258455366098295,
+ "loss": 1.2392,
+ "theoretical_loss": 3.410474729310442,
+ "tokens_seen": 2378432512
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258294884653962,
+ "loss": 1.2609,
+ "theoretical_loss": 3.410467221682531,
+ "tokens_seen": 2378498048
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002258134403209629,
+ "loss": 1.2041,
+ "theoretical_loss": 3.410459714319398,
+ "tokens_seen": 2378563584
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002257973921765296,
+ "loss": 1.2164,
+ "theoretical_loss": 3.4104522072210264,
+ "tokens_seen": 2378629120
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002257813440320963,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4104447003874,
+ "tokens_seen": 2378694656
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000225765295887663,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4104371938185016,
+ "tokens_seen": 2378760192
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002257492477432297,
+ "loss": 1.2285,
+ "theoretical_loss": 3.4104296875143154,
+ "tokens_seen": 2378825728
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002257331995987964,
+ "loss": 1.2667,
+ "theoretical_loss": 3.4104221814748237,
+ "tokens_seen": 2378891264
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 795373,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1782634258270264,
+ "objective/train/theoretical_loss": 3.410414675700011,
+ "objective/train/tokens_used": 2399416800,
+ "theoretical_loss": 3.410414675700011,
+ "tokens_seen": 2378956800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022571715145436308,
+ "loss": 1.2295,
+ "theoretical_loss": 3.410414675700011,
+ "tokens_seen": 2378956800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022570110330992978,
+ "loss": 1.1878,
+ "theoretical_loss": 3.4104071701898593,
+ "tokens_seen": 2379022336
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022568505516549648,
+ "loss": 1.2206,
+ "theoretical_loss": 3.410399664944353,
+ "tokens_seen": 2379087872
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022566900702106318,
+ "loss": 1.3167,
+ "theoretical_loss": 3.4103921599634752,
+ "tokens_seen": 2379153408
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022565295887662989,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4103846552472095,
+ "tokens_seen": 2379218944
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022563691073219659,
+ "loss": 1.2547,
+ "theoretical_loss": 3.410377150795539,
+ "tokens_seen": 2379284480
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002256208625877633,
+ "loss": 1.3127,
+ "theoretical_loss": 3.4103696466084474,
+ "tokens_seen": 2379350016
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022560481444333,
+ "loss": 1.2246,
+ "theoretical_loss": 3.4103621426859174,
+ "tokens_seen": 2379415552
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255887662988967,
+ "loss": 1.1684,
+ "theoretical_loss": 3.410354639027933,
+ "tokens_seen": 2379481088
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255727181544634,
+ "loss": 1.2786,
+ "theoretical_loss": 3.4103471356344777,
+ "tokens_seen": 2379546624
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255566700100301,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4103396325055346,
+ "tokens_seen": 2379612160
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255406218655968,
+ "loss": 1.2885,
+ "theoretical_loss": 3.4103321296410867,
+ "tokens_seen": 2379677696
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255245737211635,
+ "loss": 1.2694,
+ "theoretical_loss": 3.4103246270411183,
+ "tokens_seen": 2379743232
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002255085255767302,
+ "loss": 1.3012,
+ "theoretical_loss": 3.410317124705612,
+ "tokens_seen": 2379808768
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002254924774322969,
+ "loss": 1.2191,
+ "theoretical_loss": 3.4103096226345517,
+ "tokens_seen": 2379874304
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002254764292878636,
+ "loss": 1.153,
+ "theoretical_loss": 3.4103021208279203,
+ "tokens_seen": 2379939840
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002254603811434303,
+ "loss": 1.2488,
+ "theoretical_loss": 3.4102946192857018,
+ "tokens_seen": 2380005376
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000225444332998997,
+ "loss": 1.1691,
+ "theoretical_loss": 3.4102871180078793,
+ "tokens_seen": 2380070912
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002254282848545637,
+ "loss": 1.1865,
+ "theoretical_loss": 3.410279616994436,
+ "tokens_seen": 2380136448
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002254122367101304,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4102721162453555,
+ "tokens_seen": 2380201984
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002253961885656971,
+ "loss": 1.2199,
+ "theoretical_loss": 3.410264615760621,
+ "tokens_seen": 2380267520
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002253801404212638,
+ "loss": 1.2146,
+ "theoretical_loss": 3.4102571155402166,
+ "tokens_seen": 2380333056
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002253640922768305,
+ "loss": 1.2565,
+ "theoretical_loss": 3.410249615584125,
+ "tokens_seen": 2380398592
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002253480441323972,
+ "loss": 1.1584,
+ "theoretical_loss": 3.4102421158923297,
+ "tokens_seen": 2380464128
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022533199598796387,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4102346164648143,
+ "tokens_seen": 2380529664
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022531594784353057,
+ "loss": 1.1658,
+ "theoretical_loss": 3.410227117301562,
+ "tokens_seen": 2380595200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022529989969909727,
+ "loss": 1.33,
+ "theoretical_loss": 3.4102196184025564,
+ "tokens_seen": 2380660736
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022528385155466397,
+ "loss": 1.3188,
+ "theoretical_loss": 3.410212119767781,
+ "tokens_seen": 2380726272
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022526780341023067,
+ "loss": 1.3587,
+ "theoretical_loss": 3.4102046213972192,
+ "tokens_seen": 2380791808
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022525175526579737,
+ "loss": 1.2253,
+ "theoretical_loss": 3.410197123290854,
+ "tokens_seen": 2380857344
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022523570712136407,
+ "loss": 1.2991,
+ "theoretical_loss": 3.4101896254486697,
+ "tokens_seen": 2380922880
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022521965897693083,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4101821278706486,
+ "tokens_seen": 2380988416
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022520361083249753,
+ "loss": 1.2655,
+ "theoretical_loss": 3.410174630556775,
+ "tokens_seen": 2381053952
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022518756268806423,
+ "loss": 1.3182,
+ "theoretical_loss": 3.4101671335070316,
+ "tokens_seen": 2381119488
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022517151454363093,
+ "loss": 1.1117,
+ "theoretical_loss": 3.4101596367214024,
+ "tokens_seen": 2381185024
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022515546639919763,
+ "loss": 1.2422,
+ "theoretical_loss": 3.410152140199871,
+ "tokens_seen": 2381250560
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022513941825476433,
+ "loss": 1.2949,
+ "theoretical_loss": 3.41014464394242,
+ "tokens_seen": 2381316096
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022512337011033103,
+ "loss": 1.188,
+ "theoretical_loss": 3.410137147949033,
+ "tokens_seen": 2381381632
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022510732196589773,
+ "loss": 1.0207,
+ "theoretical_loss": 3.4101296522196947,
+ "tokens_seen": 2381447168
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250912738214644,
+ "loss": 1.279,
+ "theoretical_loss": 3.4101221567543867,
+ "tokens_seen": 2381512704
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250752256770311,
+ "loss": 1.1728,
+ "theoretical_loss": 3.410114661553094,
+ "tokens_seen": 2381578240
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250591775325978,
+ "loss": 1.2417,
+ "theoretical_loss": 3.410107166615799,
+ "tokens_seen": 2381643776
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250431293881645,
+ "loss": 1.1738,
+ "theoretical_loss": 3.4100996719424854,
+ "tokens_seen": 2381709312
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250270812437312,
+ "loss": 1.2056,
+ "theoretical_loss": 3.4100921775331368,
+ "tokens_seen": 2381774848
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002250110330992979,
+ "loss": 1.3162,
+ "theoretical_loss": 3.4100846833877365,
+ "tokens_seen": 2381840384
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002249949849548646,
+ "loss": 1.2366,
+ "theoretical_loss": 3.410077189506268,
+ "tokens_seen": 2381905920
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002249789368104313,
+ "loss": 1.3271,
+ "theoretical_loss": 3.4100696958887147,
+ "tokens_seen": 2381971456
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000224962888665998,
+ "loss": 1.2511,
+ "theoretical_loss": 3.41006220253506,
+ "tokens_seen": 2382036992
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002249468405215647,
+ "loss": 1.1874,
+ "theoretical_loss": 3.410054709445288,
+ "tokens_seen": 2382102528
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022493079237713141,
+ "loss": 1.3549,
+ "theoretical_loss": 3.410047216619381,
+ "tokens_seen": 2382168064
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 796235,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.25529146194458,
+ "objective/train/theoretical_loss": 3.4100397240573233,
+ "objective/train/tokens_used": 2402693600,
+ "theoretical_loss": 3.4100397240573233,
+ "tokens_seen": 2382233600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022491474423269811,
+ "loss": 1.2103,
+ "theoretical_loss": 3.4100397240573233,
+ "tokens_seen": 2382233600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022489869608826482,
+ "loss": 1.2223,
+ "theoretical_loss": 3.410032231759098,
+ "tokens_seen": 2382299136
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022488264794383152,
+ "loss": 1.1529,
+ "theoretical_loss": 3.4100247397246886,
+ "tokens_seen": 2382364672
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022486659979939822,
+ "loss": 1.1752,
+ "theoretical_loss": 3.4100172479540785,
+ "tokens_seen": 2382430208
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022485055165496492,
+ "loss": 1.2051,
+ "theoretical_loss": 3.4100097564472516,
+ "tokens_seen": 2382495744
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022483450351053162,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4100022652041906,
+ "tokens_seen": 2382561280
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022481845536609832,
+ "loss": 1.3021,
+ "theoretical_loss": 3.4099947742248795,
+ "tokens_seen": 2382626816
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022480240722166502,
+ "loss": 1.2265,
+ "theoretical_loss": 3.4099872835093015,
+ "tokens_seen": 2382692352
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022478635907723172,
+ "loss": 1.0809,
+ "theoretical_loss": 3.40997979305744,
+ "tokens_seen": 2382757888
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022477031093279842,
+ "loss": 1.2624,
+ "theoretical_loss": 3.409972302869279,
+ "tokens_seen": 2382823424
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022475426278836512,
+ "loss": 1.2082,
+ "theoretical_loss": 3.4099648129448017,
+ "tokens_seen": 2382888960
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022473821464393182,
+ "loss": 1.2004,
+ "theoretical_loss": 3.409957323283991,
+ "tokens_seen": 2382954496
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022472216649949852,
+ "loss": 1.3032,
+ "theoretical_loss": 3.4099498338868313,
+ "tokens_seen": 2383020032
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002247061183550652,
+ "loss": 1.27,
+ "theoretical_loss": 3.4099423447533055,
+ "tokens_seen": 2383085568
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002246900702106319,
+ "loss": 1.2191,
+ "theoretical_loss": 3.4099348558833973,
+ "tokens_seen": 2383151104
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002246740220661986,
+ "loss": 1.312,
+ "theoretical_loss": 3.40992736727709,
+ "tokens_seen": 2383216640
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002246579739217653,
+ "loss": 1.1118,
+ "theoretical_loss": 3.4099198789343665,
+ "tokens_seen": 2383282176
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000224641925777332,
+ "loss": 1.2676,
+ "theoretical_loss": 3.4099123908552116,
+ "tokens_seen": 2383347712
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002246258776328987,
+ "loss": 1.2293,
+ "theoretical_loss": 3.4099049030396076,
+ "tokens_seen": 2383413248
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002246098294884654,
+ "loss": 1.2976,
+ "theoretical_loss": 3.4098974154875386,
+ "tokens_seen": 2383478784
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245937813440321,
+ "loss": 1.2863,
+ "theoretical_loss": 3.409889928198988,
+ "tokens_seen": 2383544320
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245777331995988,
+ "loss": 1.2274,
+ "theoretical_loss": 3.4098824411739392,
+ "tokens_seen": 2383609856
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245616850551655,
+ "loss": 1.1859,
+ "theoretical_loss": 3.409874954412376,
+ "tokens_seen": 2383675392
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245456369107322,
+ "loss": 1.2085,
+ "theoretical_loss": 3.409867467914281,
+ "tokens_seen": 2383740928
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245295887662989,
+ "loss": 1.2198,
+ "theoretical_loss": 3.4098599816796384,
+ "tokens_seen": 2383806464
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002245135406218656,
+ "loss": 1.2939,
+ "theoretical_loss": 3.409852495708432,
+ "tokens_seen": 2383872000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244974924774323,
+ "loss": 1.2061,
+ "theoretical_loss": 3.4098450100006445,
+ "tokens_seen": 2383937536
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000224481444332999,
+ "loss": 1.1416,
+ "theoretical_loss": 3.4098375245562593,
+ "tokens_seen": 2384003072
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244653961885657,
+ "loss": 1.2201,
+ "theoretical_loss": 3.4098300393752607,
+ "tokens_seen": 2384068608
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244493480441324,
+ "loss": 1.2717,
+ "theoretical_loss": 3.409822554457632,
+ "tokens_seen": 2384134144
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244332998996991,
+ "loss": 1.2119,
+ "theoretical_loss": 3.4098150698033565,
+ "tokens_seen": 2384199680
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244172517552658,
+ "loss": 1.2526,
+ "theoretical_loss": 3.4098075854124175,
+ "tokens_seen": 2384265216
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002244012036108325,
+ "loss": 1.2875,
+ "theoretical_loss": 3.409800101284799,
+ "tokens_seen": 2384330752
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002243851554663992,
+ "loss": 1.1876,
+ "theoretical_loss": 3.4097926174204836,
+ "tokens_seen": 2384396288
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002243691073219659,
+ "loss": 1.2671,
+ "theoretical_loss": 3.409785133819456,
+ "tokens_seen": 2384461824
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002243530591775326,
+ "loss": 1.2548,
+ "theoretical_loss": 3.409777650481699,
+ "tokens_seen": 2384527360
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002243370110330993,
+ "loss": 1.265,
+ "theoretical_loss": 3.4097701674071956,
+ "tokens_seen": 2384592896
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000224320962888666,
+ "loss": 1.1296,
+ "theoretical_loss": 3.4097626845959303,
+ "tokens_seen": 2384658432
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022430491474423268,
+ "loss": 1.243,
+ "theoretical_loss": 3.409755202047886,
+ "tokens_seen": 2384723968
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022428886659979939,
+ "loss": 1.2226,
+ "theoretical_loss": 3.409747719763047,
+ "tokens_seen": 2384789504
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022427281845536609,
+ "loss": 1.2694,
+ "theoretical_loss": 3.4097402377413957,
+ "tokens_seen": 2384855040
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002242567703109328,
+ "loss": 1.1591,
+ "theoretical_loss": 3.4097327559829163,
+ "tokens_seen": 2384920576
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002242407221664995,
+ "loss": 1.2342,
+ "theoretical_loss": 3.4097252744875917,
+ "tokens_seen": 2384986112
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002242246740220662,
+ "loss": 1.2674,
+ "theoretical_loss": 3.4097177932554064,
+ "tokens_seen": 2385051648
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002242086258776329,
+ "loss": 1.2491,
+ "theoretical_loss": 3.409710312286343,
+ "tokens_seen": 2385117184
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002241925777331996,
+ "loss": 1.2174,
+ "theoretical_loss": 3.4097028315803857,
+ "tokens_seen": 2385182720
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002241765295887663,
+ "loss": 1.3088,
+ "theoretical_loss": 3.409695351137518,
+ "tokens_seen": 2385248256
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000224160481444333,
+ "loss": 1.287,
+ "theoretical_loss": 3.4096878709577223,
+ "tokens_seen": 2385313792
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002241444332998997,
+ "loss": 1.2305,
+ "theoretical_loss": 3.4096803910409834,
+ "tokens_seen": 2385379328
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002241283851554664,
+ "loss": 1.279,
+ "theoretical_loss": 3.4096729113872843,
+ "tokens_seen": 2385444864
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 797452,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2669398784637451,
+ "objective/train/theoretical_loss": 3.4096654319966087,
+ "objective/train/tokens_used": 2405970400,
+ "theoretical_loss": 3.4096654319966087,
+ "tokens_seen": 2385510400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002241123370110331,
+ "loss": 1.1906,
+ "theoretical_loss": 3.4096654319966087,
+ "tokens_seen": 2385510400
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002240962888665998,
+ "loss": 1.2593,
+ "theoretical_loss": 3.40965795286894,
+ "tokens_seen": 2385575936
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002240802407221665,
+ "loss": 1.1316,
+ "theoretical_loss": 3.4096504740042617,
+ "tokens_seen": 2385641472
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002240641925777332,
+ "loss": 1.2538,
+ "theoretical_loss": 3.4096429954025576,
+ "tokens_seen": 2385707008
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002240481444332999,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4096355170638106,
+ "tokens_seen": 2385772544
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002240320962888666,
+ "loss": 1.2403,
+ "theoretical_loss": 3.409628038988005,
+ "tokens_seen": 2385838080
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022401604814443332,
+ "loss": 1.1189,
+ "theoretical_loss": 3.4096205611751236,
+ "tokens_seen": 2385903616
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022400000000000002,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4096130836251506,
+ "tokens_seen": 2385969152
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022398395185556672,
+ "loss": 1.2988,
+ "theoretical_loss": 3.409605606338069,
+ "tokens_seen": 2386034688
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022396790371113343,
+ "loss": 1.2374,
+ "theoretical_loss": 3.4095981293138626,
+ "tokens_seen": 2386100224
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022395185556670013,
+ "loss": 1.2335,
+ "theoretical_loss": 3.409590652552515,
+ "tokens_seen": 2386165760
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022393580742226683,
+ "loss": 1.1489,
+ "theoretical_loss": 3.4095831760540096,
+ "tokens_seen": 2386231296
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022391975927783353,
+ "loss": 1.3527,
+ "theoretical_loss": 3.4095756998183298,
+ "tokens_seen": 2386296832
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022390371113340023,
+ "loss": 1.1674,
+ "theoretical_loss": 3.4095682238454597,
+ "tokens_seen": 2386362368
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022388766298896693,
+ "loss": 1.3141,
+ "theoretical_loss": 3.4095607481353825,
+ "tokens_seen": 2386427904
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022387161484453363,
+ "loss": 1.2816,
+ "theoretical_loss": 3.4095532726880813,
+ "tokens_seen": 2386493440
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022385556670010033,
+ "loss": 1.2809,
+ "theoretical_loss": 3.4095457975035406,
+ "tokens_seen": 2386558976
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022383951855566703,
+ "loss": 1.1744,
+ "theoretical_loss": 3.409538322581743,
+ "tokens_seen": 2386624512
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022382347041123373,
+ "loss": 1.2131,
+ "theoretical_loss": 3.4095308479226727,
+ "tokens_seen": 2386690048
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022380742226680043,
+ "loss": 1.2657,
+ "theoretical_loss": 3.4095233735263126,
+ "tokens_seen": 2386755584
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022379137412236713,
+ "loss": 1.2441,
+ "theoretical_loss": 3.4095158993926473,
+ "tokens_seen": 2386821120
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022377532597793383,
+ "loss": 1.3152,
+ "theoretical_loss": 3.4095084255216594,
+ "tokens_seen": 2386886656
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022375927783350053,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4095009519133326,
+ "tokens_seen": 2386952192
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022374322968906723,
+ "loss": 1.1604,
+ "theoretical_loss": 3.409493478567651,
+ "tokens_seen": 2387017728
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022372718154463393,
+ "loss": 1.2376,
+ "theoretical_loss": 3.409486005484598,
+ "tokens_seen": 2387083264
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022371113340020064,
+ "loss": 1.2404,
+ "theoretical_loss": 3.4094785326641563,
+ "tokens_seen": 2387148800
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022369508525576734,
+ "loss": 1.2257,
+ "theoretical_loss": 3.4094710601063105,
+ "tokens_seen": 2387214336
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000223679037111334,
+ "loss": 1.2438,
+ "theoretical_loss": 3.409463587811044,
+ "tokens_seen": 2387279872
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002236629889669007,
+ "loss": 1.2165,
+ "theoretical_loss": 3.40945611577834,
+ "tokens_seen": 2387345408
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002236469408224674,
+ "loss": 1.315,
+ "theoretical_loss": 3.409448644008182,
+ "tokens_seen": 2387410944
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002236308926780341,
+ "loss": 1.1477,
+ "theoretical_loss": 3.409441172500554,
+ "tokens_seen": 2387476480
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002236148445336008,
+ "loss": 1.2359,
+ "theoretical_loss": 3.4094337012554394,
+ "tokens_seen": 2387542016
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002235987963891675,
+ "loss": 1.2446,
+ "theoretical_loss": 3.4094262302728215,
+ "tokens_seen": 2387607552
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002235827482447342,
+ "loss": 1.1663,
+ "theoretical_loss": 3.4094187595526844,
+ "tokens_seen": 2387673088
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022356670010030091,
+ "loss": 1.3253,
+ "theoretical_loss": 3.409411289095011,
+ "tokens_seen": 2387738624
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022355065195586761,
+ "loss": 1.2764,
+ "theoretical_loss": 3.4094038188997855,
+ "tokens_seen": 2387804160
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022353460381143432,
+ "loss": 1.2126,
+ "theoretical_loss": 3.409396348966991,
+ "tokens_seen": 2387869696
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022351855566700102,
+ "loss": 1.2194,
+ "theoretical_loss": 3.409388879296612,
+ "tokens_seen": 2387935232
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022350250752256772,
+ "loss": 1.1356,
+ "theoretical_loss": 3.4093814098886304,
+ "tokens_seen": 2388000768
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022348645937813442,
+ "loss": 1.3522,
+ "theoretical_loss": 3.4093739407430315,
+ "tokens_seen": 2388066304
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022347041123370112,
+ "loss": 1.3022,
+ "theoretical_loss": 3.409366471859798,
+ "tokens_seen": 2388131840
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022345436308926782,
+ "loss": 1.3022,
+ "theoretical_loss": 3.4093590032389134,
+ "tokens_seen": 2388197376
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022343831494483452,
+ "loss": 1.3056,
+ "theoretical_loss": 3.409351534880362,
+ "tokens_seen": 2388262912
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022342226680040122,
+ "loss": 1.2872,
+ "theoretical_loss": 3.4093440667841266,
+ "tokens_seen": 2388328448
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022340621865596792,
+ "loss": 1.2934,
+ "theoretical_loss": 3.409336598950191,
+ "tokens_seen": 2388393984
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022339017051153462,
+ "loss": 1.3046,
+ "theoretical_loss": 3.4093291313785388,
+ "tokens_seen": 2388459520
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022337412236710132,
+ "loss": 1.205,
+ "theoretical_loss": 3.409321664069154,
+ "tokens_seen": 2388525056
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022335807422266802,
+ "loss": 1.2247,
+ "theoretical_loss": 3.4093141970220198,
+ "tokens_seen": 2388590592
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022334202607823472,
+ "loss": 1.3195,
+ "theoretical_loss": 3.4093067302371196,
+ "tokens_seen": 2388656128
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022332597793380142,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4092992637144377,
+ "tokens_seen": 2388721664
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 798410,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2534170150756836,
+ "objective/train/theoretical_loss": 3.409291797453957,
+ "objective/train/tokens_used": 2409247200,
+ "theoretical_loss": 3.409291797453957,
+ "tokens_seen": 2388787200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022330992978936812,
+ "loss": 1.2783,
+ "theoretical_loss": 3.409291797453957,
+ "tokens_seen": 2388787200
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232938816449348,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4092843314556616,
+ "tokens_seen": 2388852736
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232778335005015,
+ "loss": 1.2042,
+ "theoretical_loss": 3.4092768657195345,
+ "tokens_seen": 2388918272
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232617853560682,
+ "loss": 1.2259,
+ "theoretical_loss": 3.40926940024556,
+ "tokens_seen": 2388983808
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232457372116349,
+ "loss": 1.272,
+ "theoretical_loss": 3.4092619350337214,
+ "tokens_seen": 2389049344
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232296890672016,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4092544700840017,
+ "tokens_seen": 2389114880
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002232136409227683,
+ "loss": 1.2495,
+ "theoretical_loss": 3.4092470053963857,
+ "tokens_seen": 2389180416
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000223197592778335,
+ "loss": 1.3051,
+ "theoretical_loss": 3.4092395409708565,
+ "tokens_seen": 2389245952
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231815446339017,
+ "loss": 1.2099,
+ "theoretical_loss": 3.409232076807397,
+ "tokens_seen": 2389311488
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231654964894684,
+ "loss": 1.3029,
+ "theoretical_loss": 3.4092246129059918,
+ "tokens_seen": 2389377024
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231494483450351,
+ "loss": 1.2125,
+ "theoretical_loss": 3.409217149266624,
+ "tokens_seen": 2389442560
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231334002006018,
+ "loss": 1.2874,
+ "theoretical_loss": 3.4092096858892775,
+ "tokens_seen": 2389508096
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231173520561685,
+ "loss": 1.2268,
+ "theoretical_loss": 3.4092022227739354,
+ "tokens_seen": 2389573632
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002231013039117352,
+ "loss": 1.2889,
+ "theoretical_loss": 3.409194759920582,
+ "tokens_seen": 2389639168
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002230852557673019,
+ "loss": 1.3325,
+ "theoretical_loss": 3.4091872973292006,
+ "tokens_seen": 2389704704
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002230692076228686,
+ "loss": 1.2182,
+ "theoretical_loss": 3.4091798349997746,
+ "tokens_seen": 2389770240
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002230531594784353,
+ "loss": 1.2603,
+ "theoretical_loss": 3.409172372932288,
+ "tokens_seen": 2389835776
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000223037111334002,
+ "loss": 1.2273,
+ "theoretical_loss": 3.409164911126724,
+ "tokens_seen": 2389901312
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002230210631895687,
+ "loss": 1.1944,
+ "theoretical_loss": 3.4091574495830668,
+ "tokens_seen": 2389966848
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002230050150451354,
+ "loss": 1.1673,
+ "theoretical_loss": 3.4091499883012997,
+ "tokens_seen": 2390032384
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002229889669007021,
+ "loss": 1.1867,
+ "theoretical_loss": 3.409142527281406,
+ "tokens_seen": 2390097920
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002229729187562688,
+ "loss": 1.1394,
+ "theoretical_loss": 3.40913506652337,
+ "tokens_seen": 2390163456
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002229568706118355,
+ "loss": 1.2844,
+ "theoretical_loss": 3.409127606027175,
+ "tokens_seen": 2390228992
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002229408224674022,
+ "loss": 1.2896,
+ "theoretical_loss": 3.409120145792804,
+ "tokens_seen": 2390294528
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002229247743229689,
+ "loss": 1.175,
+ "theoretical_loss": 3.409112685820242,
+ "tokens_seen": 2390360064
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022290872617853559,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4091052261094714,
+ "tokens_seen": 2390425600
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022289267803410229,
+ "loss": 1.2125,
+ "theoretical_loss": 3.4090977666604765,
+ "tokens_seen": 2390491136
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.000222876629889669,
+ "loss": 1.2967,
+ "theoretical_loss": 3.4090903074732406,
+ "tokens_seen": 2390556672
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002228605817452357,
+ "loss": 1.2522,
+ "theoretical_loss": 3.409082848547748,
+ "tokens_seen": 2390622208
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002228445336008024,
+ "loss": 1.2405,
+ "theoretical_loss": 3.409075389883981,
+ "tokens_seen": 2390687744
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.0002228284854563691,
+ "loss": 1.145,
+ "theoretical_loss": 3.4090679314819248,
+ "tokens_seen": 2390753280
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022281243731193584,
+ "loss": 1.2426,
+ "theoretical_loss": 3.409060473341562,
+ "tokens_seen": 2390818816
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022279638916750254,
+ "loss": 1.3112,
+ "theoretical_loss": 3.4090530154628773,
+ "tokens_seen": 2390884352
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022278034102306924,
+ "loss": 1.2625,
+ "theoretical_loss": 3.409045557845853,
+ "tokens_seen": 2390949888
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022276429287863595,
+ "loss": 1.2141,
+ "theoretical_loss": 3.409038100490473,
+ "tokens_seen": 2391015424
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022274824473420265,
+ "loss": 1.1297,
+ "theoretical_loss": 3.4090306433967217,
+ "tokens_seen": 2391080960
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022273219658976935,
+ "loss": 1.2906,
+ "theoretical_loss": 3.4090231865645824,
+ "tokens_seen": 2391146496
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022271614844533605,
+ "loss": 1.2714,
+ "theoretical_loss": 3.4090157299940387,
+ "tokens_seen": 2391212032
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022270010030090275,
+ "loss": 1.1742,
+ "theoretical_loss": 3.4090082736850746,
+ "tokens_seen": 2391277568
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022268405215646945,
+ "loss": 1.2814,
+ "theoretical_loss": 3.409000817637673,
+ "tokens_seen": 2391343104
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022266800401203612,
+ "loss": 1.2991,
+ "theoretical_loss": 3.4089933618518176,
+ "tokens_seen": 2391408640
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022265195586760282,
+ "loss": 1.3264,
+ "theoretical_loss": 3.4089859063274934,
+ "tokens_seen": 2391474176
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022263590772316952,
+ "loss": 1.218,
+ "theoretical_loss": 3.4089784510646823,
+ "tokens_seen": 2391539712
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022261985957873622,
+ "loss": 1.2272,
+ "theoretical_loss": 3.408970996063369,
+ "tokens_seen": 2391605248
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022260381143430292,
+ "loss": 1.214,
+ "theoretical_loss": 3.408963541323537,
+ "tokens_seen": 2391670784
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022258776328986963,
+ "loss": 1.2642,
+ "theoretical_loss": 3.40895608684517,
+ "tokens_seen": 2391736320
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022257171514543633,
+ "loss": 1.2275,
+ "theoretical_loss": 3.4089486326282517,
+ "tokens_seen": 2391801856
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022255566700100303,
+ "loss": 1.2221,
+ "theoretical_loss": 3.408941178672765,
+ "tokens_seen": 2391867392
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022253961885656973,
+ "loss": 1.19,
+ "theoretical_loss": 3.4089337249786946,
+ "tokens_seen": 2391932928
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022252357071213643,
+ "loss": 1.258,
+ "theoretical_loss": 3.408926271546024,
+ "tokens_seen": 2391998464
+ },
+ {
+ "epoch": 0.72,
+ "objective/train/docs_used": 799627,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3175032138824463,
+ "objective/train/theoretical_loss": 3.4089188183747363,
+ "objective/train/tokens_used": 2412524000,
+ "theoretical_loss": 3.4089188183747363,
+ "tokens_seen": 2392064000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022250752256770313,
+ "loss": 1.2993,
+ "theoretical_loss": 3.4089188183747363,
+ "tokens_seen": 2392064000
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022249147442326983,
+ "loss": 1.3432,
+ "theoretical_loss": 3.4089113654648155,
+ "tokens_seen": 2392129536
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022247542627883653,
+ "loss": 1.2474,
+ "theoretical_loss": 3.4089039128162453,
+ "tokens_seen": 2392195072
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022245937813440323,
+ "loss": 1.3012,
+ "theoretical_loss": 3.408896460429009,
+ "tokens_seen": 2392260608
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022244332998996993,
+ "loss": 1.2082,
+ "theoretical_loss": 3.4088890083030914,
+ "tokens_seen": 2392326144
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022242728184553663,
+ "loss": 1.2625,
+ "theoretical_loss": 3.408881556438475,
+ "tokens_seen": 2392391680
+ },
+ {
+ "epoch": 0.72,
+ "learning_rate": 0.00022241123370110333,
+ "loss": 1.2528,
+ "theoretical_loss": 3.408874104835144,
+ "tokens_seen": 2392457216
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022239518555667003,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4088666534930816,
+ "tokens_seen": 2392522752
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022237913741223673,
+ "loss": 1.2028,
+ "theoretical_loss": 3.4088592024122724,
+ "tokens_seen": 2392588288
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022236308926780343,
+ "loss": 1.1418,
+ "theoretical_loss": 3.408851751592699,
+ "tokens_seen": 2392653824
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022234704112337013,
+ "loss": 1.1811,
+ "theoretical_loss": 3.408844301034346,
+ "tokens_seen": 2392719360
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022233099297893684,
+ "loss": 1.2287,
+ "theoretical_loss": 3.4088368507371967,
+ "tokens_seen": 2392784896
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022231494483450354,
+ "loss": 1.1706,
+ "theoretical_loss": 3.4088294007012347,
+ "tokens_seen": 2392850432
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022229889669007024,
+ "loss": 1.3106,
+ "theoretical_loss": 3.4088219509264435,
+ "tokens_seen": 2392915968
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002222828485456369,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4088145014128077,
+ "tokens_seen": 2392981504
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002222668004012036,
+ "loss": 1.2969,
+ "theoretical_loss": 3.4088070521603098,
+ "tokens_seen": 2393047040
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002222507522567703,
+ "loss": 1.2002,
+ "theoretical_loss": 3.4087996031689345,
+ "tokens_seen": 2393112576
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.000222234704112337,
+ "loss": 1.1961,
+ "theoretical_loss": 3.408792154438665,
+ "tokens_seen": 2393178112
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002222186559679037,
+ "loss": 1.0998,
+ "theoretical_loss": 3.408784705969485,
+ "tokens_seen": 2393243648
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002222026078234704,
+ "loss": 1.1879,
+ "theoretical_loss": 3.408777257761378,
+ "tokens_seen": 2393309184
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022218655967903711,
+ "loss": 1.2168,
+ "theoretical_loss": 3.408769809814328,
+ "tokens_seen": 2393374720
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022217051153460381,
+ "loss": 1.2903,
+ "theoretical_loss": 3.408762362128319,
+ "tokens_seen": 2393440256
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022215446339017052,
+ "loss": 1.185,
+ "theoretical_loss": 3.4087549147033345,
+ "tokens_seen": 2393505792
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022213841524573722,
+ "loss": 1.2352,
+ "theoretical_loss": 3.4087474675393574,
+ "tokens_seen": 2393571328
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022212236710130392,
+ "loss": 1.2981,
+ "theoretical_loss": 3.4087400206363725,
+ "tokens_seen": 2393636864
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022210631895687062,
+ "loss": 1.1953,
+ "theoretical_loss": 3.4087325739943632,
+ "tokens_seen": 2393702400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022209027081243732,
+ "loss": 1.2426,
+ "theoretical_loss": 3.408725127613313,
+ "tokens_seen": 2393767936
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022207422266800402,
+ "loss": 1.2118,
+ "theoretical_loss": 3.4087176814932056,
+ "tokens_seen": 2393833472
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022205817452357072,
+ "loss": 1.3016,
+ "theoretical_loss": 3.408710235634025,
+ "tokens_seen": 2393899008
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022204212637913742,
+ "loss": 1.2099,
+ "theoretical_loss": 3.4087027900357545,
+ "tokens_seen": 2393964544
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022202607823470412,
+ "loss": 1.2484,
+ "theoretical_loss": 3.408695344698378,
+ "tokens_seen": 2394030080
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022201003009027082,
+ "loss": 1.2916,
+ "theoretical_loss": 3.4086878996218797,
+ "tokens_seen": 2394095616
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022199398194583752,
+ "loss": 1.2989,
+ "theoretical_loss": 3.408680454806243,
+ "tokens_seen": 2394161152
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022197793380140422,
+ "loss": 1.1716,
+ "theoretical_loss": 3.408673010251451,
+ "tokens_seen": 2394226688
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022196188565697092,
+ "loss": 1.179,
+ "theoretical_loss": 3.408665565957488,
+ "tokens_seen": 2394292224
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022194583751253762,
+ "loss": 1.2377,
+ "theoretical_loss": 3.4086581219243377,
+ "tokens_seen": 2394357760
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022192978936810432,
+ "loss": 1.187,
+ "theoretical_loss": 3.4086506781519836,
+ "tokens_seen": 2394423296
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022191374122367102,
+ "loss": 1.1971,
+ "theoretical_loss": 3.40864323464041,
+ "tokens_seen": 2394488832
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218976930792377,
+ "loss": 1.1876,
+ "theoretical_loss": 3.4086357913896,
+ "tokens_seen": 2394554368
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218816449348044,
+ "loss": 1.2174,
+ "theoretical_loss": 3.408628348399538,
+ "tokens_seen": 2394619904
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218655967903711,
+ "loss": 1.2872,
+ "theoretical_loss": 3.4086209056702064,
+ "tokens_seen": 2394685440
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218495486459378,
+ "loss": 1.2512,
+ "theoretical_loss": 3.4086134632015903,
+ "tokens_seen": 2394750976
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218335005015045,
+ "loss": 1.2803,
+ "theoretical_loss": 3.4086060209936733,
+ "tokens_seen": 2394816512
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218174523570712,
+ "loss": 1.2183,
+ "theoretical_loss": 3.4085985790464384,
+ "tokens_seen": 2394882048
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002218014042126379,
+ "loss": 1.174,
+ "theoretical_loss": 3.4085911373598696,
+ "tokens_seen": 2394947584
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002217853560682046,
+ "loss": 1.1897,
+ "theoretical_loss": 3.4085836959339506,
+ "tokens_seen": 2395013120
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002217693079237713,
+ "loss": 1.1218,
+ "theoretical_loss": 3.4085762547686658,
+ "tokens_seen": 2395078656
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.000221753259779338,
+ "loss": 1.2278,
+ "theoretical_loss": 3.4085688138639982,
+ "tokens_seen": 2395144192
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002217372116349047,
+ "loss": 1.2749,
+ "theoretical_loss": 3.408561373219932,
+ "tokens_seen": 2395209728
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002217211634904714,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4085539328364503,
+ "tokens_seen": 2395275264
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 800556,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0488104820251465,
+ "objective/train/theoretical_loss": 3.4085464927135374,
+ "objective/train/tokens_used": 2415800800,
+ "theoretical_loss": 3.4085464927135374,
+ "tokens_seen": 2395340800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002217051153460381,
+ "loss": 1.0931,
+ "theoretical_loss": 3.4085464927135374,
+ "tokens_seen": 2395340800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002216890672016048,
+ "loss": 1.2178,
+ "theoretical_loss": 3.408539052851177,
+ "tokens_seen": 2395406336
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002216730190571715,
+ "loss": 1.1604,
+ "theoretical_loss": 3.408531613249353,
+ "tokens_seen": 2395471872
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002216569709127382,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4085241739080487,
+ "tokens_seen": 2395537408
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002216409227683049,
+ "loss": 1.2212,
+ "theoretical_loss": 3.4085167348272476,
+ "tokens_seen": 2395602944
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002216248746238716,
+ "loss": 1.0609,
+ "theoretical_loss": 3.4085092960069345,
+ "tokens_seen": 2395668480
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022160882647943834,
+ "loss": 1.2801,
+ "theoretical_loss": 3.408501857447092,
+ "tokens_seen": 2395734016
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022159277833500504,
+ "loss": 1.3119,
+ "theoretical_loss": 3.408494419147705,
+ "tokens_seen": 2395799552
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022157673019057174,
+ "loss": 1.2373,
+ "theoretical_loss": 3.408486981108757,
+ "tokens_seen": 2395865088
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022156068204613844,
+ "loss": 1.1034,
+ "theoretical_loss": 3.4084795433302304,
+ "tokens_seen": 2395930624
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022154463390170514,
+ "loss": 1.2031,
+ "theoretical_loss": 3.408472105812111,
+ "tokens_seen": 2395996160
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022152858575727184,
+ "loss": 1.2993,
+ "theoretical_loss": 3.4084646685543807,
+ "tokens_seen": 2396061696
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022151253761283854,
+ "loss": 1.3134,
+ "theoretical_loss": 3.4084572315570245,
+ "tokens_seen": 2396127232
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022149648946840524,
+ "loss": 1.2956,
+ "theoretical_loss": 3.408449794820026,
+ "tokens_seen": 2396192768
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022148044132397194,
+ "loss": 1.315,
+ "theoretical_loss": 3.4084423583433683,
+ "tokens_seen": 2396258304
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022146439317953864,
+ "loss": 1.2522,
+ "theoretical_loss": 3.408434922127036,
+ "tokens_seen": 2396323840
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022144834503510534,
+ "loss": 1.2484,
+ "theoretical_loss": 3.408427486171012,
+ "tokens_seen": 2396389376
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022143229689067204,
+ "loss": 1.2024,
+ "theoretical_loss": 3.408420050475281,
+ "tokens_seen": 2396454912
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022141624874623874,
+ "loss": 1.1701,
+ "theoretical_loss": 3.4084126150398255,
+ "tokens_seen": 2396520448
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022140020060180545,
+ "loss": 1.2244,
+ "theoretical_loss": 3.408405179864631,
+ "tokens_seen": 2396585984
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022138415245737215,
+ "loss": 1.2387,
+ "theoretical_loss": 3.40839774494968,
+ "tokens_seen": 2396651520
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022136810431293885,
+ "loss": 1.1914,
+ "theoretical_loss": 3.4083903102949566,
+ "tokens_seen": 2396717056
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022135205616850555,
+ "loss": 1.2835,
+ "theoretical_loss": 3.408382875900445,
+ "tokens_seen": 2396782592
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022133600802407225,
+ "loss": 1.2794,
+ "theoretical_loss": 3.408375441766128,
+ "tokens_seen": 2396848128
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022131995987963895,
+ "loss": 1.1989,
+ "theoretical_loss": 3.4083680078919905,
+ "tokens_seen": 2396913664
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022130391173520565,
+ "loss": 1.211,
+ "theoretical_loss": 3.4083605742780154,
+ "tokens_seen": 2396979200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022128786359077235,
+ "loss": 1.2108,
+ "theoretical_loss": 3.408353140924187,
+ "tokens_seen": 2397044736
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022127181544633905,
+ "loss": 1.2058,
+ "theoretical_loss": 3.4083457078304886,
+ "tokens_seen": 2397110272
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022125576730190572,
+ "loss": 1.3112,
+ "theoretical_loss": 3.408338274996905,
+ "tokens_seen": 2397175808
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022123971915747242,
+ "loss": 1.2715,
+ "theoretical_loss": 3.4083308424234184,
+ "tokens_seen": 2397241344
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022122367101303913,
+ "loss": 1.1728,
+ "theoretical_loss": 3.408323410110014,
+ "tokens_seen": 2397306880
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022120762286860583,
+ "loss": 1.2145,
+ "theoretical_loss": 3.4083159780566747,
+ "tokens_seen": 2397372416
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022119157472417253,
+ "loss": 1.2114,
+ "theoretical_loss": 3.408308546263385,
+ "tokens_seen": 2397437952
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022117552657973923,
+ "loss": 1.2763,
+ "theoretical_loss": 3.4083011147301283,
+ "tokens_seen": 2397503488
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022115947843530593,
+ "loss": 1.2531,
+ "theoretical_loss": 3.4082936834568884,
+ "tokens_seen": 2397569024
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022114343029087263,
+ "loss": 1.2448,
+ "theoretical_loss": 3.4082862524436495,
+ "tokens_seen": 2397634560
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022112738214643933,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4082788216903945,
+ "tokens_seen": 2397700096
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022111133400200603,
+ "loss": 1.3908,
+ "theoretical_loss": 3.4082713911971076,
+ "tokens_seen": 2397765632
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022109528585757273,
+ "loss": 1.2614,
+ "theoretical_loss": 3.4082639609637733,
+ "tokens_seen": 2397831168
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022107923771313943,
+ "loss": 1.2747,
+ "theoretical_loss": 3.408256530990375,
+ "tokens_seen": 2397896704
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022106318956870613,
+ "loss": 1.1815,
+ "theoretical_loss": 3.408249101276896,
+ "tokens_seen": 2397962240
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022104714142427283,
+ "loss": 1.1686,
+ "theoretical_loss": 3.40824167182332,
+ "tokens_seen": 2398027776
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022103109327983953,
+ "loss": 1.2351,
+ "theoretical_loss": 3.4082342426296317,
+ "tokens_seen": 2398093312
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022101504513540623,
+ "loss": 1.1671,
+ "theoretical_loss": 3.4082268136958143,
+ "tokens_seen": 2398158848
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022099899699097293,
+ "loss": 1.245,
+ "theoretical_loss": 3.4082193850218516,
+ "tokens_seen": 2398224384
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022098294884653963,
+ "loss": 1.1604,
+ "theoretical_loss": 3.408211956607728,
+ "tokens_seen": 2398289920
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022096690070210634,
+ "loss": 1.2951,
+ "theoretical_loss": 3.408204528453427,
+ "tokens_seen": 2398355456
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022095085255767304,
+ "loss": 1.21,
+ "theoretical_loss": 3.4081971005589318,
+ "tokens_seen": 2398420992
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022093480441323974,
+ "loss": 1.2221,
+ "theoretical_loss": 3.4081896729242267,
+ "tokens_seen": 2398486528
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022091875626880644,
+ "loss": 1.305,
+ "theoretical_loss": 3.4081822455492956,
+ "tokens_seen": 2398552064
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 801784,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2902165651321411,
+ "objective/train/theoretical_loss": 3.4081748184341225,
+ "objective/train/tokens_used": 2419077600,
+ "theoretical_loss": 3.4081748184341225,
+ "tokens_seen": 2398617600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022090270812437314,
+ "loss": 1.2427,
+ "theoretical_loss": 3.4081748184341225,
+ "tokens_seen": 2398617600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022088665997993984,
+ "loss": 1.2642,
+ "theoretical_loss": 3.408167391578691,
+ "tokens_seen": 2398683136
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002208706118355065,
+ "loss": 1.3026,
+ "theoretical_loss": 3.4081599649829846,
+ "tokens_seen": 2398748672
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002208545636910732,
+ "loss": 1.2524,
+ "theoretical_loss": 3.4081525386469878,
+ "tokens_seen": 2398814208
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002208385155466399,
+ "loss": 1.2735,
+ "theoretical_loss": 3.4081451125706836,
+ "tokens_seen": 2398879744
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022082246740220661,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4081376867540563,
+ "tokens_seen": 2398945280
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022080641925777331,
+ "loss": 1.2326,
+ "theoretical_loss": 3.4081302611970896,
+ "tokens_seen": 2399010816
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022079037111334002,
+ "loss": 1.2457,
+ "theoretical_loss": 3.4081228358997677,
+ "tokens_seen": 2399076352
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022077432296890672,
+ "loss": 1.1637,
+ "theoretical_loss": 3.408115410862074,
+ "tokens_seen": 2399141888
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022075827482447342,
+ "loss": 1.2047,
+ "theoretical_loss": 3.4081079860839925,
+ "tokens_seen": 2399207424
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022074222668004012,
+ "loss": 1.227,
+ "theoretical_loss": 3.408100561565507,
+ "tokens_seen": 2399272960
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022072617853560682,
+ "loss": 1.28,
+ "theoretical_loss": 3.4080931373066012,
+ "tokens_seen": 2399338496
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022071013039117352,
+ "loss": 1.1592,
+ "theoretical_loss": 3.4080857133072593,
+ "tokens_seen": 2399404032
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022069408224674022,
+ "loss": 1.3136,
+ "theoretical_loss": 3.408078289567465,
+ "tokens_seen": 2399469568
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022067803410230692,
+ "loss": 1.2349,
+ "theoretical_loss": 3.4080708660872014,
+ "tokens_seen": 2399535104
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022066198595787362,
+ "loss": 1.2151,
+ "theoretical_loss": 3.4080634428664536,
+ "tokens_seen": 2399600640
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022064593781344032,
+ "loss": 1.227,
+ "theoretical_loss": 3.408056019905205,
+ "tokens_seen": 2399666176
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022062988966900702,
+ "loss": 1.2477,
+ "theoretical_loss": 3.4080485972034387,
+ "tokens_seen": 2399731712
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022061384152457372,
+ "loss": 1.2814,
+ "theoretical_loss": 3.4080411747611388,
+ "tokens_seen": 2399797248
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022059779338014042,
+ "loss": 1.2047,
+ "theoretical_loss": 3.40803375257829,
+ "tokens_seen": 2399862784
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022058174523570712,
+ "loss": 1.3245,
+ "theoretical_loss": 3.4080263306548755,
+ "tokens_seen": 2399928320
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022056569709127382,
+ "loss": 1.2007,
+ "theoretical_loss": 3.4080189089908792,
+ "tokens_seen": 2399993856
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022054964894684052,
+ "loss": 1.3298,
+ "theoretical_loss": 3.4080114875862852,
+ "tokens_seen": 2400059392
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022053360080240723,
+ "loss": 1.344,
+ "theoretical_loss": 3.408004066441077,
+ "tokens_seen": 2400124928
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022051755265797393,
+ "loss": 1.2251,
+ "theoretical_loss": 3.4079966455552384,
+ "tokens_seen": 2400190464
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022050150451354063,
+ "loss": 1.3187,
+ "theoretical_loss": 3.4079892249287536,
+ "tokens_seen": 2400256000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002204854563691073,
+ "loss": 1.2017,
+ "theoretical_loss": 3.407981804561606,
+ "tokens_seen": 2400321536
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.000220469408224674,
+ "loss": 1.3254,
+ "theoretical_loss": 3.40797438445378,
+ "tokens_seen": 2400387072
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002204533600802407,
+ "loss": 1.2585,
+ "theoretical_loss": 3.4079669646052597,
+ "tokens_seen": 2400452608
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002204373119358074,
+ "loss": 1.2464,
+ "theoretical_loss": 3.407959545016028,
+ "tokens_seen": 2400518144
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002204212637913741,
+ "loss": 1.2757,
+ "theoretical_loss": 3.4079521256860694,
+ "tokens_seen": 2400583680
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022040521564694086,
+ "loss": 1.1847,
+ "theoretical_loss": 3.407944706615367,
+ "tokens_seen": 2400649216
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022038916750250756,
+ "loss": 1.1446,
+ "theoretical_loss": 3.407937287803906,
+ "tokens_seen": 2400714752
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022037311935807426,
+ "loss": 1.2517,
+ "theoretical_loss": 3.4079298692516695,
+ "tokens_seen": 2400780288
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022035707121364096,
+ "loss": 1.2751,
+ "theoretical_loss": 3.407922450958641,
+ "tokens_seen": 2400845824
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022034102306920766,
+ "loss": 1.0793,
+ "theoretical_loss": 3.407915032924805,
+ "tokens_seen": 2400911360
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022032497492477436,
+ "loss": 1.1774,
+ "theoretical_loss": 3.407907615150145,
+ "tokens_seen": 2400976896
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022030892678034106,
+ "loss": 1.3753,
+ "theoretical_loss": 3.407900197634645,
+ "tokens_seen": 2401042432
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022029287863590776,
+ "loss": 1.2475,
+ "theoretical_loss": 3.4078927803782886,
+ "tokens_seen": 2401107968
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022027683049147446,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4078853633810606,
+ "tokens_seen": 2401173504
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022026078234704116,
+ "loss": 1.209,
+ "theoretical_loss": 3.4078779466429436,
+ "tokens_seen": 2401239040
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022024473420260784,
+ "loss": 1.2558,
+ "theoretical_loss": 3.4078705301639225,
+ "tokens_seen": 2401304576
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022022868605817454,
+ "loss": 1.2546,
+ "theoretical_loss": 3.4078631139439803,
+ "tokens_seen": 2401370112
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022021263791374124,
+ "loss": 1.3184,
+ "theoretical_loss": 3.407855697983102,
+ "tokens_seen": 2401435648
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022019658976930794,
+ "loss": 1.1715,
+ "theoretical_loss": 3.407848282281271,
+ "tokens_seen": 2401501184
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022018054162487464,
+ "loss": 1.0906,
+ "theoretical_loss": 3.40784086683847,
+ "tokens_seen": 2401566720
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022016449348044134,
+ "loss": 1.2322,
+ "theoretical_loss": 3.4078334516546844,
+ "tokens_seen": 2401632256
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022014844533600804,
+ "loss": 1.3676,
+ "theoretical_loss": 3.407826036729898,
+ "tokens_seen": 2401697792
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022013239719157474,
+ "loss": 1.193,
+ "theoretical_loss": 3.4078186220640934,
+ "tokens_seen": 2401763328
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022011634904714144,
+ "loss": 1.2047,
+ "theoretical_loss": 3.407811207657256,
+ "tokens_seen": 2401828864
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 802719,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1629681587219238,
+ "objective/train/theoretical_loss": 3.407803793509369,
+ "objective/train/tokens_used": 2422354400,
+ "theoretical_loss": 3.407803793509369,
+ "tokens_seen": 2401894400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022010030090270814,
+ "loss": 1.2282,
+ "theoretical_loss": 3.407803793509369,
+ "tokens_seen": 2401894400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022008425275827484,
+ "loss": 1.2457,
+ "theoretical_loss": 3.4077963796204163,
+ "tokens_seen": 2401959936
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022006820461384154,
+ "loss": 1.2122,
+ "theoretical_loss": 3.4077889659903815,
+ "tokens_seen": 2402025472
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022005215646940824,
+ "loss": 1.2799,
+ "theoretical_loss": 3.4077815526192494,
+ "tokens_seen": 2402091008
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022003610832497494,
+ "loss": 1.2321,
+ "theoretical_loss": 3.407774139507003,
+ "tokens_seen": 2402156544
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022002006018054165,
+ "loss": 1.2217,
+ "theoretical_loss": 3.4077667266536267,
+ "tokens_seen": 2402222080
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00022000401203610835,
+ "loss": 1.1387,
+ "theoretical_loss": 3.4077593140591036,
+ "tokens_seen": 2402287616
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021998796389167505,
+ "loss": 1.3249,
+ "theoretical_loss": 3.407751901723419,
+ "tokens_seen": 2402353152
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021997191574724175,
+ "loss": 1.2354,
+ "theoretical_loss": 3.4077444896465554,
+ "tokens_seen": 2402418688
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021995586760280845,
+ "loss": 1.3425,
+ "theoretical_loss": 3.407737077828498,
+ "tokens_seen": 2402484224
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021993981945837515,
+ "loss": 1.216,
+ "theoretical_loss": 3.4077296662692294,
+ "tokens_seen": 2402549760
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021992377131394185,
+ "loss": 1.2117,
+ "theoretical_loss": 3.4077222549687343,
+ "tokens_seen": 2402615296
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021990772316950855,
+ "loss": 1.0909,
+ "theoretical_loss": 3.4077148439269966,
+ "tokens_seen": 2402680832
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021989167502507525,
+ "loss": 1.1767,
+ "theoretical_loss": 3.407707433144,
+ "tokens_seen": 2402746368
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021987562688064195,
+ "loss": 1.2863,
+ "theoretical_loss": 3.4077000226197285,
+ "tokens_seen": 2402811904
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021985957873620862,
+ "loss": 1.3756,
+ "theoretical_loss": 3.4076926123541655,
+ "tokens_seen": 2402877440
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021984353059177533,
+ "loss": 1.242,
+ "theoretical_loss": 3.407685202347296,
+ "tokens_seen": 2402942976
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021982748244734203,
+ "loss": 1.3055,
+ "theoretical_loss": 3.407677792599103,
+ "tokens_seen": 2403008512
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021981143430290873,
+ "loss": 1.2478,
+ "theoretical_loss": 3.4076703831095707,
+ "tokens_seen": 2403074048
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021979538615847543,
+ "loss": 1.2237,
+ "theoretical_loss": 3.4076629738786828,
+ "tokens_seen": 2403139584
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021977933801404213,
+ "loss": 1.2059,
+ "theoretical_loss": 3.407655564906424,
+ "tokens_seen": 2403205120
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021976328986960883,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4076481561927774,
+ "tokens_seen": 2403270656
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021974724172517553,
+ "loss": 1.1795,
+ "theoretical_loss": 3.407640747737727,
+ "tokens_seen": 2403336192
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021973119358074223,
+ "loss": 1.1827,
+ "theoretical_loss": 3.407633339541257,
+ "tokens_seen": 2403401728
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021971514543630893,
+ "loss": 1.1742,
+ "theoretical_loss": 3.4076259316033513,
+ "tokens_seen": 2403467264
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021969909729187563,
+ "loss": 1.3358,
+ "theoretical_loss": 3.407618523923994,
+ "tokens_seen": 2403532800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021968304914744233,
+ "loss": 1.2418,
+ "theoretical_loss": 3.4076111165031686,
+ "tokens_seen": 2403598336
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021966700100300903,
+ "loss": 1.2747,
+ "theoretical_loss": 3.407603709340859,
+ "tokens_seen": 2403663872
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021965095285857573,
+ "loss": 1.2629,
+ "theoretical_loss": 3.4075963024370495,
+ "tokens_seen": 2403729408
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021963490471414243,
+ "loss": 1.1858,
+ "theoretical_loss": 3.407588895791724,
+ "tokens_seen": 2403794944
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021961885656970913,
+ "loss": 1.3022,
+ "theoretical_loss": 3.407581489404866,
+ "tokens_seen": 2403860480
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021960280842527583,
+ "loss": 1.2555,
+ "theoretical_loss": 3.40757408327646,
+ "tokens_seen": 2403926016
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021958676028084254,
+ "loss": 1.1274,
+ "theoretical_loss": 3.407566677406489,
+ "tokens_seen": 2403991552
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021957071213640924,
+ "loss": 1.3347,
+ "theoretical_loss": 3.4075592717949386,
+ "tokens_seen": 2404057088
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021955466399197594,
+ "loss": 1.281,
+ "theoretical_loss": 3.4075518664417914,
+ "tokens_seen": 2404122624
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021953861584754264,
+ "loss": 1.1966,
+ "theoretical_loss": 3.4075444613470314,
+ "tokens_seen": 2404188160
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021952256770310934,
+ "loss": 1.3001,
+ "theoretical_loss": 3.4075370565106433,
+ "tokens_seen": 2404253696
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021950651955867604,
+ "loss": 1.211,
+ "theoretical_loss": 3.4075296519326104,
+ "tokens_seen": 2404319232
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021949047141424274,
+ "loss": 1.3744,
+ "theoretical_loss": 3.4075222476129166,
+ "tokens_seen": 2404384768
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002194744232698094,
+ "loss": 1.1826,
+ "theoretical_loss": 3.4075148435515463,
+ "tokens_seen": 2404450304
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021945837512537611,
+ "loss": 1.3,
+ "theoretical_loss": 3.407507439748483,
+ "tokens_seen": 2404515840
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021944232698094281,
+ "loss": 1.292,
+ "theoretical_loss": 3.407500036203711,
+ "tokens_seen": 2404581376
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021942627883650951,
+ "loss": 1.3132,
+ "theoretical_loss": 3.407492632917214,
+ "tokens_seen": 2404646912
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021941023069207622,
+ "loss": 1.2438,
+ "theoretical_loss": 3.407485229888976,
+ "tokens_seen": 2404712448
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021939418254764292,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4074778271189814,
+ "tokens_seen": 2404777984
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021937813440320962,
+ "loss": 1.3085,
+ "theoretical_loss": 3.4074704246072134,
+ "tokens_seen": 2404843520
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021936208625877632,
+ "loss": 1.2801,
+ "theoretical_loss": 3.4074630223536566,
+ "tokens_seen": 2404909056
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021934603811434302,
+ "loss": 1.2171,
+ "theoretical_loss": 3.407455620358294,
+ "tokens_seen": 2404974592
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021932998996990972,
+ "loss": 1.1716,
+ "theoretical_loss": 3.407448218621111,
+ "tokens_seen": 2405040128
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021931394182547642,
+ "loss": 1.3297,
+ "theoretical_loss": 3.4074408171420907,
+ "tokens_seen": 2405105664
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 804018,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.286693811416626,
+ "objective/train/theoretical_loss": 3.407433415921217,
+ "objective/train/tokens_used": 2425631200,
+ "theoretical_loss": 3.407433415921217,
+ "tokens_seen": 2405171200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021929789368104312,
+ "loss": 1.2416,
+ "theoretical_loss": 3.407433415921217,
+ "tokens_seen": 2405171200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021928184553660982,
+ "loss": 1.2942,
+ "theoretical_loss": 3.407426014958474,
+ "tokens_seen": 2405236736
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021926579739217652,
+ "loss": 1.1975,
+ "theoretical_loss": 3.407418614253846,
+ "tokens_seen": 2405302272
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021924974924774322,
+ "loss": 1.2337,
+ "theoretical_loss": 3.407411213807316,
+ "tokens_seen": 2405367808
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021923370110330992,
+ "loss": 1.2871,
+ "theoretical_loss": 3.407403813618869,
+ "tokens_seen": 2405433344
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021921765295887665,
+ "loss": 1.1959,
+ "theoretical_loss": 3.4073964136884887,
+ "tokens_seen": 2405498880
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021920160481444335,
+ "loss": 1.3029,
+ "theoretical_loss": 3.407389014016159,
+ "tokens_seen": 2405564416
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021918555667001005,
+ "loss": 1.2495,
+ "theoretical_loss": 3.4073816146018636,
+ "tokens_seen": 2405629952
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021916950852557675,
+ "loss": 1.2344,
+ "theoretical_loss": 3.4073742154455866,
+ "tokens_seen": 2405695488
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021915346038114345,
+ "loss": 1.3191,
+ "theoretical_loss": 3.407366816547312,
+ "tokens_seen": 2405761024
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021913741223671015,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4073594179070246,
+ "tokens_seen": 2405826560
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021912136409227685,
+ "loss": 1.2481,
+ "theoretical_loss": 3.407352019524707,
+ "tokens_seen": 2405892096
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021910531594784355,
+ "loss": 1.2132,
+ "theoretical_loss": 3.4073446214003438,
+ "tokens_seen": 2405957632
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021908926780341026,
+ "loss": 1.226,
+ "theoretical_loss": 3.4073372235339194,
+ "tokens_seen": 2406023168
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021907321965897696,
+ "loss": 1.3179,
+ "theoretical_loss": 3.4073298259254168,
+ "tokens_seen": 2406088704
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021905717151454366,
+ "loss": 1.2909,
+ "theoretical_loss": 3.407322428574821,
+ "tokens_seen": 2406154240
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021904112337011036,
+ "loss": 1.2786,
+ "theoretical_loss": 3.4073150314821152,
+ "tokens_seen": 2406219776
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021902507522567706,
+ "loss": 1.2361,
+ "theoretical_loss": 3.4073076346472844,
+ "tokens_seen": 2406285312
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021900902708124376,
+ "loss": 1.216,
+ "theoretical_loss": 3.4073002380703112,
+ "tokens_seen": 2406350848
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021899297893681046,
+ "loss": 1.1832,
+ "theoretical_loss": 3.4072928417511807,
+ "tokens_seen": 2406416384
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021897693079237716,
+ "loss": 1.2809,
+ "theoretical_loss": 3.407285445689876,
+ "tokens_seen": 2406481920
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021896088264794386,
+ "loss": 1.2102,
+ "theoretical_loss": 3.407278049886382,
+ "tokens_seen": 2406547456
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021894483450351056,
+ "loss": 1.2919,
+ "theoretical_loss": 3.407270654340682,
+ "tokens_seen": 2406612992
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021892878635907726,
+ "loss": 1.2262,
+ "theoretical_loss": 3.4072632590527605,
+ "tokens_seen": 2406678528
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021891273821464396,
+ "loss": 1.2917,
+ "theoretical_loss": 3.407255864022601,
+ "tokens_seen": 2406744064
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021889669007021066,
+ "loss": 1.2023,
+ "theoretical_loss": 3.407248469250188,
+ "tokens_seen": 2406809600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021888064192577736,
+ "loss": 1.1954,
+ "theoretical_loss": 3.4072410747355057,
+ "tokens_seen": 2406875136
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021886459378134406,
+ "loss": 1.2671,
+ "theoretical_loss": 3.4072336804785373,
+ "tokens_seen": 2406940672
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021884854563691074,
+ "loss": 1.2969,
+ "theoretical_loss": 3.4072262864792666,
+ "tokens_seen": 2407006208
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021883249749247744,
+ "loss": 1.3416,
+ "theoretical_loss": 3.4072188927376788,
+ "tokens_seen": 2407071744
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021881644934804414,
+ "loss": 1.3323,
+ "theoretical_loss": 3.4072114992537568,
+ "tokens_seen": 2407137280
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021880040120361084,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4072041060274856,
+ "tokens_seen": 2407202816
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021878435305917754,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4071967130588483,
+ "tokens_seen": 2407268352
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021876830491474424,
+ "loss": 1.2558,
+ "theoretical_loss": 3.4071893203478294,
+ "tokens_seen": 2407333888
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021875225677031094,
+ "loss": 1.185,
+ "theoretical_loss": 3.407181927894413,
+ "tokens_seen": 2407399424
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021873620862587764,
+ "loss": 1.0756,
+ "theoretical_loss": 3.4071745356985823,
+ "tokens_seen": 2407464960
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021872016048144434,
+ "loss": 1.193,
+ "theoretical_loss": 3.4071671437603226,
+ "tokens_seen": 2407530496
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021870411233701104,
+ "loss": 1.1683,
+ "theoretical_loss": 3.407159752079617,
+ "tokens_seen": 2407596032
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021868806419257774,
+ "loss": 1.2234,
+ "theoretical_loss": 3.4071523606564496,
+ "tokens_seen": 2407661568
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021867201604814444,
+ "loss": 1.1158,
+ "theoretical_loss": 3.4071449694908047,
+ "tokens_seen": 2407727104
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021865596790371115,
+ "loss": 1.1532,
+ "theoretical_loss": 3.4071375785826663,
+ "tokens_seen": 2407792640
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021863991975927785,
+ "loss": 1.2889,
+ "theoretical_loss": 3.407130187932018,
+ "tokens_seen": 2407858176
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021862387161484455,
+ "loss": 1.2277,
+ "theoretical_loss": 3.4071227975388445,
+ "tokens_seen": 2407923712
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021860782347041125,
+ "loss": 1.1693,
+ "theoretical_loss": 3.407115407403129,
+ "tokens_seen": 2407989248
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021859177532597795,
+ "loss": 1.2897,
+ "theoretical_loss": 3.4071080175248563,
+ "tokens_seen": 2408054784
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021857572718154465,
+ "loss": 1.2243,
+ "theoretical_loss": 3.4071006279040104,
+ "tokens_seen": 2408120320
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021855967903711135,
+ "loss": 1.1861,
+ "theoretical_loss": 3.4070932385405746,
+ "tokens_seen": 2408185856
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021854363089267805,
+ "loss": 1.2689,
+ "theoretical_loss": 3.4070858494345333,
+ "tokens_seen": 2408251392
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021852758274824475,
+ "loss": 1.2051,
+ "theoretical_loss": 3.4070784605858706,
+ "tokens_seen": 2408316928
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021851153460381145,
+ "loss": 1.1992,
+ "theoretical_loss": 3.407071071994571,
+ "tokens_seen": 2408382464
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 804944,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1068869829177856,
+ "objective/train/theoretical_loss": 3.4070636836606174,
+ "objective/train/tokens_used": 2428908000,
+ "theoretical_loss": 3.4070636836606174,
+ "tokens_seen": 2408448000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021849548645937815,
+ "loss": 1.1614,
+ "theoretical_loss": 3.4070636836606174,
+ "tokens_seen": 2408448000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021847943831494485,
+ "loss": 1.3202,
+ "theoretical_loss": 3.407056295583995,
+ "tokens_seen": 2408513536
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021846339017051153,
+ "loss": 1.1991,
+ "theoretical_loss": 3.407048907764687,
+ "tokens_seen": 2408579072
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021844734202607823,
+ "loss": 1.2757,
+ "theoretical_loss": 3.407041520202678,
+ "tokens_seen": 2408644608
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021843129388164493,
+ "loss": 1.2079,
+ "theoretical_loss": 3.4070341328979517,
+ "tokens_seen": 2408710144
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021841524573721163,
+ "loss": 1.1319,
+ "theoretical_loss": 3.407026745850492,
+ "tokens_seen": 2408775680
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021839919759277833,
+ "loss": 1.1695,
+ "theoretical_loss": 3.4070193590602833,
+ "tokens_seen": 2408841216
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021838314944834503,
+ "loss": 1.2489,
+ "theoretical_loss": 3.40701197252731,
+ "tokens_seen": 2408906752
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021836710130391173,
+ "loss": 1.3349,
+ "theoretical_loss": 3.407004586251555,
+ "tokens_seen": 2408972288
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021835105315947843,
+ "loss": 1.4175,
+ "theoretical_loss": 3.406997200233003,
+ "tokens_seen": 2409037824
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021833500501504513,
+ "loss": 1.2911,
+ "theoretical_loss": 3.406989814471639,
+ "tokens_seen": 2409103360
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021831895687061183,
+ "loss": 1.1659,
+ "theoretical_loss": 3.4069824289674453,
+ "tokens_seen": 2409168896
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021830290872617853,
+ "loss": 1.2157,
+ "theoretical_loss": 3.406975043720407,
+ "tokens_seen": 2409234432
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021828686058174523,
+ "loss": 1.1952,
+ "theoretical_loss": 3.4069676587305078,
+ "tokens_seen": 2409299968
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021827081243731193,
+ "loss": 1.1623,
+ "theoretical_loss": 3.406960273997732,
+ "tokens_seen": 2409365504
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021825476429287863,
+ "loss": 1.143,
+ "theoretical_loss": 3.406952889522063,
+ "tokens_seen": 2409431040
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021823871614844533,
+ "loss": 1.3046,
+ "theoretical_loss": 3.406945505303486,
+ "tokens_seen": 2409496576
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021822266800401204,
+ "loss": 1.2892,
+ "theoretical_loss": 3.4069381213419843,
+ "tokens_seen": 2409562112
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021820661985957874,
+ "loss": 1.2039,
+ "theoretical_loss": 3.406930737637542,
+ "tokens_seen": 2409627648
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021819057171514544,
+ "loss": 1.2999,
+ "theoretical_loss": 3.4069233541901434,
+ "tokens_seen": 2409693184
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021817452357071214,
+ "loss": 1.2166,
+ "theoretical_loss": 3.406915970999772,
+ "tokens_seen": 2409758720
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021815847542627884,
+ "loss": 1.1984,
+ "theoretical_loss": 3.4069085880664125,
+ "tokens_seen": 2409824256
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021814242728184554,
+ "loss": 1.1704,
+ "theoretical_loss": 3.406901205390049,
+ "tokens_seen": 2409889792
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021812637913741224,
+ "loss": 1.2759,
+ "theoretical_loss": 3.406893822970665,
+ "tokens_seen": 2409955328
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021811033099297894,
+ "loss": 1.1901,
+ "theoretical_loss": 3.406886440808245,
+ "tokens_seen": 2410020864
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021809428284854564,
+ "loss": 1.1847,
+ "theoretical_loss": 3.406879058902773,
+ "tokens_seen": 2410086400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021807823470411231,
+ "loss": 1.2611,
+ "theoretical_loss": 3.406871677254233,
+ "tokens_seen": 2410151936
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021806218655967901,
+ "loss": 1.2936,
+ "theoretical_loss": 3.4068642958626087,
+ "tokens_seen": 2410217472
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021804613841524572,
+ "loss": 1.3191,
+ "theoretical_loss": 3.406856914727885,
+ "tokens_seen": 2410283008
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021803009027081242,
+ "loss": 1.1937,
+ "theoretical_loss": 3.4068495338500453,
+ "tokens_seen": 2410348544
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021801404212637917,
+ "loss": 1.3084,
+ "theoretical_loss": 3.406842153229074,
+ "tokens_seen": 2410414080
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021799799398194587,
+ "loss": 1.1911,
+ "theoretical_loss": 3.406834772864955,
+ "tokens_seen": 2410479616
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021798194583751257,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4068273927576724,
+ "tokens_seen": 2410545152
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021796589769307927,
+ "loss": 1.3018,
+ "theoretical_loss": 3.40682001290721,
+ "tokens_seen": 2410610688
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021794984954864597,
+ "loss": 1.2263,
+ "theoretical_loss": 3.406812633313553,
+ "tokens_seen": 2410676224
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021793380140421267,
+ "loss": 1.2147,
+ "theoretical_loss": 3.4068052539766844,
+ "tokens_seen": 2410741760
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021791775325977937,
+ "loss": 1.1811,
+ "theoretical_loss": 3.406797874896588,
+ "tokens_seen": 2410807296
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021790170511534608,
+ "loss": 1.198,
+ "theoretical_loss": 3.406790496073249,
+ "tokens_seen": 2410872832
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021788565697091278,
+ "loss": 1.2642,
+ "theoretical_loss": 3.4067831175066514,
+ "tokens_seen": 2410938368
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021786960882647948,
+ "loss": 1.2298,
+ "theoretical_loss": 3.406775739196778,
+ "tokens_seen": 2411003904
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021785356068204618,
+ "loss": 1.1643,
+ "theoretical_loss": 3.406768361143614,
+ "tokens_seen": 2411069440
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021783751253761288,
+ "loss": 1.2284,
+ "theoretical_loss": 3.406760983347144,
+ "tokens_seen": 2411134976
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021782146439317955,
+ "loss": 1.2323,
+ "theoretical_loss": 3.4067536058073506,
+ "tokens_seen": 2411200512
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021780541624874625,
+ "loss": 1.1574,
+ "theoretical_loss": 3.4067462285242187,
+ "tokens_seen": 2411266048
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021778936810431295,
+ "loss": 1.2621,
+ "theoretical_loss": 3.406738851497732,
+ "tokens_seen": 2411331584
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021777331995987965,
+ "loss": 1.291,
+ "theoretical_loss": 3.4067314747278754,
+ "tokens_seen": 2411397120
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021775727181544635,
+ "loss": 1.2088,
+ "theoretical_loss": 3.406724098214632,
+ "tokens_seen": 2411462656
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021774122367101305,
+ "loss": 1.2948,
+ "theoretical_loss": 3.406716721957987,
+ "tokens_seen": 2411528192
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021772517552657976,
+ "loss": 1.1687,
+ "theoretical_loss": 3.406709345957924,
+ "tokens_seen": 2411593728
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021770912738214646,
+ "loss": 1.1936,
+ "theoretical_loss": 3.406701970214426,
+ "tokens_seen": 2411659264
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 806106,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.20001220703125,
+ "objective/train/theoretical_loss": 3.4066945947274787,
+ "objective/train/tokens_used": 2432184800,
+ "theoretical_loss": 3.4066945947274787,
+ "tokens_seen": 2411724800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021769307923771316,
+ "loss": 1.2759,
+ "theoretical_loss": 3.4066945947274787,
+ "tokens_seen": 2411724800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021767703109327986,
+ "loss": 1.3005,
+ "theoretical_loss": 3.4066872194970657,
+ "tokens_seen": 2411790336
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021766098294884656,
+ "loss": 1.3568,
+ "theoretical_loss": 3.406679844523171,
+ "tokens_seen": 2411855872
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021764493480441326,
+ "loss": 1.2956,
+ "theoretical_loss": 3.4066724698057786,
+ "tokens_seen": 2411921408
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021762888665997996,
+ "loss": 1.3647,
+ "theoretical_loss": 3.4066650953448727,
+ "tokens_seen": 2411986944
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021761283851554666,
+ "loss": 1.2306,
+ "theoretical_loss": 3.4066577211404376,
+ "tokens_seen": 2412052480
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021759679037111336,
+ "loss": 1.3112,
+ "theoretical_loss": 3.4066503471924574,
+ "tokens_seen": 2412118016
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021758074222668006,
+ "loss": 1.2315,
+ "theoretical_loss": 3.406642973500916,
+ "tokens_seen": 2412183552
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021756469408224676,
+ "loss": 1.2179,
+ "theoretical_loss": 3.4066356000657976,
+ "tokens_seen": 2412249088
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021754864593781346,
+ "loss": 1.2854,
+ "theoretical_loss": 3.406628226887086,
+ "tokens_seen": 2412314624
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021753259779338016,
+ "loss": 1.2285,
+ "theoretical_loss": 3.4066208539647658,
+ "tokens_seen": 2412380160
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021751654964894686,
+ "loss": 1.2477,
+ "theoretical_loss": 3.406613481298821,
+ "tokens_seen": 2412445696
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021750050150451356,
+ "loss": 1.2439,
+ "theoretical_loss": 3.406606108889236,
+ "tokens_seen": 2412511232
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021748445336008026,
+ "loss": 1.3353,
+ "theoretical_loss": 3.406598736735994,
+ "tokens_seen": 2412576768
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021746840521564697,
+ "loss": 1.1836,
+ "theoretical_loss": 3.40659136483908,
+ "tokens_seen": 2412642304
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021745235707121367,
+ "loss": 1.1719,
+ "theoretical_loss": 3.4065839931984776,
+ "tokens_seen": 2412707840
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021743630892678034,
+ "loss": 1.1542,
+ "theoretical_loss": 3.4065766218141715,
+ "tokens_seen": 2412773376
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021742026078234704,
+ "loss": 1.274,
+ "theoretical_loss": 3.4065692506861454,
+ "tokens_seen": 2412838912
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021740421263791374,
+ "loss": 1.196,
+ "theoretical_loss": 3.4065618798143835,
+ "tokens_seen": 2412904448
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021738816449348044,
+ "loss": 1.2472,
+ "theoretical_loss": 3.4065545091988696,
+ "tokens_seen": 2412969984
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021737211634904714,
+ "loss": 1.2604,
+ "theoretical_loss": 3.406547138839589,
+ "tokens_seen": 2413035520
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021735606820461384,
+ "loss": 1.2249,
+ "theoretical_loss": 3.406539768736524,
+ "tokens_seen": 2413101056
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021734002006018054,
+ "loss": 1.202,
+ "theoretical_loss": 3.4065323988896603,
+ "tokens_seen": 2413166592
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021732397191574724,
+ "loss": 1.297,
+ "theoretical_loss": 3.4065250292989817,
+ "tokens_seen": 2413232128
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021730792377131394,
+ "loss": 1.2883,
+ "theoretical_loss": 3.406517659964472,
+ "tokens_seen": 2413297664
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021729187562688065,
+ "loss": 1.1252,
+ "theoretical_loss": 3.406510290886115,
+ "tokens_seen": 2413363200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021727582748244735,
+ "loss": 1.1489,
+ "theoretical_loss": 3.406502922063896,
+ "tokens_seen": 2413428736
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021725977933801405,
+ "loss": 1.1782,
+ "theoretical_loss": 3.4064955534977983,
+ "tokens_seen": 2413494272
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021724373119358075,
+ "loss": 1.2845,
+ "theoretical_loss": 3.406488185187806,
+ "tokens_seen": 2413559808
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021722768304914745,
+ "loss": 1.1681,
+ "theoretical_loss": 3.4064808171339034,
+ "tokens_seen": 2413625344
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021721163490471415,
+ "loss": 1.1187,
+ "theoretical_loss": 3.4064734493360747,
+ "tokens_seen": 2413690880
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021719558676028085,
+ "loss": 1.1811,
+ "theoretical_loss": 3.4064660817943038,
+ "tokens_seen": 2413756416
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021717953861584755,
+ "loss": 1.261,
+ "theoretical_loss": 3.4064587145085756,
+ "tokens_seen": 2413821952
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021716349047141425,
+ "loss": 1.2773,
+ "theoretical_loss": 3.4064513474788733,
+ "tokens_seen": 2413887488
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021714744232698095,
+ "loss": 1.1646,
+ "theoretical_loss": 3.406443980705182,
+ "tokens_seen": 2413953024
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021713139418254765,
+ "loss": 1.1667,
+ "theoretical_loss": 3.4064366141874847,
+ "tokens_seen": 2414018560
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021711534603811435,
+ "loss": 1.181,
+ "theoretical_loss": 3.4064292479257667,
+ "tokens_seen": 2414084096
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021709929789368105,
+ "loss": 1.361,
+ "theoretical_loss": 3.4064218819200116,
+ "tokens_seen": 2414149632
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021708324974924775,
+ "loss": 1.1868,
+ "theoretical_loss": 3.4064145161702033,
+ "tokens_seen": 2414215168
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021706720160481445,
+ "loss": 1.308,
+ "theoretical_loss": 3.4064071506763263,
+ "tokens_seen": 2414280704
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021705115346038113,
+ "loss": 1.1809,
+ "theoretical_loss": 3.406399785438365,
+ "tokens_seen": 2414346240
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021703510531594783,
+ "loss": 1.2798,
+ "theoretical_loss": 3.406392420456303,
+ "tokens_seen": 2414411776
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021701905717151453,
+ "loss": 1.2482,
+ "theoretical_loss": 3.4063850557301247,
+ "tokens_seen": 2414477312
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021700300902708123,
+ "loss": 1.2254,
+ "theoretical_loss": 3.4063776912598147,
+ "tokens_seen": 2414542848
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021698696088264793,
+ "loss": 1.3024,
+ "theoretical_loss": 3.4063703270453565,
+ "tokens_seen": 2414608384
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021697091273821463,
+ "loss": 1.2442,
+ "theoretical_loss": 3.406362963086735,
+ "tokens_seen": 2414673920
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021695486459378133,
+ "loss": 1.3155,
+ "theoretical_loss": 3.406355599383933,
+ "tokens_seen": 2414739456
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021693881644934803,
+ "loss": 1.2043,
+ "theoretical_loss": 3.4063482359369366,
+ "tokens_seen": 2414804992
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021692276830491473,
+ "loss": 1.254,
+ "theoretical_loss": 3.4063408727457283,
+ "tokens_seen": 2414870528
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021690672016048143,
+ "loss": 1.1903,
+ "theoretical_loss": 3.406333509810293,
+ "tokens_seen": 2414936064
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 806981,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.205592393875122,
+ "objective/train/theoretical_loss": 3.406326147130615,
+ "objective/train/tokens_used": 2435461600,
+ "theoretical_loss": 3.406326147130615,
+ "tokens_seen": 2415001600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021689067201604813,
+ "loss": 1.1965,
+ "theoretical_loss": 3.406326147130615,
+ "tokens_seen": 2415001600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021687462387161483,
+ "loss": 1.1723,
+ "theoretical_loss": 3.4063187847066785,
+ "tokens_seen": 2415067136
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021685857572718154,
+ "loss": 1.1292,
+ "theoretical_loss": 3.406311422538467,
+ "tokens_seen": 2415132672
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021684252758274824,
+ "loss": 1.1958,
+ "theoretical_loss": 3.4063040606259656,
+ "tokens_seen": 2415198208
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021682647943831494,
+ "loss": 1.3287,
+ "theoretical_loss": 3.4062966989691574,
+ "tokens_seen": 2415263744
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021681043129388166,
+ "loss": 1.219,
+ "theoretical_loss": 3.406289337568028,
+ "tokens_seen": 2415329280
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021679438314944836,
+ "loss": 1.1809,
+ "theoretical_loss": 3.40628197642256,
+ "tokens_seen": 2415394816
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021677833500501507,
+ "loss": 1.1934,
+ "theoretical_loss": 3.406274615532739,
+ "tokens_seen": 2415460352
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021676228686058177,
+ "loss": 1.206,
+ "theoretical_loss": 3.406267254898548,
+ "tokens_seen": 2415525888
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021674623871614847,
+ "loss": 1.1547,
+ "theoretical_loss": 3.4062598945199722,
+ "tokens_seen": 2415591424
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021673019057171517,
+ "loss": 1.1022,
+ "theoretical_loss": 3.406252534396995,
+ "tokens_seen": 2415656960
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021671414242728187,
+ "loss": 1.1831,
+ "theoretical_loss": 3.406245174529601,
+ "tokens_seen": 2415722496
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021669809428284857,
+ "loss": 1.1534,
+ "theoretical_loss": 3.4062378149177746,
+ "tokens_seen": 2415788032
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021668204613841527,
+ "loss": 1.1985,
+ "theoretical_loss": 3.4062304555614995,
+ "tokens_seen": 2415853568
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021666599799398197,
+ "loss": 1.3183,
+ "theoretical_loss": 3.40622309646076,
+ "tokens_seen": 2415919104
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021664994984954867,
+ "loss": 1.1317,
+ "theoretical_loss": 3.4062157376155406,
+ "tokens_seen": 2415984640
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021663390170511537,
+ "loss": 1.2001,
+ "theoretical_loss": 3.406208379025825,
+ "tokens_seen": 2416050176
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021661785356068207,
+ "loss": 1.2048,
+ "theoretical_loss": 3.406201020691598,
+ "tokens_seen": 2416115712
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021660180541624877,
+ "loss": 1.1123,
+ "theoretical_loss": 3.4061936626128433,
+ "tokens_seen": 2416181248
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021658575727181547,
+ "loss": 1.2456,
+ "theoretical_loss": 3.406186304789545,
+ "tokens_seen": 2416246784
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021656970912738217,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4061789472216883,
+ "tokens_seen": 2416312320
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021655366098294887,
+ "loss": 1.2157,
+ "theoretical_loss": 3.406171589909256,
+ "tokens_seen": 2416377856
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021653761283851557,
+ "loss": 1.2315,
+ "theoretical_loss": 3.4061642328522335,
+ "tokens_seen": 2416443392
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021652156469408228,
+ "loss": 1.1321,
+ "theoretical_loss": 3.4061568760506042,
+ "tokens_seen": 2416508928
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021650551654964898,
+ "loss": 1.1914,
+ "theoretical_loss": 3.406149519504353,
+ "tokens_seen": 2416574464
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021648946840521568,
+ "loss": 1.2061,
+ "theoretical_loss": 3.406142163213463,
+ "tokens_seen": 2416640000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021647342026078238,
+ "loss": 1.1527,
+ "theoretical_loss": 3.40613480717792,
+ "tokens_seen": 2416705536
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021645737211634908,
+ "loss": 1.3139,
+ "theoretical_loss": 3.406127451397707,
+ "tokens_seen": 2416771072
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021644132397191578,
+ "loss": 1.174,
+ "theoretical_loss": 3.4061200958728084,
+ "tokens_seen": 2416836608
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021642527582748245,
+ "loss": 1.2709,
+ "theoretical_loss": 3.406112740603209,
+ "tokens_seen": 2416902144
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021640922768304915,
+ "loss": 1.2287,
+ "theoretical_loss": 3.406105385588892,
+ "tokens_seen": 2416967680
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021639317953861585,
+ "loss": 1.178,
+ "theoretical_loss": 3.4060980308298427,
+ "tokens_seen": 2417033216
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021637713139418255,
+ "loss": 1.2428,
+ "theoretical_loss": 3.406090676326045,
+ "tokens_seen": 2417098752
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021636108324974925,
+ "loss": 1.2643,
+ "theoretical_loss": 3.406083322077482,
+ "tokens_seen": 2417164288
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021634503510531596,
+ "loss": 1.2671,
+ "theoretical_loss": 3.40607596808414,
+ "tokens_seen": 2417229824
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021632898696088266,
+ "loss": 1.2439,
+ "theoretical_loss": 3.4060686143460015,
+ "tokens_seen": 2417295360
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021631293881644936,
+ "loss": 1.1798,
+ "theoretical_loss": 3.4060612608630514,
+ "tokens_seen": 2417360896
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021629689067201606,
+ "loss": 1.2148,
+ "theoretical_loss": 3.406053907635274,
+ "tokens_seen": 2417426432
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021628084252758276,
+ "loss": 1.1562,
+ "theoretical_loss": 3.4060465546626535,
+ "tokens_seen": 2417491968
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021626479438314946,
+ "loss": 1.1926,
+ "theoretical_loss": 3.4060392019451737,
+ "tokens_seen": 2417557504
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021624874623871616,
+ "loss": 1.2193,
+ "theoretical_loss": 3.4060318494828192,
+ "tokens_seen": 2417623040
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021623269809428286,
+ "loss": 1.2266,
+ "theoretical_loss": 3.406024497275574,
+ "tokens_seen": 2417688576
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021621664994984956,
+ "loss": 1.3415,
+ "theoretical_loss": 3.406017145323423,
+ "tokens_seen": 2417754112
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021620060180541626,
+ "loss": 1.2108,
+ "theoretical_loss": 3.4060097936263496,
+ "tokens_seen": 2417819648
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021618455366098296,
+ "loss": 1.3139,
+ "theoretical_loss": 3.4060024421843385,
+ "tokens_seen": 2417885184
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021616850551654966,
+ "loss": 1.1279,
+ "theoretical_loss": 3.4059950909973735,
+ "tokens_seen": 2417950720
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021615245737211636,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4059877400654397,
+ "tokens_seen": 2418016256
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021613640922768306,
+ "loss": 1.2424,
+ "theoretical_loss": 3.4059803893885205,
+ "tokens_seen": 2418081792
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021612036108324976,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4059730389666005,
+ "tokens_seen": 2418147328
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021610431293881646,
+ "loss": 1.3228,
+ "theoretical_loss": 3.4059656887996637,
+ "tokens_seen": 2418212864
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 808156,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3060191869735718,
+ "objective/train/theoretical_loss": 3.405958338887695,
+ "objective/train/tokens_used": 2438738400,
+ "theoretical_loss": 3.405958338887695,
+ "tokens_seen": 2418278400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021608826479438317,
+ "loss": 1.2718,
+ "theoretical_loss": 3.405958338887695,
+ "tokens_seen": 2418278400
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021607221664994987,
+ "loss": 1.3015,
+ "theoretical_loss": 3.4059509892306776,
+ "tokens_seen": 2418343936
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021605616850551657,
+ "loss": 1.2481,
+ "theoretical_loss": 3.4059436398285965,
+ "tokens_seen": 2418409472
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021604012036108324,
+ "loss": 1.2096,
+ "theoretical_loss": 3.405936290681436,
+ "tokens_seen": 2418475008
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021602407221664994,
+ "loss": 1.1896,
+ "theoretical_loss": 3.40592894178918,
+ "tokens_seen": 2418540544
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021600802407221664,
+ "loss": 1.1642,
+ "theoretical_loss": 3.4059215931518128,
+ "tokens_seen": 2418606080
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021599197592778334,
+ "loss": 1.2329,
+ "theoretical_loss": 3.4059142447693187,
+ "tokens_seen": 2418671616
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021597592778335004,
+ "loss": 1.2242,
+ "theoretical_loss": 3.405906896641682,
+ "tokens_seen": 2418737152
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021595987963891674,
+ "loss": 1.3611,
+ "theoretical_loss": 3.405899548768887,
+ "tokens_seen": 2418802688
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021594383149448344,
+ "loss": 1.1732,
+ "theoretical_loss": 3.405892201150918,
+ "tokens_seen": 2418868224
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021592778335005014,
+ "loss": 1.1405,
+ "theoretical_loss": 3.405884853787759,
+ "tokens_seen": 2418933760
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021591173520561685,
+ "loss": 1.1953,
+ "theoretical_loss": 3.4058775066793943,
+ "tokens_seen": 2418999296
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021589568706118355,
+ "loss": 1.2064,
+ "theoretical_loss": 3.4058701598258083,
+ "tokens_seen": 2419064832
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021587963891675025,
+ "loss": 1.2331,
+ "theoretical_loss": 3.4058628132269857,
+ "tokens_seen": 2419130368
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021586359077231695,
+ "loss": 1.2219,
+ "theoretical_loss": 3.40585546688291,
+ "tokens_seen": 2419195904
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021584754262788365,
+ "loss": 1.1949,
+ "theoretical_loss": 3.4058481207935656,
+ "tokens_seen": 2419261440
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021583149448345035,
+ "loss": 1.2762,
+ "theoretical_loss": 3.405840774958937,
+ "tokens_seen": 2419326976
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021581544633901705,
+ "loss": 1.3346,
+ "theoretical_loss": 3.4058334293790082,
+ "tokens_seen": 2419392512
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021579939819458375,
+ "loss": 1.2321,
+ "theoretical_loss": 3.405826084053764,
+ "tokens_seen": 2419458048
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021578335005015045,
+ "loss": 1.1883,
+ "theoretical_loss": 3.4058187389831884,
+ "tokens_seen": 2419523584
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021576730190571715,
+ "loss": 1.2577,
+ "theoretical_loss": 3.4058113941672654,
+ "tokens_seen": 2419589120
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021575125376128385,
+ "loss": 1.1782,
+ "theoretical_loss": 3.4058040496059796,
+ "tokens_seen": 2419654656
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021573520561685055,
+ "loss": 1.0571,
+ "theoretical_loss": 3.4057967052993154,
+ "tokens_seen": 2419720192
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021571915747241725,
+ "loss": 1.1607,
+ "theoretical_loss": 3.4057893612472565,
+ "tokens_seen": 2419785728
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021570310932798395,
+ "loss": 1.3551,
+ "theoretical_loss": 3.4057820174497877,
+ "tokens_seen": 2419851264
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021568706118355065,
+ "loss": 1.2453,
+ "theoretical_loss": 3.405774673906893,
+ "tokens_seen": 2419916800
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021567101303911735,
+ "loss": 1.269,
+ "theoretical_loss": 3.4057673306185565,
+ "tokens_seen": 2419982336
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021565496489468403,
+ "loss": 1.2418,
+ "theoretical_loss": 3.405759987584763,
+ "tokens_seen": 2420047872
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021563891675025073,
+ "loss": 1.2023,
+ "theoretical_loss": 3.4057526448054967,
+ "tokens_seen": 2420113408
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021562286860581743,
+ "loss": 1.238,
+ "theoretical_loss": 3.405745302280742,
+ "tokens_seen": 2420178944
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021560682046138418,
+ "loss": 1.2424,
+ "theoretical_loss": 3.4057379600104825,
+ "tokens_seen": 2420244480
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021559077231695089,
+ "loss": 1.3595,
+ "theoretical_loss": 3.405730617994703,
+ "tokens_seen": 2420310016
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021557472417251759,
+ "loss": 1.1866,
+ "theoretical_loss": 3.4057232762333878,
+ "tokens_seen": 2420375552
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002155586760280843,
+ "loss": 1.1627,
+ "theoretical_loss": 3.405715934726521,
+ "tokens_seen": 2420441088
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.000215542627883651,
+ "loss": 1.2064,
+ "theoretical_loss": 3.4057085934740865,
+ "tokens_seen": 2420506624
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002155265797392177,
+ "loss": 1.2131,
+ "theoretical_loss": 3.40570125247607,
+ "tokens_seen": 2420572160
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002155105315947844,
+ "loss": 1.1337,
+ "theoretical_loss": 3.4056939117324543,
+ "tokens_seen": 2420637696
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154944834503511,
+ "loss": 1.1833,
+ "theoretical_loss": 3.4056865712432245,
+ "tokens_seen": 2420703232
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154784353059178,
+ "loss": 1.1638,
+ "theoretical_loss": 3.4056792310083646,
+ "tokens_seen": 2420768768
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154623871614845,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4056718910278585,
+ "tokens_seen": 2420834304
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154463390170512,
+ "loss": 1.2134,
+ "theoretical_loss": 3.405664551301691,
+ "tokens_seen": 2420899840
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154302908726179,
+ "loss": 1.1343,
+ "theoretical_loss": 3.405657211829847,
+ "tokens_seen": 2420965376
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002154142427281846,
+ "loss": 1.1838,
+ "theoretical_loss": 3.40564987261231,
+ "tokens_seen": 2421030912
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021539819458375127,
+ "loss": 1.2957,
+ "theoretical_loss": 3.4056425336490648,
+ "tokens_seen": 2421096448
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021538214643931797,
+ "loss": 1.2216,
+ "theoretical_loss": 3.4056351949400945,
+ "tokens_seen": 2421161984
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021536609829488467,
+ "loss": 1.2648,
+ "theoretical_loss": 3.405627856485385,
+ "tokens_seen": 2421227520
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021535005015045137,
+ "loss": 1.2796,
+ "theoretical_loss": 3.4056205182849197,
+ "tokens_seen": 2421293056
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021533400200601807,
+ "loss": 1.2723,
+ "theoretical_loss": 3.405613180338683,
+ "tokens_seen": 2421358592
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021531795386158477,
+ "loss": 1.2495,
+ "theoretical_loss": 3.4056058426466596,
+ "tokens_seen": 2421424128
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021530190571715147,
+ "loss": 1.2431,
+ "theoretical_loss": 3.405598505208833,
+ "tokens_seen": 2421489664
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 809199,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3278754949569702,
+ "objective/train/theoretical_loss": 3.4055911680251887,
+ "objective/train/tokens_used": 2442015200,
+ "theoretical_loss": 3.4055911680251887,
+ "tokens_seen": 2421555200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021528585757271817,
+ "loss": 1.2899,
+ "theoretical_loss": 3.4055911680251887,
+ "tokens_seen": 2421555200
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021526980942828487,
+ "loss": 1.1465,
+ "theoretical_loss": 3.4055838310957096,
+ "tokens_seen": 2421620736
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021525376128385157,
+ "loss": 1.2543,
+ "theoretical_loss": 3.4055764944203815,
+ "tokens_seen": 2421686272
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021523771313941827,
+ "loss": 1.1722,
+ "theoretical_loss": 3.405569157999188,
+ "tokens_seen": 2421751808
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021522166499498497,
+ "loss": 1.2255,
+ "theoretical_loss": 3.4055618218321126,
+ "tokens_seen": 2421817344
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021520561685055167,
+ "loss": 1.3018,
+ "theoretical_loss": 3.405554485919141,
+ "tokens_seen": 2421882880
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021518956870611837,
+ "loss": 1.264,
+ "theoretical_loss": 3.405547150260257,
+ "tokens_seen": 2421948416
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021517352056168507,
+ "loss": 1.1722,
+ "theoretical_loss": 3.405539814855445,
+ "tokens_seen": 2422013952
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021515747241725178,
+ "loss": 1.2487,
+ "theoretical_loss": 3.405532479704689,
+ "tokens_seen": 2422079488
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021514142427281848,
+ "loss": 1.2918,
+ "theoretical_loss": 3.4055251448079735,
+ "tokens_seen": 2422145024
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021512537612838518,
+ "loss": 1.1913,
+ "theoretical_loss": 3.405517810165283,
+ "tokens_seen": 2422210560
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021510932798395188,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4055104757766017,
+ "tokens_seen": 2422276096
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021509327983951858,
+ "loss": 1.305,
+ "theoretical_loss": 3.4055031416419137,
+ "tokens_seen": 2422341632
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021507723169508528,
+ "loss": 1.1993,
+ "theoretical_loss": 3.405495807761204,
+ "tokens_seen": 2422407168
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021506118355065198,
+ "loss": 1.1444,
+ "theoretical_loss": 3.405488474134456,
+ "tokens_seen": 2422472704
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021504513540621868,
+ "loss": 1.2683,
+ "theoretical_loss": 3.4054811407616548,
+ "tokens_seen": 2422538240
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021502908726178538,
+ "loss": 1.2111,
+ "theoretical_loss": 3.405473807642785,
+ "tokens_seen": 2422603776
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021501303911735205,
+ "loss": 1.1786,
+ "theoretical_loss": 3.4054664747778296,
+ "tokens_seen": 2422669312
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021499699097291875,
+ "loss": 1.268,
+ "theoretical_loss": 3.405459142166774,
+ "tokens_seen": 2422734848
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021498094282848546,
+ "loss": 1.2715,
+ "theoretical_loss": 3.4054518098096023,
+ "tokens_seen": 2422800384
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021496489468405216,
+ "loss": 1.2586,
+ "theoretical_loss": 3.4054444777062987,
+ "tokens_seen": 2422865920
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021494884653961886,
+ "loss": 1.3042,
+ "theoretical_loss": 3.405437145856848,
+ "tokens_seen": 2422931456
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021493279839518556,
+ "loss": 1.1765,
+ "theoretical_loss": 3.405429814261234,
+ "tokens_seen": 2422996992
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021491675025075226,
+ "loss": 1.1783,
+ "theoretical_loss": 3.4054224829194415,
+ "tokens_seen": 2423062528
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021490070210631896,
+ "loss": 1.2545,
+ "theoretical_loss": 3.405415151831454,
+ "tokens_seen": 2423128064
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021488465396188566,
+ "loss": 1.1972,
+ "theoretical_loss": 3.405407820997257,
+ "tokens_seen": 2423193600
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021486860581745236,
+ "loss": 1.299,
+ "theoretical_loss": 3.4054004904168345,
+ "tokens_seen": 2423259136
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021485255767301906,
+ "loss": 1.1559,
+ "theoretical_loss": 3.40539316009017,
+ "tokens_seen": 2423324672
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021483650952858576,
+ "loss": 1.203,
+ "theoretical_loss": 3.405385830017249,
+ "tokens_seen": 2423390208
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021482046138415246,
+ "loss": 1.2275,
+ "theoretical_loss": 3.405378500198055,
+ "tokens_seen": 2423455744
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021480441323971916,
+ "loss": 1.1968,
+ "theoretical_loss": 3.4053711706325736,
+ "tokens_seen": 2423521280
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021478836509528586,
+ "loss": 1.1728,
+ "theoretical_loss": 3.4053638413207876,
+ "tokens_seen": 2423586816
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021477231695085256,
+ "loss": 1.3198,
+ "theoretical_loss": 3.4053565122626823,
+ "tokens_seen": 2423652352
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021475626880641926,
+ "loss": 1.2855,
+ "theoretical_loss": 3.4053491834582417,
+ "tokens_seen": 2423717888
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021474022066198596,
+ "loss": 1.2311,
+ "theoretical_loss": 3.40534185490745,
+ "tokens_seen": 2423783424
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021472417251755267,
+ "loss": 1.2111,
+ "theoretical_loss": 3.405334526610292,
+ "tokens_seen": 2423848960
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021470812437311937,
+ "loss": 1.1828,
+ "theoretical_loss": 3.405327198566752,
+ "tokens_seen": 2423914496
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021469207622868607,
+ "loss": 1.2072,
+ "theoretical_loss": 3.4053198707768146,
+ "tokens_seen": 2423980032
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021467602808425277,
+ "loss": 1.1728,
+ "theoretical_loss": 3.405312543240463,
+ "tokens_seen": 2424045568
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021465997993981947,
+ "loss": 1.2729,
+ "theoretical_loss": 3.405305215957683,
+ "tokens_seen": 2424111104
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021464393179538617,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4052978889284584,
+ "tokens_seen": 2424176640
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021462788365095284,
+ "loss": 1.2795,
+ "theoretical_loss": 3.4052905621527736,
+ "tokens_seen": 2424242176
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021461183550651954,
+ "loss": 1.2214,
+ "theoretical_loss": 3.4052832356306126,
+ "tokens_seen": 2424307712
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021459578736208624,
+ "loss": 1.2644,
+ "theoretical_loss": 3.40527590936196,
+ "tokens_seen": 2424373248
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021457973921765294,
+ "loss": 1.1287,
+ "theoretical_loss": 3.4052685833468006,
+ "tokens_seen": 2424438784
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021456369107321964,
+ "loss": 1.1832,
+ "theoretical_loss": 3.4052612575851184,
+ "tokens_seen": 2424504320
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021454764292878635,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4052539320768975,
+ "tokens_seen": 2424569856
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021453159478435305,
+ "loss": 1.2711,
+ "theoretical_loss": 3.405246606822123,
+ "tokens_seen": 2424635392
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021451554663991975,
+ "loss": 1.1784,
+ "theoretical_loss": 3.405239281820779,
+ "tokens_seen": 2424700928
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021449949849548645,
+ "loss": 1.203,
+ "theoretical_loss": 3.405231957072849,
+ "tokens_seen": 2424766464
+ },
+ {
+ "epoch": 0.73,
+ "objective/train/docs_used": 810525,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1189380884170532,
+ "objective/train/theoretical_loss": 3.405224632578319,
+ "objective/train/tokens_used": 2445292000,
+ "theoretical_loss": 3.405224632578319,
+ "tokens_seen": 2424832000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021448345035105315,
+ "loss": 1.2053,
+ "theoretical_loss": 3.405224632578319,
+ "tokens_seen": 2424832000
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021446740220661985,
+ "loss": 1.2598,
+ "theoretical_loss": 3.405217308337172,
+ "tokens_seen": 2424897536
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021445135406218655,
+ "loss": 1.2035,
+ "theoretical_loss": 3.4052099843493933,
+ "tokens_seen": 2424963072
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021443530591775325,
+ "loss": 1.2279,
+ "theoretical_loss": 3.405202660614967,
+ "tokens_seen": 2425028608
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021441925777331995,
+ "loss": 1.2426,
+ "theoretical_loss": 3.405195337133877,
+ "tokens_seen": 2425094144
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.0002144032096288867,
+ "loss": 1.1973,
+ "theoretical_loss": 3.405188013906108,
+ "tokens_seen": 2425159680
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021438716148445338,
+ "loss": 1.2866,
+ "theoretical_loss": 3.405180690931645,
+ "tokens_seen": 2425225216
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021437111334002008,
+ "loss": 1.2717,
+ "theoretical_loss": 3.4051733682104715,
+ "tokens_seen": 2425290752
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021435506519558678,
+ "loss": 1.2488,
+ "theoretical_loss": 3.405166045742572,
+ "tokens_seen": 2425356288
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021433901705115348,
+ "loss": 1.1991,
+ "theoretical_loss": 3.4051587235279315,
+ "tokens_seen": 2425421824
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 0.00021432296890672018,
+ "loss": 1.2548,
+ "theoretical_loss": 3.4051514015665343,
+ "tokens_seen": 2425487360
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021430692076228688,
+ "loss": 1.2152,
+ "theoretical_loss": 3.4051440798583643,
+ "tokens_seen": 2425552896
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021429087261785358,
+ "loss": 1.2238,
+ "theoretical_loss": 3.405136758403406,
+ "tokens_seen": 2425618432
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021427482447342028,
+ "loss": 1.2284,
+ "theoretical_loss": 3.4051294372016443,
+ "tokens_seen": 2425683968
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021425877632898698,
+ "loss": 1.1835,
+ "theoretical_loss": 3.405122116253063,
+ "tokens_seen": 2425749504
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021424272818455368,
+ "loss": 1.2238,
+ "theoretical_loss": 3.405114795557647,
+ "tokens_seen": 2425815040
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021422668004012038,
+ "loss": 1.2675,
+ "theoretical_loss": 3.4051074751153805,
+ "tokens_seen": 2425880576
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021421063189568709,
+ "loss": 1.2621,
+ "theoretical_loss": 3.405100154926248,
+ "tokens_seen": 2425946112
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021419458375125379,
+ "loss": 1.2111,
+ "theoretical_loss": 3.405092834990233,
+ "tokens_seen": 2426011648
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002141785356068205,
+ "loss": 1.1756,
+ "theoretical_loss": 3.405085515307322,
+ "tokens_seen": 2426077184
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002141624874623872,
+ "loss": 1.3343,
+ "theoretical_loss": 3.4050781958774974,
+ "tokens_seen": 2426142720
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002141464393179539,
+ "loss": 1.2369,
+ "theoretical_loss": 3.4050708767007443,
+ "tokens_seen": 2426208256
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002141303911735206,
+ "loss": 1.3483,
+ "theoretical_loss": 3.405063557777047,
+ "tokens_seen": 2426273792
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002141143430290873,
+ "loss": 1.2461,
+ "theoretical_loss": 3.4050562391063903,
+ "tokens_seen": 2426339328
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000214098294884654,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4050489206887584,
+ "tokens_seen": 2426404864
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002140822467402207,
+ "loss": 1.2222,
+ "theoretical_loss": 3.4050416025241357,
+ "tokens_seen": 2426470400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002140661985957874,
+ "loss": 1.1978,
+ "theoretical_loss": 3.4050342846125066,
+ "tokens_seen": 2426535936
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002140501504513541,
+ "loss": 1.2462,
+ "theoretical_loss": 3.4050269669538555,
+ "tokens_seen": 2426601472
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002140341023069208,
+ "loss": 1.2623,
+ "theoretical_loss": 3.405019649548167,
+ "tokens_seen": 2426667008
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002140180541624875,
+ "loss": 1.3263,
+ "theoretical_loss": 3.405012332395425,
+ "tokens_seen": 2426732544
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021400200601805417,
+ "loss": 1.3777,
+ "theoretical_loss": 3.4050050154956146,
+ "tokens_seen": 2426798080
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021398595787362087,
+ "loss": 1.168,
+ "theoretical_loss": 3.40499769884872,
+ "tokens_seen": 2426863616
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021396990972918757,
+ "loss": 1.2123,
+ "theoretical_loss": 3.404990382454726,
+ "tokens_seen": 2426929152
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021395386158475427,
+ "loss": 1.2737,
+ "theoretical_loss": 3.4049830663136156,
+ "tokens_seen": 2426994688
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021393781344032097,
+ "loss": 1.3026,
+ "theoretical_loss": 3.404975750425375,
+ "tokens_seen": 2427060224
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021392176529588767,
+ "loss": 1.2274,
+ "theoretical_loss": 3.4049684347899873,
+ "tokens_seen": 2427125760
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021390571715145437,
+ "loss": 1.2076,
+ "theoretical_loss": 3.404961119407438,
+ "tokens_seen": 2427191296
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021388966900702107,
+ "loss": 1.2909,
+ "theoretical_loss": 3.404953804277711,
+ "tokens_seen": 2427256832
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021387362086258777,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4049464894007904,
+ "tokens_seen": 2427322368
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021385757271815447,
+ "loss": 1.3397,
+ "theoretical_loss": 3.4049391747766613,
+ "tokens_seen": 2427387904
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021384152457372117,
+ "loss": 1.2544,
+ "theoretical_loss": 3.404931860405308,
+ "tokens_seen": 2427453440
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021382547642928787,
+ "loss": 1.3042,
+ "theoretical_loss": 3.4049245462867144,
+ "tokens_seen": 2427518976
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021380942828485457,
+ "loss": 1.145,
+ "theoretical_loss": 3.404917232420865,
+ "tokens_seen": 2427584512
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021379338014042127,
+ "loss": 1.1955,
+ "theoretical_loss": 3.4049099188077454,
+ "tokens_seen": 2427650048
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021377733199598798,
+ "loss": 1.2189,
+ "theoretical_loss": 3.404902605447339,
+ "tokens_seen": 2427715584
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021376128385155468,
+ "loss": 1.2325,
+ "theoretical_loss": 3.40489529233963,
+ "tokens_seen": 2427781120
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021374523570712138,
+ "loss": 1.1721,
+ "theoretical_loss": 3.4048879794846036,
+ "tokens_seen": 2427846656
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021372918756268808,
+ "loss": 1.2649,
+ "theoretical_loss": 3.404880666882244,
+ "tokens_seen": 2427912192
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021371313941825478,
+ "loss": 1.1599,
+ "theoretical_loss": 3.4048733545325356,
+ "tokens_seen": 2427977728
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021369709127382148,
+ "loss": 1.1854,
+ "theoretical_loss": 3.404866042435463,
+ "tokens_seen": 2428043264
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 811717,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1337710618972778,
+ "objective/train/theoretical_loss": 3.4048587305910103,
+ "objective/train/tokens_used": 2448568800,
+ "theoretical_loss": 3.4048587305910103,
+ "tokens_seen": 2428108800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021368104312938818,
+ "loss": 1.1389,
+ "theoretical_loss": 3.4048587305910103,
+ "tokens_seen": 2428108800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021366499498495488,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4048514189991623,
+ "tokens_seen": 2428174336
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021364894684052158,
+ "loss": 1.2767,
+ "theoretical_loss": 3.404844107659903,
+ "tokens_seen": 2428239872
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021363289869608828,
+ "loss": 1.1604,
+ "theoretical_loss": 3.4048367965732176,
+ "tokens_seen": 2428305408
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021361685055165495,
+ "loss": 1.2239,
+ "theoretical_loss": 3.40482948573909,
+ "tokens_seen": 2428370944
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021360080240722166,
+ "loss": 1.1767,
+ "theoretical_loss": 3.4048221751575047,
+ "tokens_seen": 2428436480
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021358475426278836,
+ "loss": 1.1925,
+ "theoretical_loss": 3.404814864828446,
+ "tokens_seen": 2428502016
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021356870611835506,
+ "loss": 1.124,
+ "theoretical_loss": 3.4048075547518994,
+ "tokens_seen": 2428567552
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021355265797392176,
+ "loss": 1.2132,
+ "theoretical_loss": 3.404800244927848,
+ "tokens_seen": 2428633088
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021353660982948846,
+ "loss": 1.2134,
+ "theoretical_loss": 3.404792935356277,
+ "tokens_seen": 2428698624
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021352056168505516,
+ "loss": 1.1678,
+ "theoretical_loss": 3.404785626037171,
+ "tokens_seen": 2428764160
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021350451354062186,
+ "loss": 1.3398,
+ "theoretical_loss": 3.4047783169705137,
+ "tokens_seen": 2428829696
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021348846539618856,
+ "loss": 1.2887,
+ "theoretical_loss": 3.40477100815629,
+ "tokens_seen": 2428895232
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021347241725175526,
+ "loss": 1.2398,
+ "theoretical_loss": 3.404763699594485,
+ "tokens_seen": 2428960768
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021345636910732196,
+ "loss": 1.1675,
+ "theoretical_loss": 3.404756391285082,
+ "tokens_seen": 2429026304
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021344032096288866,
+ "loss": 1.2322,
+ "theoretical_loss": 3.4047490832280665,
+ "tokens_seen": 2429091840
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021342427281845536,
+ "loss": 1.2196,
+ "theoretical_loss": 3.4047417754234224,
+ "tokens_seen": 2429157376
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021340822467402206,
+ "loss": 1.1684,
+ "theoretical_loss": 3.404734467871134,
+ "tokens_seen": 2429222912
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021339217652958876,
+ "loss": 1.2434,
+ "theoretical_loss": 3.404727160571187,
+ "tokens_seen": 2429288448
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021337612838515546,
+ "loss": 1.2655,
+ "theoretical_loss": 3.4047198535235643,
+ "tokens_seen": 2429353984
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021336008024072216,
+ "loss": 1.1372,
+ "theoretical_loss": 3.404712546728251,
+ "tokens_seen": 2429419520
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021334403209628887,
+ "loss": 1.2468,
+ "theoretical_loss": 3.4047052401852316,
+ "tokens_seen": 2429485056
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021332798395185557,
+ "loss": 1.2789,
+ "theoretical_loss": 3.404697933894491,
+ "tokens_seen": 2429550592
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021331193580742227,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4046906278560134,
+ "tokens_seen": 2429616128
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021329588766298897,
+ "loss": 1.2194,
+ "theoretical_loss": 3.4046833220697827,
+ "tokens_seen": 2429681664
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021327983951855567,
+ "loss": 1.2251,
+ "theoretical_loss": 3.4046760165357846,
+ "tokens_seen": 2429747200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021326379137412237,
+ "loss": 1.1926,
+ "theoretical_loss": 3.404668711254002,
+ "tokens_seen": 2429812736
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021324774322968907,
+ "loss": 1.3904,
+ "theoretical_loss": 3.4046614062244207,
+ "tokens_seen": 2429878272
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021323169508525574,
+ "loss": 1.1188,
+ "theoretical_loss": 3.404654101447025,
+ "tokens_seen": 2429943808
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021321564694082244,
+ "loss": 1.1759,
+ "theoretical_loss": 3.404646796921799,
+ "tokens_seen": 2430009344
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131995987963892,
+ "loss": 1.229,
+ "theoretical_loss": 3.404639492648727,
+ "tokens_seen": 2430074880
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131835506519559,
+ "loss": 1.1467,
+ "theoretical_loss": 3.404632188627794,
+ "tokens_seen": 2430140416
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131675025075226,
+ "loss": 1.1525,
+ "theoretical_loss": 3.4046248848589844,
+ "tokens_seen": 2430205952
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131514543630893,
+ "loss": 1.2374,
+ "theoretical_loss": 3.4046175813422828,
+ "tokens_seen": 2430271488
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000213135406218656,
+ "loss": 1.2248,
+ "theoretical_loss": 3.4046102780776732,
+ "tokens_seen": 2430337024
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131193580742227,
+ "loss": 1.2395,
+ "theoretical_loss": 3.4046029750651408,
+ "tokens_seen": 2430402560
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002131033099297894,
+ "loss": 1.3251,
+ "theoretical_loss": 3.4045956723046693,
+ "tokens_seen": 2430468096
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130872617853561,
+ "loss": 1.247,
+ "theoretical_loss": 3.404588369796244,
+ "tokens_seen": 2430533632
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130712136409228,
+ "loss": 1.2932,
+ "theoretical_loss": 3.404581067539849,
+ "tokens_seen": 2430599168
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130551654964895,
+ "loss": 1.2816,
+ "theoretical_loss": 3.404573765535469,
+ "tokens_seen": 2430664704
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130391173520562,
+ "loss": 1.2838,
+ "theoretical_loss": 3.404566463783088,
+ "tokens_seen": 2430730240
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130230692076229,
+ "loss": 1.3536,
+ "theoretical_loss": 3.404559162282691,
+ "tokens_seen": 2430795776
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002130070210631896,
+ "loss": 1.276,
+ "theoretical_loss": 3.404551861034262,
+ "tokens_seen": 2430861312
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021299097291875628,
+ "loss": 1.2649,
+ "theoretical_loss": 3.404544560037787,
+ "tokens_seen": 2430926848
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021297492477432298,
+ "loss": 1.2465,
+ "theoretical_loss": 3.4045372592932486,
+ "tokens_seen": 2430992384
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021295887662988968,
+ "loss": 1.2106,
+ "theoretical_loss": 3.404529958800632,
+ "tokens_seen": 2431057920
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021294282848545638,
+ "loss": 1.1642,
+ "theoretical_loss": 3.404522658559922,
+ "tokens_seen": 2431123456
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021292678034102308,
+ "loss": 1.2176,
+ "theoretical_loss": 3.4045153585711034,
+ "tokens_seen": 2431188992
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021291073219658978,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4045080588341596,
+ "tokens_seen": 2431254528
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021289468405215648,
+ "loss": 1.2658,
+ "theoretical_loss": 3.404500759349076,
+ "tokens_seen": 2431320064
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 812751,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2283116579055786,
+ "objective/train/theoretical_loss": 3.4044934601158374,
+ "objective/train/tokens_used": 2451845600,
+ "theoretical_loss": 3.4044934601158374,
+ "tokens_seen": 2431385600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021287863590772318,
+ "loss": 1.2107,
+ "theoretical_loss": 3.4044934601158374,
+ "tokens_seen": 2431385600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021286258776328988,
+ "loss": 1.1902,
+ "theoretical_loss": 3.404486161134427,
+ "tokens_seen": 2431451136
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021284653961885659,
+ "loss": 1.2542,
+ "theoretical_loss": 3.404478862404831,
+ "tokens_seen": 2431516672
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021283049147442329,
+ "loss": 1.1443,
+ "theoretical_loss": 3.4044715639270327,
+ "tokens_seen": 2431582208
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021281444332999,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4044642657010167,
+ "tokens_seen": 2431647744
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127983951855567,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4044569677267686,
+ "tokens_seen": 2431713280
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127823470411234,
+ "loss": 1.1241,
+ "theoretical_loss": 3.4044496700042717,
+ "tokens_seen": 2431778816
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127662988966901,
+ "loss": 1.3111,
+ "theoretical_loss": 3.404442372533511,
+ "tokens_seen": 2431844352
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127502507522568,
+ "loss": 1.2639,
+ "theoretical_loss": 3.404435075314471,
+ "tokens_seen": 2431909888
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127342026078235,
+ "loss": 1.2626,
+ "theoretical_loss": 3.404427778347136,
+ "tokens_seen": 2431975424
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127181544633902,
+ "loss": 1.3172,
+ "theoretical_loss": 3.4044204816314916,
+ "tokens_seen": 2432040960
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002127021063189569,
+ "loss": 1.2782,
+ "theoretical_loss": 3.4044131851675212,
+ "tokens_seen": 2432106496
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002126860581745236,
+ "loss": 1.2304,
+ "theoretical_loss": 3.4044058889552096,
+ "tokens_seen": 2432172032
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002126700100300903,
+ "loss": 1.2716,
+ "theoretical_loss": 3.4043985929945415,
+ "tokens_seen": 2432237568
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000212653961885657,
+ "loss": 1.2557,
+ "theoretical_loss": 3.404391297285501,
+ "tokens_seen": 2432303104
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002126379137412237,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4043840018280735,
+ "tokens_seen": 2432368640
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002126218655967904,
+ "loss": 1.3045,
+ "theoretical_loss": 3.4043767066222426,
+ "tokens_seen": 2432434176
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021260581745235707,
+ "loss": 1.2566,
+ "theoretical_loss": 3.404369411667994,
+ "tokens_seen": 2432499712
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021258976930792377,
+ "loss": 1.2935,
+ "theoretical_loss": 3.4043621169653107,
+ "tokens_seen": 2432565248
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021257372116349047,
+ "loss": 1.2836,
+ "theoretical_loss": 3.4043548225141786,
+ "tokens_seen": 2432630784
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021255767301905717,
+ "loss": 1.2883,
+ "theoretical_loss": 3.404347528314582,
+ "tokens_seen": 2432696320
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021254162487462387,
+ "loss": 1.1908,
+ "theoretical_loss": 3.4043402343665043,
+ "tokens_seen": 2432761856
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021252557673019057,
+ "loss": 1.2137,
+ "theoretical_loss": 3.4043329406699314,
+ "tokens_seen": 2432827392
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021250952858575727,
+ "loss": 1.2071,
+ "theoretical_loss": 3.4043256472248475,
+ "tokens_seen": 2432892928
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021249348044132397,
+ "loss": 1.1696,
+ "theoretical_loss": 3.404318354031237,
+ "tokens_seen": 2432958464
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021247743229689067,
+ "loss": 1.2651,
+ "theoretical_loss": 3.4043110610890848,
+ "tokens_seen": 2433024000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021246138415245737,
+ "loss": 1.2124,
+ "theoretical_loss": 3.404303768398375,
+ "tokens_seen": 2433089536
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021244533600802407,
+ "loss": 1.1757,
+ "theoretical_loss": 3.4042964759590917,
+ "tokens_seen": 2433155072
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021242928786359077,
+ "loss": 1.2648,
+ "theoretical_loss": 3.404289183771221,
+ "tokens_seen": 2433220608
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021241323971915748,
+ "loss": 1.2172,
+ "theoretical_loss": 3.4042818918347457,
+ "tokens_seen": 2433286144
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021239719157472418,
+ "loss": 1.2475,
+ "theoretical_loss": 3.4042746001496518,
+ "tokens_seen": 2433351680
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021238114343029088,
+ "loss": 1.2223,
+ "theoretical_loss": 3.404267308715923,
+ "tokens_seen": 2433417216
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021236509528585758,
+ "loss": 1.1876,
+ "theoretical_loss": 3.4042600175335442,
+ "tokens_seen": 2433482752
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021234904714142428,
+ "loss": 1.239,
+ "theoretical_loss": 3.4042527266024996,
+ "tokens_seen": 2433548288
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021233299899699098,
+ "loss": 1.2575,
+ "theoretical_loss": 3.4042454359227743,
+ "tokens_seen": 2433613824
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021231695085255768,
+ "loss": 1.2248,
+ "theoretical_loss": 3.404238145494353,
+ "tokens_seen": 2433679360
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021230090270812438,
+ "loss": 1.1558,
+ "theoretical_loss": 3.4042308553172194,
+ "tokens_seen": 2433744896
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021228485456369108,
+ "loss": 1.2034,
+ "theoretical_loss": 3.4042235653913586,
+ "tokens_seen": 2433810432
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021226880641925778,
+ "loss": 1.1756,
+ "theoretical_loss": 3.4042162757167556,
+ "tokens_seen": 2433875968
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021225275827482448,
+ "loss": 1.1321,
+ "theoretical_loss": 3.404208986293394,
+ "tokens_seen": 2433941504
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021223671013039118,
+ "loss": 1.2973,
+ "theoretical_loss": 3.404201697121259,
+ "tokens_seen": 2434007040
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021222066198595786,
+ "loss": 1.309,
+ "theoretical_loss": 3.4041944082003353,
+ "tokens_seen": 2434072576
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021220461384152456,
+ "loss": 1.2782,
+ "theoretical_loss": 3.404187119530607,
+ "tokens_seen": 2434138112
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021218856569709126,
+ "loss": 1.2751,
+ "theoretical_loss": 3.4041798311120592,
+ "tokens_seen": 2434203648
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021217251755265796,
+ "loss": 1.2166,
+ "theoretical_loss": 3.4041725429446763,
+ "tokens_seen": 2434269184
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021215646940822466,
+ "loss": 1.208,
+ "theoretical_loss": 3.4041652550284427,
+ "tokens_seen": 2434334720
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021214042126379136,
+ "loss": 1.1791,
+ "theoretical_loss": 3.404157967363343,
+ "tokens_seen": 2434400256
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021212437311935806,
+ "loss": 1.2623,
+ "theoretical_loss": 3.404150679949362,
+ "tokens_seen": 2434465792
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021210832497492476,
+ "loss": 1.2136,
+ "theoretical_loss": 3.404143392786484,
+ "tokens_seen": 2434531328
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021209227683049146,
+ "loss": 1.1994,
+ "theoretical_loss": 3.404136105874694,
+ "tokens_seen": 2434596864
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 814053,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2832529544830322,
+ "objective/train/theoretical_loss": 3.4041288192139763,
+ "objective/train/tokens_used": 2455122400,
+ "theoretical_loss": 3.4041288192139763,
+ "tokens_seen": 2434662400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021207622868605816,
+ "loss": 1.2401,
+ "theoretical_loss": 3.4041288192139763,
+ "tokens_seen": 2434662400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021206018054162486,
+ "loss": 1.2534,
+ "theoretical_loss": 3.4041215328043153,
+ "tokens_seen": 2434727936
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021204413239719156,
+ "loss": 1.2171,
+ "theoretical_loss": 3.4041142466456957,
+ "tokens_seen": 2434793472
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021202808425275826,
+ "loss": 1.2908,
+ "theoretical_loss": 3.4041069607381025,
+ "tokens_seen": 2434859008
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021201203610832496,
+ "loss": 1.1228,
+ "theoretical_loss": 3.40409967508152,
+ "tokens_seen": 2434924544
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021199598796389172,
+ "loss": 1.2748,
+ "theoretical_loss": 3.4040923896759328,
+ "tokens_seen": 2434990080
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021197993981945842,
+ "loss": 1.1732,
+ "theoretical_loss": 3.4040851045213256,
+ "tokens_seen": 2435055616
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002119638916750251,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4040778196176826,
+ "tokens_seen": 2435121152
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002119478435305918,
+ "loss": 1.2046,
+ "theoretical_loss": 3.404070534964989,
+ "tokens_seen": 2435186688
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002119317953861585,
+ "loss": 1.1096,
+ "theoretical_loss": 3.404063250563229,
+ "tokens_seen": 2435252224
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002119157472417252,
+ "loss": 1.1968,
+ "theoretical_loss": 3.4040559664123875,
+ "tokens_seen": 2435317760
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118996990972919,
+ "loss": 1.1899,
+ "theoretical_loss": 3.4040486825124487,
+ "tokens_seen": 2435383296
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118836509528586,
+ "loss": 1.2778,
+ "theoretical_loss": 3.4040413988633973,
+ "tokens_seen": 2435448832
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118676028084253,
+ "loss": 1.2325,
+ "theoretical_loss": 3.404034115465218,
+ "tokens_seen": 2435514368
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000211851554663992,
+ "loss": 1.2795,
+ "theoretical_loss": 3.404026832317896,
+ "tokens_seen": 2435579904
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118355065195587,
+ "loss": 1.3063,
+ "theoretical_loss": 3.404019549421415,
+ "tokens_seen": 2435645440
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118194583751254,
+ "loss": 1.3204,
+ "theoretical_loss": 3.40401226677576,
+ "tokens_seen": 2435710976
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002118034102306921,
+ "loss": 1.2707,
+ "theoretical_loss": 3.4040049843809155,
+ "tokens_seen": 2435776512
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117873620862588,
+ "loss": 1.1912,
+ "theoretical_loss": 3.4039977022368664,
+ "tokens_seen": 2435842048
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117713139418255,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4039904203435967,
+ "tokens_seen": 2435907584
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117552657973922,
+ "loss": 1.2157,
+ "theoretical_loss": 3.4039831387010917,
+ "tokens_seen": 2435973120
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117392176529589,
+ "loss": 1.3422,
+ "theoretical_loss": 3.4039758573093355,
+ "tokens_seen": 2436038656
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117231695085256,
+ "loss": 1.2587,
+ "theoretical_loss": 3.4039685761683134,
+ "tokens_seen": 2436104192
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002117071213640923,
+ "loss": 1.3304,
+ "theoretical_loss": 3.4039612952780094,
+ "tokens_seen": 2436169728
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000211691073219659,
+ "loss": 1.2743,
+ "theoretical_loss": 3.4039540146384084,
+ "tokens_seen": 2436235264
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002116750250752257,
+ "loss": 1.3233,
+ "theoretical_loss": 3.4039467342494945,
+ "tokens_seen": 2436300800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002116589769307924,
+ "loss": 1.2461,
+ "theoretical_loss": 3.4039394541112533,
+ "tokens_seen": 2436366336
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002116429287863591,
+ "loss": 1.2261,
+ "theoretical_loss": 3.4039321742236686,
+ "tokens_seen": 2436431872
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002116268806419258,
+ "loss": 1.1847,
+ "theoretical_loss": 3.4039248945867255,
+ "tokens_seen": 2436497408
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002116108324974925,
+ "loss": 1.2524,
+ "theoretical_loss": 3.403917615200408,
+ "tokens_seen": 2436562944
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002115947843530592,
+ "loss": 1.181,
+ "theoretical_loss": 3.4039103360647016,
+ "tokens_seen": 2436628480
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021157873620862588,
+ "loss": 1.202,
+ "theoretical_loss": 3.4039030571795905,
+ "tokens_seen": 2436694016
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021156268806419258,
+ "loss": 1.2898,
+ "theoretical_loss": 3.403895778545059,
+ "tokens_seen": 2436759552
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021154663991975928,
+ "loss": 1.2485,
+ "theoretical_loss": 3.4038885001610923,
+ "tokens_seen": 2436825088
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021153059177532598,
+ "loss": 1.2855,
+ "theoretical_loss": 3.4038812220276746,
+ "tokens_seen": 2436890624
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021151454363089268,
+ "loss": 1.2716,
+ "theoretical_loss": 3.403873944144791,
+ "tokens_seen": 2436956160
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021149849548645938,
+ "loss": 1.2029,
+ "theoretical_loss": 3.403866666512426,
+ "tokens_seen": 2437021696
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021148244734202609,
+ "loss": 1.0867,
+ "theoretical_loss": 3.403859389130564,
+ "tokens_seen": 2437087232
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021146639919759279,
+ "loss": 1.1786,
+ "theoretical_loss": 3.4038521119991896,
+ "tokens_seen": 2437152768
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021145035105315949,
+ "loss": 1.1941,
+ "theoretical_loss": 3.4038448351182877,
+ "tokens_seen": 2437218304
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002114343029087262,
+ "loss": 1.2116,
+ "theoretical_loss": 3.403837558487843,
+ "tokens_seen": 2437283840
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002114182547642929,
+ "loss": 1.2469,
+ "theoretical_loss": 3.4038302821078394,
+ "tokens_seen": 2437349376
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002114022066198596,
+ "loss": 1.2363,
+ "theoretical_loss": 3.4038230059782624,
+ "tokens_seen": 2437414912
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002113861584754263,
+ "loss": 1.2721,
+ "theoretical_loss": 3.403815730099097,
+ "tokens_seen": 2437480448
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000211370110330993,
+ "loss": 1.2924,
+ "theoretical_loss": 3.403808454470327,
+ "tokens_seen": 2437545984
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002113540621865597,
+ "loss": 1.1763,
+ "theoretical_loss": 3.403801179091937,
+ "tokens_seen": 2437611520
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002113380140421264,
+ "loss": 1.2239,
+ "theoretical_loss": 3.403793903963912,
+ "tokens_seen": 2437677056
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002113219658976931,
+ "loss": 1.1886,
+ "theoretical_loss": 3.4037866290862366,
+ "tokens_seen": 2437742592
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002113059177532598,
+ "loss": 1.1461,
+ "theoretical_loss": 3.4037793544588952,
+ "tokens_seen": 2437808128
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002112898696088265,
+ "loss": 1.1742,
+ "theoretical_loss": 3.403772080081873,
+ "tokens_seen": 2437873664
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 815053,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3028794527053833,
+ "objective/train/theoretical_loss": 3.4037648059551544,
+ "objective/train/tokens_used": 2458399200,
+ "theoretical_loss": 3.4037648059551544,
+ "tokens_seen": 2437939200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002112738214643932,
+ "loss": 1.2959,
+ "theoretical_loss": 3.4037648059551544,
+ "tokens_seen": 2437939200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002112577733199599,
+ "loss": 1.2335,
+ "theoretical_loss": 3.403757532078724,
+ "tokens_seen": 2438004736
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002112417251755266,
+ "loss": 1.1654,
+ "theoretical_loss": 3.4037502584525665,
+ "tokens_seen": 2438070272
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002112256770310933,
+ "loss": 1.2656,
+ "theoretical_loss": 3.4037429850766667,
+ "tokens_seen": 2438135808
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021120962888666,
+ "loss": 1.2331,
+ "theoretical_loss": 3.403735711951009,
+ "tokens_seen": 2438201344
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021119358074222667,
+ "loss": 1.2808,
+ "theoretical_loss": 3.4037284390755778,
+ "tokens_seen": 2438266880
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021117753259779337,
+ "loss": 1.3389,
+ "theoretical_loss": 3.403721166450359,
+ "tokens_seen": 2438332416
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021116148445336007,
+ "loss": 1.2447,
+ "theoretical_loss": 3.4037138940753353,
+ "tokens_seen": 2438397952
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021114543630892677,
+ "loss": 1.1622,
+ "theoretical_loss": 3.4037066219504934,
+ "tokens_seen": 2438463488
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021112938816449347,
+ "loss": 1.3385,
+ "theoretical_loss": 3.403699350075817,
+ "tokens_seen": 2438529024
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021111334002006017,
+ "loss": 1.2154,
+ "theoretical_loss": 3.4036920784512903,
+ "tokens_seen": 2438594560
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021109729187562687,
+ "loss": 1.2222,
+ "theoretical_loss": 3.4036848070768984,
+ "tokens_seen": 2438660096
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021108124373119357,
+ "loss": 1.2631,
+ "theoretical_loss": 3.4036775359526263,
+ "tokens_seen": 2438725632
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021106519558676027,
+ "loss": 1.3168,
+ "theoretical_loss": 3.4036702650784587,
+ "tokens_seen": 2438791168
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021104914744232698,
+ "loss": 1.3053,
+ "theoretical_loss": 3.40366299445438,
+ "tokens_seen": 2438856704
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021103309929789368,
+ "loss": 1.2031,
+ "theoretical_loss": 3.403655724080375,
+ "tokens_seen": 2438922240
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021101705115346038,
+ "loss": 1.2824,
+ "theoretical_loss": 3.4036484539564276,
+ "tokens_seen": 2438987776
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021100100300902708,
+ "loss": 1.1969,
+ "theoretical_loss": 3.4036411840825234,
+ "tokens_seen": 2439053312
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021098495486459378,
+ "loss": 1.2629,
+ "theoretical_loss": 3.403633914458647,
+ "tokens_seen": 2439118848
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021096890672016048,
+ "loss": 1.1861,
+ "theoretical_loss": 3.403626645084783,
+ "tokens_seen": 2439184384
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021095285857572718,
+ "loss": 1.2027,
+ "theoretical_loss": 3.4036193759609157,
+ "tokens_seen": 2439249920
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021093681043129388,
+ "loss": 1.2814,
+ "theoretical_loss": 3.4036121070870307,
+ "tokens_seen": 2439315456
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021092076228686058,
+ "loss": 1.2502,
+ "theoretical_loss": 3.4036048384631115,
+ "tokens_seen": 2439380992
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021090471414242728,
+ "loss": 1.2428,
+ "theoretical_loss": 3.4035975700891434,
+ "tokens_seen": 2439446528
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021088866599799398,
+ "loss": 1.2376,
+ "theoretical_loss": 3.4035903019651115,
+ "tokens_seen": 2439512064
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021087261785356068,
+ "loss": 1.2151,
+ "theoretical_loss": 3.4035830340909996,
+ "tokens_seen": 2439577600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021085656970912738,
+ "loss": 1.2245,
+ "theoretical_loss": 3.403575766466793,
+ "tokens_seen": 2439643136
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021084052156469408,
+ "loss": 1.1639,
+ "theoretical_loss": 3.4035684990924766,
+ "tokens_seen": 2439708672
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021082447342026078,
+ "loss": 1.1733,
+ "theoretical_loss": 3.403561231968034,
+ "tokens_seen": 2439774208
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002108084252758275,
+ "loss": 1.2355,
+ "theoretical_loss": 3.403553965093451,
+ "tokens_seen": 2439839744
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002107923771313942,
+ "loss": 1.25,
+ "theoretical_loss": 3.403546698468712,
+ "tokens_seen": 2439905280
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002107763289869609,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4035394320938015,
+ "tokens_seen": 2439970816
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021076028084252761,
+ "loss": 1.2048,
+ "theoretical_loss": 3.403532165968705,
+ "tokens_seen": 2440036352
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021074423269809431,
+ "loss": 1.1948,
+ "theoretical_loss": 3.4035249000934056,
+ "tokens_seen": 2440101888
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021072818455366101,
+ "loss": 1.3336,
+ "theoretical_loss": 3.4035176344678892,
+ "tokens_seen": 2440167424
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021071213640922772,
+ "loss": 1.2303,
+ "theoretical_loss": 3.40351036909214,
+ "tokens_seen": 2440232960
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021069608826479442,
+ "loss": 1.1314,
+ "theoretical_loss": 3.4035031039661434,
+ "tokens_seen": 2440298496
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021068004012036112,
+ "loss": 1.2898,
+ "theoretical_loss": 3.4034958390898833,
+ "tokens_seen": 2440364032
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021066399197592782,
+ "loss": 1.3442,
+ "theoretical_loss": 3.403488574463345,
+ "tokens_seen": 2440429568
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021064794383149452,
+ "loss": 1.2493,
+ "theoretical_loss": 3.403481310086513,
+ "tokens_seen": 2440495104
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021063189568706122,
+ "loss": 1.2961,
+ "theoretical_loss": 3.403474045959372,
+ "tokens_seen": 2440560640
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021061584754262792,
+ "loss": 1.2053,
+ "theoretical_loss": 3.4034667820819067,
+ "tokens_seen": 2440626176
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021059979939819462,
+ "loss": 1.1157,
+ "theoretical_loss": 3.4034595184541017,
+ "tokens_seen": 2440691712
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021058375125376132,
+ "loss": 1.2675,
+ "theoretical_loss": 3.403452255075942,
+ "tokens_seen": 2440757248
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000210567703109328,
+ "loss": 1.2813,
+ "theoretical_loss": 3.4034449919474117,
+ "tokens_seen": 2440822784
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002105516549648947,
+ "loss": 1.2083,
+ "theoretical_loss": 3.4034377290684965,
+ "tokens_seen": 2440888320
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002105356068204614,
+ "loss": 1.2027,
+ "theoretical_loss": 3.40343046643918,
+ "tokens_seen": 2440953856
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002105195586760281,
+ "loss": 1.2281,
+ "theoretical_loss": 3.403423204059448,
+ "tokens_seen": 2441019392
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002105035105315948,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4034159419292846,
+ "tokens_seen": 2441084928
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002104874623871615,
+ "loss": 1.1941,
+ "theoretical_loss": 3.4034086800486745,
+ "tokens_seen": 2441150464
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 816249,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.226874589920044,
+ "objective/train/theoretical_loss": 3.403401418417603,
+ "objective/train/tokens_used": 2461676000,
+ "theoretical_loss": 3.403401418417603,
+ "tokens_seen": 2441216000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002104714142427282,
+ "loss": 1.214,
+ "theoretical_loss": 3.403401418417603,
+ "tokens_seen": 2441216000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002104553660982949,
+ "loss": 1.1483,
+ "theoretical_loss": 3.403394157036054,
+ "tokens_seen": 2441281536
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002104393179538616,
+ "loss": 1.2136,
+ "theoretical_loss": 3.4033868959040126,
+ "tokens_seen": 2441347072
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002104232698094283,
+ "loss": 1.2692,
+ "theoretical_loss": 3.4033796350214636,
+ "tokens_seen": 2441412608
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000210407221664995,
+ "loss": 1.2736,
+ "theoretical_loss": 3.403372374388392,
+ "tokens_seen": 2441478144
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103911735205617,
+ "loss": 1.2639,
+ "theoretical_loss": 3.4033651140047816,
+ "tokens_seen": 2441543680
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103751253761284,
+ "loss": 1.1667,
+ "theoretical_loss": 3.4033578538706184,
+ "tokens_seen": 2441609216
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103590772316951,
+ "loss": 1.243,
+ "theoretical_loss": 3.403350593985886,
+ "tokens_seen": 2441674752
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103430290872618,
+ "loss": 1.2145,
+ "theoretical_loss": 3.40334333435057,
+ "tokens_seen": 2441740288
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103269809428285,
+ "loss": 1.1826,
+ "theoretical_loss": 3.4033360749646544,
+ "tokens_seen": 2441805824
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002103109327983952,
+ "loss": 1.2529,
+ "theoretical_loss": 3.4033288158281247,
+ "tokens_seen": 2441871360
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002102948846539619,
+ "loss": 1.2879,
+ "theoretical_loss": 3.403321556940965,
+ "tokens_seen": 2441936896
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002102788365095286,
+ "loss": 1.256,
+ "theoretical_loss": 3.4033142983031603,
+ "tokens_seen": 2442002432
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002102627883650953,
+ "loss": 1.2301,
+ "theoretical_loss": 3.4033070399146954,
+ "tokens_seen": 2442067968
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000210246740220662,
+ "loss": 1.2517,
+ "theoretical_loss": 3.403299781775555,
+ "tokens_seen": 2442133504
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002102306920762287,
+ "loss": 1.1799,
+ "theoretical_loss": 3.4032925238857237,
+ "tokens_seen": 2442199040
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002102146439317954,
+ "loss": 1.2366,
+ "theoretical_loss": 3.4032852662451862,
+ "tokens_seen": 2442264576
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002101985957873621,
+ "loss": 1.1749,
+ "theoretical_loss": 3.4032780088539276,
+ "tokens_seen": 2442330112
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021018254764292878,
+ "loss": 1.2633,
+ "theoretical_loss": 3.4032707517119327,
+ "tokens_seen": 2442395648
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021016649949849548,
+ "loss": 1.2142,
+ "theoretical_loss": 3.4032634948191856,
+ "tokens_seen": 2442461184
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021015045135406218,
+ "loss": 1.2413,
+ "theoretical_loss": 3.403256238175672,
+ "tokens_seen": 2442526720
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021013440320962888,
+ "loss": 1.2829,
+ "theoretical_loss": 3.4032489817813754,
+ "tokens_seen": 2442592256
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021011835506519558,
+ "loss": 1.1604,
+ "theoretical_loss": 3.4032417256362817,
+ "tokens_seen": 2442657792
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021010230692076229,
+ "loss": 1.2321,
+ "theoretical_loss": 3.4032344697403754,
+ "tokens_seen": 2442723328
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00021008625877632899,
+ "loss": 1.1591,
+ "theoretical_loss": 3.403227214093641,
+ "tokens_seen": 2442788864
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002100702106318957,
+ "loss": 1.2773,
+ "theoretical_loss": 3.403219958696063,
+ "tokens_seen": 2442854400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002100541624874624,
+ "loss": 1.1995,
+ "theoretical_loss": 3.4032127035476263,
+ "tokens_seen": 2442919936
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002100381143430291,
+ "loss": 1.2714,
+ "theoretical_loss": 3.4032054486483165,
+ "tokens_seen": 2442985472
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002100220661985958,
+ "loss": 1.1922,
+ "theoretical_loss": 3.4031981939981173,
+ "tokens_seen": 2443051008
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002100060180541625,
+ "loss": 1.318,
+ "theoretical_loss": 3.403190939597014,
+ "tokens_seen": 2443116544
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002099899699097292,
+ "loss": 1.1664,
+ "theoretical_loss": 3.4031836854449917,
+ "tokens_seen": 2443182080
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002099739217652959,
+ "loss": 1.1963,
+ "theoretical_loss": 3.4031764315420343,
+ "tokens_seen": 2443247616
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002099578736208626,
+ "loss": 1.2503,
+ "theoretical_loss": 3.4031691778881266,
+ "tokens_seen": 2443313152
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002099418254764293,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4031619244832543,
+ "tokens_seen": 2443378688
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000209925777331996,
+ "loss": 1.161,
+ "theoretical_loss": 3.4031546713274015,
+ "tokens_seen": 2443444224
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002099097291875627,
+ "loss": 1.3132,
+ "theoretical_loss": 3.403147418420553,
+ "tokens_seen": 2443509760
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098936810431294,
+ "loss": 1.363,
+ "theoretical_loss": 3.403140165762694,
+ "tokens_seen": 2443575296
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098776328986961,
+ "loss": 1.1775,
+ "theoretical_loss": 3.403132913353809,
+ "tokens_seen": 2443640832
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098615847542628,
+ "loss": 1.26,
+ "theoretical_loss": 3.403125661193882,
+ "tokens_seen": 2443706368
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098455366098295,
+ "loss": 1.2412,
+ "theoretical_loss": 3.403118409282899,
+ "tokens_seen": 2443771904
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098294884653962,
+ "loss": 1.2138,
+ "theoretical_loss": 3.4031111576208444,
+ "tokens_seen": 2443837440
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002098134403209629,
+ "loss": 1.1496,
+ "theoretical_loss": 3.4031039062077024,
+ "tokens_seen": 2443902976
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020979739217652957,
+ "loss": 1.201,
+ "theoretical_loss": 3.4030966550434587,
+ "tokens_seen": 2443968512
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020978134403209627,
+ "loss": 1.2342,
+ "theoretical_loss": 3.4030894041280977,
+ "tokens_seen": 2444034048
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020976529588766297,
+ "loss": 1.2555,
+ "theoretical_loss": 3.403082153461604,
+ "tokens_seen": 2444099584
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020974924774322967,
+ "loss": 1.2585,
+ "theoretical_loss": 3.4030749030439624,
+ "tokens_seen": 2444165120
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020973319959879637,
+ "loss": 1.2239,
+ "theoretical_loss": 3.4030676528751576,
+ "tokens_seen": 2444230656
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020971715145436307,
+ "loss": 1.2328,
+ "theoretical_loss": 3.4030604029551745,
+ "tokens_seen": 2444296192
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020970110330992977,
+ "loss": 1.2435,
+ "theoretical_loss": 3.4030531532839987,
+ "tokens_seen": 2444361728
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020968505516549647,
+ "loss": 1.2117,
+ "theoretical_loss": 3.4030459038616137,
+ "tokens_seen": 2444427264
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 817070,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.277119517326355,
+ "objective/train/theoretical_loss": 3.4030386546880047,
+ "objective/train/tokens_used": 2464952800,
+ "theoretical_loss": 3.4030386546880047,
+ "tokens_seen": 2444492800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020966900702106318,
+ "loss": 1.2573,
+ "theoretical_loss": 3.4030386546880047,
+ "tokens_seen": 2444492800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020965295887662988,
+ "loss": 1.2495,
+ "theoretical_loss": 3.4030314057631568,
+ "tokens_seen": 2444558336
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020963691073219658,
+ "loss": 1.2834,
+ "theoretical_loss": 3.4030241570870547,
+ "tokens_seen": 2444623872
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020962086258776328,
+ "loss": 1.2938,
+ "theoretical_loss": 3.403016908659683,
+ "tokens_seen": 2444689408
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020960481444333003,
+ "loss": 1.2004,
+ "theoretical_loss": 3.403009660481027,
+ "tokens_seen": 2444754944
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020958876629889673,
+ "loss": 1.3432,
+ "theoretical_loss": 3.4030024125510705,
+ "tokens_seen": 2444820480
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020957271815446343,
+ "loss": 1.2426,
+ "theoretical_loss": 3.4029951648697994,
+ "tokens_seen": 2444886016
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020955667001003013,
+ "loss": 1.2545,
+ "theoretical_loss": 3.4029879174371978,
+ "tokens_seen": 2444951552
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002095406218655968,
+ "loss": 1.1728,
+ "theoretical_loss": 3.402980670253251,
+ "tokens_seen": 2445017088
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002095245737211635,
+ "loss": 1.1433,
+ "theoretical_loss": 3.4029734233179436,
+ "tokens_seen": 2445082624
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002095085255767302,
+ "loss": 1.2018,
+ "theoretical_loss": 3.40296617663126,
+ "tokens_seen": 2445148160
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002094924774322969,
+ "loss": 1.2357,
+ "theoretical_loss": 3.4029589301931855,
+ "tokens_seen": 2445213696
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002094764292878636,
+ "loss": 1.3183,
+ "theoretical_loss": 3.4029516840037046,
+ "tokens_seen": 2445279232
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002094603811434303,
+ "loss": 1.32,
+ "theoretical_loss": 3.4029444380628027,
+ "tokens_seen": 2445344768
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000209444332998997,
+ "loss": 1.2979,
+ "theoretical_loss": 3.4029371923704637,
+ "tokens_seen": 2445410304
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002094282848545637,
+ "loss": 1.2137,
+ "theoretical_loss": 3.402929946926673,
+ "tokens_seen": 2445475840
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002094122367101304,
+ "loss": 1.2905,
+ "theoretical_loss": 3.4029227017314154,
+ "tokens_seen": 2445541376
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002093961885656971,
+ "loss": 1.2501,
+ "theoretical_loss": 3.4029154567846756,
+ "tokens_seen": 2445606912
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020938014042126381,
+ "loss": 1.1957,
+ "theoretical_loss": 3.4029082120864382,
+ "tokens_seen": 2445672448
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020936409227683051,
+ "loss": 1.3486,
+ "theoretical_loss": 3.402900967636689,
+ "tokens_seen": 2445737984
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020934804413239722,
+ "loss": 1.2664,
+ "theoretical_loss": 3.4028937234354113,
+ "tokens_seen": 2445803520
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020933199598796392,
+ "loss": 1.1844,
+ "theoretical_loss": 3.4028864794825906,
+ "tokens_seen": 2445869056
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020931594784353062,
+ "loss": 1.1847,
+ "theoretical_loss": 3.4028792357782125,
+ "tokens_seen": 2445934592
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020929989969909732,
+ "loss": 1.1499,
+ "theoretical_loss": 3.4028719923222606,
+ "tokens_seen": 2446000128
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020928385155466402,
+ "loss": 1.3032,
+ "theoretical_loss": 3.4028647491147206,
+ "tokens_seen": 2446065664
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020926780341023072,
+ "loss": 1.2617,
+ "theoretical_loss": 3.4028575061555766,
+ "tokens_seen": 2446131200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020925175526579742,
+ "loss": 1.2375,
+ "theoretical_loss": 3.4028502634448143,
+ "tokens_seen": 2446196736
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020923570712136412,
+ "loss": 1.1323,
+ "theoretical_loss": 3.4028430209824174,
+ "tokens_seen": 2446262272
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020921965897693082,
+ "loss": 1.3822,
+ "theoretical_loss": 3.402835778768372,
+ "tokens_seen": 2446327808
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020920361083249752,
+ "loss": 1.2304,
+ "theoretical_loss": 3.402828536802662,
+ "tokens_seen": 2446393344
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020918756268806422,
+ "loss": 1.2206,
+ "theoretical_loss": 3.4028212950852725,
+ "tokens_seen": 2446458880
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020917151454363092,
+ "loss": 1.2382,
+ "theoretical_loss": 3.402814053616188,
+ "tokens_seen": 2446524416
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002091554663991976,
+ "loss": 1.272,
+ "theoretical_loss": 3.4028068123953945,
+ "tokens_seen": 2446589952
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002091394182547643,
+ "loss": 1.1713,
+ "theoretical_loss": 3.4027995714228756,
+ "tokens_seen": 2446655488
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000209123370110331,
+ "loss": 1.3244,
+ "theoretical_loss": 3.4027923306986168,
+ "tokens_seen": 2446721024
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002091073219658977,
+ "loss": 1.2806,
+ "theoretical_loss": 3.4027850902226024,
+ "tokens_seen": 2446786560
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090912738214644,
+ "loss": 1.168,
+ "theoretical_loss": 3.402777849994818,
+ "tokens_seen": 2446852096
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090752256770311,
+ "loss": 1.2777,
+ "theoretical_loss": 3.4027706100152475,
+ "tokens_seen": 2446917632
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090591775325978,
+ "loss": 1.3528,
+ "theoretical_loss": 3.4027633702838767,
+ "tokens_seen": 2446983168
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090431293881645,
+ "loss": 1.2697,
+ "theoretical_loss": 3.4027561308006895,
+ "tokens_seen": 2447048704
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090270812437312,
+ "loss": 1.2678,
+ "theoretical_loss": 3.402748891565672,
+ "tokens_seen": 2447114240
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002090110330992979,
+ "loss": 1.2762,
+ "theoretical_loss": 3.4027416525788072,
+ "tokens_seen": 2447179776
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002089949849548646,
+ "loss": 1.2328,
+ "theoretical_loss": 3.402734413840082,
+ "tokens_seen": 2447245312
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002089789368104313,
+ "loss": 1.1153,
+ "theoretical_loss": 3.4027271753494794,
+ "tokens_seen": 2447310848
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000208962888665998,
+ "loss": 1.2687,
+ "theoretical_loss": 3.4027199371069856,
+ "tokens_seen": 2447376384
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002089468405215647,
+ "loss": 1.2612,
+ "theoretical_loss": 3.402712699112585,
+ "tokens_seen": 2447441920
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002089307923771314,
+ "loss": 1.2083,
+ "theoretical_loss": 3.4027054613662626,
+ "tokens_seen": 2447507456
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002089147442326981,
+ "loss": 1.2288,
+ "theoretical_loss": 3.402698223868003,
+ "tokens_seen": 2447572992
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088986960882648,
+ "loss": 1.3482,
+ "theoretical_loss": 3.402690986617791,
+ "tokens_seen": 2447638528
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088826479438315,
+ "loss": 1.23,
+ "theoretical_loss": 3.4026837496156115,
+ "tokens_seen": 2447704064
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 818155,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3235154151916504,
+ "objective/train/theoretical_loss": 3.4026765128614493,
+ "objective/train/tokens_used": 2468229600,
+ "theoretical_loss": 3.4026765128614493,
+ "tokens_seen": 2447769600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088665997993982,
+ "loss": 1.3005,
+ "theoretical_loss": 3.4026765128614493,
+ "tokens_seen": 2447769600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088505516549649,
+ "loss": 1.2108,
+ "theoretical_loss": 3.40266927635529,
+ "tokens_seen": 2447835136
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088345035105316,
+ "loss": 1.1519,
+ "theoretical_loss": 3.4026620400971175,
+ "tokens_seen": 2447900672
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002088184553660983,
+ "loss": 1.2319,
+ "theoretical_loss": 3.402654804086917,
+ "tokens_seen": 2447966208
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000208802407221665,
+ "loss": 1.151,
+ "theoretical_loss": 3.4026475683246735,
+ "tokens_seen": 2448031744
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002087863590772317,
+ "loss": 1.1803,
+ "theoretical_loss": 3.402640332810372,
+ "tokens_seen": 2448097280
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020877031093279838,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4026330975439967,
+ "tokens_seen": 2448162816
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020875426278836508,
+ "loss": 1.2013,
+ "theoretical_loss": 3.4026258625255332,
+ "tokens_seen": 2448228352
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020873821464393179,
+ "loss": 1.2461,
+ "theoretical_loss": 3.402618627754966,
+ "tokens_seen": 2448293888
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020872216649949849,
+ "loss": 1.2373,
+ "theoretical_loss": 3.40261139323228,
+ "tokens_seen": 2448359424
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020870611835506519,
+ "loss": 1.217,
+ "theoretical_loss": 3.4026041589574603,
+ "tokens_seen": 2448424960
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002086900702106319,
+ "loss": 1.3279,
+ "theoretical_loss": 3.4025969249304913,
+ "tokens_seen": 2448490496
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002086740220661986,
+ "loss": 1.2907,
+ "theoretical_loss": 3.4025896911513587,
+ "tokens_seen": 2448556032
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002086579739217653,
+ "loss": 1.1812,
+ "theoretical_loss": 3.4025824576200465,
+ "tokens_seen": 2448621568
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000208641925777332,
+ "loss": 1.2431,
+ "theoretical_loss": 3.4025752243365397,
+ "tokens_seen": 2448687104
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002086258776328987,
+ "loss": 1.2983,
+ "theoretical_loss": 3.4025679913008235,
+ "tokens_seen": 2448752640
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002086098294884654,
+ "loss": 1.3082,
+ "theoretical_loss": 3.402560758512883,
+ "tokens_seen": 2448818176
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085937813440321,
+ "loss": 1.1862,
+ "theoretical_loss": 3.4025535259727024,
+ "tokens_seen": 2448883712
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085777331995988,
+ "loss": 1.2106,
+ "theoretical_loss": 3.4025462936802673,
+ "tokens_seen": 2448949248
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085616850551655,
+ "loss": 1.2551,
+ "theoretical_loss": 3.402539061635562,
+ "tokens_seen": 2449014784
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085456369107322,
+ "loss": 1.2744,
+ "theoretical_loss": 3.402531829838572,
+ "tokens_seen": 2449080320
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085295887662989,
+ "loss": 1.0943,
+ "theoretical_loss": 3.402524598289281,
+ "tokens_seen": 2449145856
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002085135406218656,
+ "loss": 1.2161,
+ "theoretical_loss": 3.4025173669876754,
+ "tokens_seen": 2449211392
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002084974924774323,
+ "loss": 1.2318,
+ "theoretical_loss": 3.402510135933739,
+ "tokens_seen": 2449276928
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000208481444332999,
+ "loss": 1.2265,
+ "theoretical_loss": 3.402502905127457,
+ "tokens_seen": 2449342464
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002084653961885657,
+ "loss": 1.2588,
+ "theoretical_loss": 3.402495674568815,
+ "tokens_seen": 2449408000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002084493480441324,
+ "loss": 1.2115,
+ "theoretical_loss": 3.4024884442577967,
+ "tokens_seen": 2449473536
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002084332998996991,
+ "loss": 1.2941,
+ "theoretical_loss": 3.4024812141943874,
+ "tokens_seen": 2449539072
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002084172517552658,
+ "loss": 1.1251,
+ "theoretical_loss": 3.4024739843785725,
+ "tokens_seen": 2449604608
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020840120361083253,
+ "loss": 1.2314,
+ "theoretical_loss": 3.4024667548103364,
+ "tokens_seen": 2449670144
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020838515546639923,
+ "loss": 1.3059,
+ "theoretical_loss": 3.4024595254896646,
+ "tokens_seen": 2449735680
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020836910732196593,
+ "loss": 1.2533,
+ "theoretical_loss": 3.402452296416541,
+ "tokens_seen": 2449801216
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020835305917753263,
+ "loss": 1.2457,
+ "theoretical_loss": 3.4024450675909512,
+ "tokens_seen": 2449866752
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020833701103309933,
+ "loss": 1.2098,
+ "theoretical_loss": 3.4024378390128796,
+ "tokens_seen": 2449932288
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020832096288866603,
+ "loss": 1.1117,
+ "theoretical_loss": 3.402430610682312,
+ "tokens_seen": 2449997824
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020830491474423273,
+ "loss": 1.1781,
+ "theoretical_loss": 3.4024233825992325,
+ "tokens_seen": 2450063360
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020828886659979943,
+ "loss": 1.1087,
+ "theoretical_loss": 3.4024161547636265,
+ "tokens_seen": 2450128896
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020827281845536613,
+ "loss": 1.2669,
+ "theoretical_loss": 3.4024089271754785,
+ "tokens_seen": 2450194432
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020825677031093283,
+ "loss": 1.2313,
+ "theoretical_loss": 3.4024016998347735,
+ "tokens_seen": 2450259968
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020824072216649953,
+ "loss": 1.1969,
+ "theoretical_loss": 3.4023944727414968,
+ "tokens_seen": 2450325504
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020822467402206623,
+ "loss": 1.1854,
+ "theoretical_loss": 3.4023872458956324,
+ "tokens_seen": 2450391040
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020820862587763293,
+ "loss": 1.1056,
+ "theoretical_loss": 3.402380019297166,
+ "tokens_seen": 2450456576
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020819257773319963,
+ "loss": 1.2405,
+ "theoretical_loss": 3.4023727929460827,
+ "tokens_seen": 2450522112
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020817652958876633,
+ "loss": 1.0622,
+ "theoretical_loss": 3.402365566842367,
+ "tokens_seen": 2450587648
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020816048144433304,
+ "loss": 1.2725,
+ "theoretical_loss": 3.4023583409860034,
+ "tokens_seen": 2450653184
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002081444332998997,
+ "loss": 1.2608,
+ "theoretical_loss": 3.4023511153769777,
+ "tokens_seen": 2450718720
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002081283851554664,
+ "loss": 1.2369,
+ "theoretical_loss": 3.4023438900152745,
+ "tokens_seen": 2450784256
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002081123370110331,
+ "loss": 1.1828,
+ "theoretical_loss": 3.4023366649008784,
+ "tokens_seen": 2450849792
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002080962888665998,
+ "loss": 1.2195,
+ "theoretical_loss": 3.4023294400337747,
+ "tokens_seen": 2450915328
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002080802407221665,
+ "loss": 1.1741,
+ "theoretical_loss": 3.402322215413948,
+ "tokens_seen": 2450980864
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 819116,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2947700023651123,
+ "objective/train/theoretical_loss": 3.4023149910413832,
+ "objective/train/tokens_used": 2471506400,
+ "theoretical_loss": 3.4023149910413832,
+ "tokens_seen": 2451046400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002080641925777332,
+ "loss": 1.2673,
+ "theoretical_loss": 3.4023149910413832,
+ "tokens_seen": 2451046400
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002080481444332999,
+ "loss": 1.2936,
+ "theoretical_loss": 3.402307766916066,
+ "tokens_seen": 2451111936
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002080320962888666,
+ "loss": 1.1986,
+ "theoretical_loss": 3.4023005430379802,
+ "tokens_seen": 2451177472
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020801604814443331,
+ "loss": 1.1496,
+ "theoretical_loss": 3.4022933194071117,
+ "tokens_seen": 2451243008
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020800000000000001,
+ "loss": 1.2418,
+ "theoretical_loss": 3.402286096023445,
+ "tokens_seen": 2451308544
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020798395185556671,
+ "loss": 1.098,
+ "theoretical_loss": 3.4022788728869644,
+ "tokens_seen": 2451374080
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020796790371113342,
+ "loss": 1.2987,
+ "theoretical_loss": 3.402271649997656,
+ "tokens_seen": 2451439616
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020795185556670012,
+ "loss": 1.2791,
+ "theoretical_loss": 3.4022644273555045,
+ "tokens_seen": 2451505152
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020793580742226682,
+ "loss": 1.3005,
+ "theoretical_loss": 3.402257204960494,
+ "tokens_seen": 2451570688
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020791975927783352,
+ "loss": 1.2636,
+ "theoretical_loss": 3.4022499828126103,
+ "tokens_seen": 2451636224
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020790371113340022,
+ "loss": 1.201,
+ "theoretical_loss": 3.4022427609118377,
+ "tokens_seen": 2451701760
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020788766298896692,
+ "loss": 1.142,
+ "theoretical_loss": 3.402235539258162,
+ "tokens_seen": 2451767296
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020787161484453362,
+ "loss": 1.163,
+ "theoretical_loss": 3.402228317851567,
+ "tokens_seen": 2451832832
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020785556670010032,
+ "loss": 1.2044,
+ "theoretical_loss": 3.4022210966920383,
+ "tokens_seen": 2451898368
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020783951855566702,
+ "loss": 1.2662,
+ "theoretical_loss": 3.402213875779561,
+ "tokens_seen": 2451963904
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020782347041123372,
+ "loss": 1.2094,
+ "theoretical_loss": 3.40220665511412,
+ "tokens_seen": 2452029440
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020780742226680042,
+ "loss": 1.2085,
+ "theoretical_loss": 3.4021994346957,
+ "tokens_seen": 2452094976
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020779137412236712,
+ "loss": 1.2771,
+ "theoretical_loss": 3.4021922145242858,
+ "tokens_seen": 2452160512
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020777532597793382,
+ "loss": 1.258,
+ "theoretical_loss": 3.4021849945998626,
+ "tokens_seen": 2452226048
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002077592778335005,
+ "loss": 1.1628,
+ "theoretical_loss": 3.4021777749224156,
+ "tokens_seen": 2452291584
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002077432296890672,
+ "loss": 1.2177,
+ "theoretical_loss": 3.4021705554919293,
+ "tokens_seen": 2452357120
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002077271815446339,
+ "loss": 1.2857,
+ "theoretical_loss": 3.4021633363083885,
+ "tokens_seen": 2452422656
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002077111334002006,
+ "loss": 1.2425,
+ "theoretical_loss": 3.4021561173717787,
+ "tokens_seen": 2452488192
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002076950852557673,
+ "loss": 1.1928,
+ "theoretical_loss": 3.402148898682085,
+ "tokens_seen": 2452553728
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000207679037111334,
+ "loss": 1.2478,
+ "theoretical_loss": 3.402141680239292,
+ "tokens_seen": 2452619264
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002076629889669007,
+ "loss": 1.164,
+ "theoretical_loss": 3.402134462043384,
+ "tokens_seen": 2452684800
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002076469408224674,
+ "loss": 1.2529,
+ "theoretical_loss": 3.402127244094347,
+ "tokens_seen": 2452750336
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002076308926780341,
+ "loss": 1.1733,
+ "theoretical_loss": 3.402120026392166,
+ "tokens_seen": 2452815872
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002076148445336008,
+ "loss": 1.2298,
+ "theoretical_loss": 3.402112808936825,
+ "tokens_seen": 2452881408
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075987963891675,
+ "loss": 1.2405,
+ "theoretical_loss": 3.4021055917283096,
+ "tokens_seen": 2452946944
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075827482447342,
+ "loss": 1.1828,
+ "theoretical_loss": 3.402098374766605,
+ "tokens_seen": 2453012480
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075667001003009,
+ "loss": 1.2505,
+ "theoretical_loss": 3.4020911580516953,
+ "tokens_seen": 2453078016
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075506519558676,
+ "loss": 1.1903,
+ "theoretical_loss": 3.402083941583566,
+ "tokens_seen": 2453143552
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075346038114343,
+ "loss": 1.1833,
+ "theoretical_loss": 3.4020767253622024,
+ "tokens_seen": 2453209088
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000207518555667001,
+ "loss": 1.2091,
+ "theoretical_loss": 3.402069509387589,
+ "tokens_seen": 2453274624
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002075025075225677,
+ "loss": 1.2554,
+ "theoretical_loss": 3.402062293659711,
+ "tokens_seen": 2453340160
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074864593781344,
+ "loss": 1.1478,
+ "theoretical_loss": 3.4020550781785532,
+ "tokens_seen": 2453405696
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074704112337011,
+ "loss": 1.2385,
+ "theoretical_loss": 3.4020478629441007,
+ "tokens_seen": 2453471232
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074543630892678,
+ "loss": 1.2135,
+ "theoretical_loss": 3.4020406479563383,
+ "tokens_seen": 2453536768
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074383149448345,
+ "loss": 1.2746,
+ "theoretical_loss": 3.4020334332152515,
+ "tokens_seen": 2453602304
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074222668004012,
+ "loss": 1.2242,
+ "theoretical_loss": 3.402026218720825,
+ "tokens_seen": 2453667840
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002074062186559679,
+ "loss": 1.235,
+ "theoretical_loss": 3.4020190044730425,
+ "tokens_seen": 2453733376
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002073901705115346,
+ "loss": 1.1238,
+ "theoretical_loss": 3.4020117904718914,
+ "tokens_seen": 2453798912
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020737412236710128,
+ "loss": 1.2187,
+ "theoretical_loss": 3.4020045767173546,
+ "tokens_seen": 2453864448
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020735807422266799,
+ "loss": 1.2126,
+ "theoretical_loss": 3.4019973632094183,
+ "tokens_seen": 2453929984
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020734202607823469,
+ "loss": 1.2886,
+ "theoretical_loss": 3.4019901499480674,
+ "tokens_seen": 2453995520
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002073259779338014,
+ "loss": 1.198,
+ "theoretical_loss": 3.401982936933286,
+ "tokens_seen": 2454061056
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002073099297893681,
+ "loss": 1.3155,
+ "theoretical_loss": 3.40197572416506,
+ "tokens_seen": 2454126592
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072938816449348,
+ "loss": 1.2259,
+ "theoretical_loss": 3.401968511643374,
+ "tokens_seen": 2454192128
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072778335005015,
+ "loss": 1.3032,
+ "theoretical_loss": 3.401961299368213,
+ "tokens_seen": 2454257664
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 820455,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.244768738746643,
+ "objective/train/theoretical_loss": 3.401954087339562,
+ "objective/train/tokens_used": 2474783200,
+ "theoretical_loss": 3.401954087339562,
+ "tokens_seen": 2454323200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072617853560682,
+ "loss": 1.208,
+ "theoretical_loss": 3.401954087339562,
+ "tokens_seen": 2454323200
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072457372116349,
+ "loss": 1.2968,
+ "theoretical_loss": 3.4019468755574063,
+ "tokens_seen": 2454388736
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072296890672016,
+ "loss": 1.2244,
+ "theoretical_loss": 3.4019396640217305,
+ "tokens_seen": 2454454272
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002072136409227683,
+ "loss": 1.2645,
+ "theoretical_loss": 3.4019324527325194,
+ "tokens_seen": 2454519808
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020719759277833505,
+ "loss": 1.2233,
+ "theoretical_loss": 3.4019252416897587,
+ "tokens_seen": 2454585344
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020718154463390175,
+ "loss": 1.2393,
+ "theoretical_loss": 3.401918030893433,
+ "tokens_seen": 2454650880
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020716549648946845,
+ "loss": 1.2546,
+ "theoretical_loss": 3.401910820343527,
+ "tokens_seen": 2454716416
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020714944834503515,
+ "loss": 1.2543,
+ "theoretical_loss": 3.401903610040026,
+ "tokens_seen": 2454781952
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020713340020060182,
+ "loss": 1.2841,
+ "theoretical_loss": 3.4018963999829155,
+ "tokens_seen": 2454847488
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020711735205616852,
+ "loss": 1.279,
+ "theoretical_loss": 3.4018891901721795,
+ "tokens_seen": 2454913024
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020710130391173522,
+ "loss": 1.2405,
+ "theoretical_loss": 3.401881980607804,
+ "tokens_seen": 2454978560
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020708525576730192,
+ "loss": 1.3013,
+ "theoretical_loss": 3.4018747712897732,
+ "tokens_seen": 2455044096
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020706920762286862,
+ "loss": 1.2081,
+ "theoretical_loss": 3.4018675622180723,
+ "tokens_seen": 2455109632
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020705315947843532,
+ "loss": 1.3198,
+ "theoretical_loss": 3.4018603533926868,
+ "tokens_seen": 2455175168
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020703711133400203,
+ "loss": 1.3187,
+ "theoretical_loss": 3.401853144813601,
+ "tokens_seen": 2455240704
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020702106318956873,
+ "loss": 1.3014,
+ "theoretical_loss": 3.4018459364808007,
+ "tokens_seen": 2455306240
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020700501504513543,
+ "loss": 1.0781,
+ "theoretical_loss": 3.4018387283942704,
+ "tokens_seen": 2455371776
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020698896690070213,
+ "loss": 1.1851,
+ "theoretical_loss": 3.401831520553995,
+ "tokens_seen": 2455437312
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020697291875626883,
+ "loss": 1.2065,
+ "theoretical_loss": 3.4018243129599597,
+ "tokens_seen": 2455502848
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020695687061183553,
+ "loss": 1.2741,
+ "theoretical_loss": 3.4018171056121496,
+ "tokens_seen": 2455568384
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020694082246740223,
+ "loss": 1.2952,
+ "theoretical_loss": 3.4018098985105496,
+ "tokens_seen": 2455633920
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020692477432296893,
+ "loss": 1.1389,
+ "theoretical_loss": 3.4018026916551447,
+ "tokens_seen": 2455699456
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020690872617853563,
+ "loss": 1.2579,
+ "theoretical_loss": 3.4017954850459198,
+ "tokens_seen": 2455764992
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020689267803410233,
+ "loss": 1.2653,
+ "theoretical_loss": 3.4017882786828606,
+ "tokens_seen": 2455830528
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020687662988966903,
+ "loss": 1.2538,
+ "theoretical_loss": 3.401781072565951,
+ "tokens_seen": 2455896064
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020686058174523573,
+ "loss": 1.2709,
+ "theoretical_loss": 3.4017738666951773,
+ "tokens_seen": 2455961600
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020684453360080243,
+ "loss": 1.2868,
+ "theoretical_loss": 3.4017666610705235,
+ "tokens_seen": 2456027136
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020682848545636913,
+ "loss": 1.2406,
+ "theoretical_loss": 3.401759455691975,
+ "tokens_seen": 2456092672
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020681243731193583,
+ "loss": 1.1935,
+ "theoretical_loss": 3.401752250559517,
+ "tokens_seen": 2456158208
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020679638916750253,
+ "loss": 1.3005,
+ "theoretical_loss": 3.401745045673134,
+ "tokens_seen": 2456223744
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020678034102306924,
+ "loss": 1.2398,
+ "theoretical_loss": 3.4017378410328116,
+ "tokens_seen": 2456289280
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020676429287863594,
+ "loss": 1.2245,
+ "theoretical_loss": 3.401730636638535,
+ "tokens_seen": 2456354816
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002067482447342026,
+ "loss": 1.213,
+ "theoretical_loss": 3.401723432490288,
+ "tokens_seen": 2456420352
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002067321965897693,
+ "loss": 1.2237,
+ "theoretical_loss": 3.4017162285880573,
+ "tokens_seen": 2456485888
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.000206716148445336,
+ "loss": 1.3082,
+ "theoretical_loss": 3.4017090249318267,
+ "tokens_seen": 2456551424
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002067001003009027,
+ "loss": 1.2062,
+ "theoretical_loss": 3.4017018215215815,
+ "tokens_seen": 2456616960
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002066840521564694,
+ "loss": 1.1916,
+ "theoretical_loss": 3.4016946183573076,
+ "tokens_seen": 2456682496
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002066680040120361,
+ "loss": 1.1984,
+ "theoretical_loss": 3.4016874154389884,
+ "tokens_seen": 2456748032
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002066519558676028,
+ "loss": 1.2414,
+ "theoretical_loss": 3.4016802127666104,
+ "tokens_seen": 2456813568
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020663590772316951,
+ "loss": 1.1979,
+ "theoretical_loss": 3.4016730103401582,
+ "tokens_seen": 2456879104
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020661985957873621,
+ "loss": 1.2383,
+ "theoretical_loss": 3.4016658081596165,
+ "tokens_seen": 2456944640
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020660381143430292,
+ "loss": 1.2672,
+ "theoretical_loss": 3.401658606224971,
+ "tokens_seen": 2457010176
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020658776328986962,
+ "loss": 1.1914,
+ "theoretical_loss": 3.4016514045362056,
+ "tokens_seen": 2457075712
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020657171514543632,
+ "loss": 1.2629,
+ "theoretical_loss": 3.4016442030933067,
+ "tokens_seen": 2457141248
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020655566700100302,
+ "loss": 1.1803,
+ "theoretical_loss": 3.4016370018962587,
+ "tokens_seen": 2457206784
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020653961885656972,
+ "loss": 1.3264,
+ "theoretical_loss": 3.4016298009450465,
+ "tokens_seen": 2457272320
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020652357071213642,
+ "loss": 1.2146,
+ "theoretical_loss": 3.4016226002396555,
+ "tokens_seen": 2457337856
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020650752256770312,
+ "loss": 1.2644,
+ "theoretical_loss": 3.4016153997800704,
+ "tokens_seen": 2457403392
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020649147442326982,
+ "loss": 1.2561,
+ "theoretical_loss": 3.401608199566277,
+ "tokens_seen": 2457468928
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020647542627883652,
+ "loss": 1.1833,
+ "theoretical_loss": 3.4016009995982595,
+ "tokens_seen": 2457534464
+ },
+ {
+ "epoch": 0.74,
+ "objective/train/docs_used": 821745,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2310079336166382,
+ "objective/train/theoretical_loss": 3.401593799876003,
+ "objective/train/tokens_used": 2478060000,
+ "theoretical_loss": 3.401593799876003,
+ "tokens_seen": 2457600000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020645937813440322,
+ "loss": 1.1677,
+ "theoretical_loss": 3.401593799876003,
+ "tokens_seen": 2457600000
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020644332998996992,
+ "loss": 1.1647,
+ "theoretical_loss": 3.401586600399493,
+ "tokens_seen": 2457665536
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020642728184553662,
+ "loss": 1.3308,
+ "theoretical_loss": 3.401579401168714,
+ "tokens_seen": 2457731072
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020641123370110332,
+ "loss": 1.1624,
+ "theoretical_loss": 3.4015722021836523,
+ "tokens_seen": 2457796608
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020639518555667002,
+ "loss": 1.2597,
+ "theoretical_loss": 3.4015650034442917,
+ "tokens_seen": 2457862144
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.00020637913741223672,
+ "loss": 1.3063,
+ "theoretical_loss": 3.4015578049506177,
+ "tokens_seen": 2457927680
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002063630892678034,
+ "loss": 1.224,
+ "theoretical_loss": 3.401550606702615,
+ "tokens_seen": 2457993216
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002063470411233701,
+ "loss": 1.1333,
+ "theoretical_loss": 3.4015434087002694,
+ "tokens_seen": 2458058752
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002063309929789368,
+ "loss": 1.3672,
+ "theoretical_loss": 3.4015362109435654,
+ "tokens_seen": 2458124288
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002063149448345035,
+ "loss": 1.1334,
+ "theoretical_loss": 3.401529013432488,
+ "tokens_seen": 2458189824
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002062988966900702,
+ "loss": 1.2159,
+ "theoretical_loss": 3.401521816167023,
+ "tokens_seen": 2458255360
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002062828485456369,
+ "loss": 1.2202,
+ "theoretical_loss": 3.4015146191471546,
+ "tokens_seen": 2458320896
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002062668004012036,
+ "loss": 1.1894,
+ "theoretical_loss": 3.4015074223728687,
+ "tokens_seen": 2458386432
+ },
+ {
+ "epoch": 0.74,
+ "learning_rate": 0.0002062507522567703,
+ "loss": 1.2767,
+ "theoretical_loss": 3.4015002258441496,
+ "tokens_seen": 2458451968
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.000206234704112337,
+ "loss": 1.2532,
+ "theoretical_loss": 3.4014930295609824,
+ "tokens_seen": 2458517504
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002062186559679037,
+ "loss": 1.2043,
+ "theoretical_loss": 3.401485833523353,
+ "tokens_seen": 2458583040
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002062026078234704,
+ "loss": 1.23,
+ "theoretical_loss": 3.4014786377312456,
+ "tokens_seen": 2458648576
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061865596790371,
+ "loss": 1.2265,
+ "theoretical_loss": 3.401471442184646,
+ "tokens_seen": 2458714112
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061705115346038,
+ "loss": 1.2015,
+ "theoretical_loss": 3.4014642468835383,
+ "tokens_seen": 2458779648
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061544633901705,
+ "loss": 1.24,
+ "theoretical_loss": 3.401457051827909,
+ "tokens_seen": 2458845184
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061384152457372,
+ "loss": 1.1823,
+ "theoretical_loss": 3.4014498570177416,
+ "tokens_seen": 2458910720
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061223671013039,
+ "loss": 1.2175,
+ "theoretical_loss": 3.4014426624530225,
+ "tokens_seen": 2458976256
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002061063189568706,
+ "loss": 1.2433,
+ "theoretical_loss": 3.401435468133736,
+ "tokens_seen": 2459041792
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002060902708124373,
+ "loss": 1.3019,
+ "theoretical_loss": 3.4014282740598674,
+ "tokens_seen": 2459107328
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.000206074222668004,
+ "loss": 1.2126,
+ "theoretical_loss": 3.401421080231402,
+ "tokens_seen": 2459172864
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002060581745235707,
+ "loss": 1.268,
+ "theoretical_loss": 3.401413886648325,
+ "tokens_seen": 2459238400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002060421263791374,
+ "loss": 1.2252,
+ "theoretical_loss": 3.40140669331062,
+ "tokens_seen": 2459303936
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002060260782347041,
+ "loss": 1.2317,
+ "theoretical_loss": 3.4013995002182744,
+ "tokens_seen": 2459369472
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002060100300902708,
+ "loss": 1.1872,
+ "theoretical_loss": 3.401392307371272,
+ "tokens_seen": 2459435008
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020599398194583754,
+ "loss": 1.2033,
+ "theoretical_loss": 3.401385114769598,
+ "tokens_seen": 2459500544
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020597793380140424,
+ "loss": 1.1539,
+ "theoretical_loss": 3.4013779224132374,
+ "tokens_seen": 2459566080
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020596188565697094,
+ "loss": 1.2724,
+ "theoretical_loss": 3.401370730302176,
+ "tokens_seen": 2459631616
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020594583751253764,
+ "loss": 1.332,
+ "theoretical_loss": 3.4013635384363976,
+ "tokens_seen": 2459697152
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020592978936810434,
+ "loss": 1.371,
+ "theoretical_loss": 3.4013563468158887,
+ "tokens_seen": 2459762688
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020591374122367104,
+ "loss": 1.1665,
+ "theoretical_loss": 3.4013491554406334,
+ "tokens_seen": 2459828224
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020589769307923774,
+ "loss": 1.2184,
+ "theoretical_loss": 3.4013419643106175,
+ "tokens_seen": 2459893760
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020588164493480444,
+ "loss": 1.2622,
+ "theoretical_loss": 3.4013347734258255,
+ "tokens_seen": 2459959296
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020586559679037114,
+ "loss": 1.2384,
+ "theoretical_loss": 3.401327582786243,
+ "tokens_seen": 2460024832
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020584954864593785,
+ "loss": 1.2298,
+ "theoretical_loss": 3.4013203923918547,
+ "tokens_seen": 2460090368
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020583350050150455,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4013132022426458,
+ "tokens_seen": 2460155904
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020581745235707125,
+ "loss": 1.222,
+ "theoretical_loss": 3.4013060123386016,
+ "tokens_seen": 2460221440
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020580140421263795,
+ "loss": 1.1952,
+ "theoretical_loss": 3.401298822679707,
+ "tokens_seen": 2460286976
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020578535606820465,
+ "loss": 1.2134,
+ "theoretical_loss": 3.4012916332659477,
+ "tokens_seen": 2460352512
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020576930792377135,
+ "loss": 1.4222,
+ "theoretical_loss": 3.401284444097308,
+ "tokens_seen": 2460418048
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020575325977933805,
+ "loss": 1.3339,
+ "theoretical_loss": 3.4012772551737736,
+ "tokens_seen": 2460483584
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020573721163490475,
+ "loss": 1.1863,
+ "theoretical_loss": 3.4012700664953286,
+ "tokens_seen": 2460549120
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020572116349047142,
+ "loss": 1.1546,
+ "theoretical_loss": 3.40126287806196,
+ "tokens_seen": 2460614656
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020570511534603812,
+ "loss": 1.3119,
+ "theoretical_loss": 3.401255689873651,
+ "tokens_seen": 2460680192
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020568906720160482,
+ "loss": 1.2238,
+ "theoretical_loss": 3.4012485019303877,
+ "tokens_seen": 2460745728
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020567301905717153,
+ "loss": 1.2401,
+ "theoretical_loss": 3.401241314232155,
+ "tokens_seen": 2460811264
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 822717,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.288038730621338,
+ "objective/train/theoretical_loss": 3.401234126778938,
+ "objective/train/tokens_used": 2481336800,
+ "theoretical_loss": 3.401234126778938,
+ "tokens_seen": 2460876800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020565697091273823,
+ "loss": 1.239,
+ "theoretical_loss": 3.401234126778938,
+ "tokens_seen": 2460876800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020564092276830493,
+ "loss": 1.2188,
+ "theoretical_loss": 3.4012269395707224,
+ "tokens_seen": 2460942336
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020562487462387163,
+ "loss": 1.2133,
+ "theoretical_loss": 3.4012197526074925,
+ "tokens_seen": 2461007872
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020560882647943833,
+ "loss": 1.2739,
+ "theoretical_loss": 3.401212565889234,
+ "tokens_seen": 2461073408
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020559277833500503,
+ "loss": 1.2217,
+ "theoretical_loss": 3.401205379415931,
+ "tokens_seen": 2461138944
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020557673019057173,
+ "loss": 1.2564,
+ "theoretical_loss": 3.40119819318757,
+ "tokens_seen": 2461204480
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020556068204613843,
+ "loss": 1.2609,
+ "theoretical_loss": 3.4011910072041354,
+ "tokens_seen": 2461270016
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020554463390170513,
+ "loss": 1.349,
+ "theoretical_loss": 3.4011838214656125,
+ "tokens_seen": 2461335552
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020552858575727183,
+ "loss": 1.3105,
+ "theoretical_loss": 3.401176635971986,
+ "tokens_seen": 2461401088
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020551253761283853,
+ "loss": 1.1493,
+ "theoretical_loss": 3.4011694507232417,
+ "tokens_seen": 2461466624
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020549648946840523,
+ "loss": 1.2055,
+ "theoretical_loss": 3.4011622657193645,
+ "tokens_seen": 2461532160
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020548044132397193,
+ "loss": 1.2198,
+ "theoretical_loss": 3.40115508096034,
+ "tokens_seen": 2461597696
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020546439317953863,
+ "loss": 1.1566,
+ "theoretical_loss": 3.401147896446152,
+ "tokens_seen": 2461663232
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020544834503510533,
+ "loss": 1.2226,
+ "theoretical_loss": 3.4011407121767867,
+ "tokens_seen": 2461728768
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020543229689067203,
+ "loss": 1.1852,
+ "theoretical_loss": 3.4011335281522292,
+ "tokens_seen": 2461794304
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020541624874623874,
+ "loss": 1.2178,
+ "theoretical_loss": 3.4011263443724644,
+ "tokens_seen": 2461859840
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020540020060180544,
+ "loss": 1.2647,
+ "theoretical_loss": 3.401119160837477,
+ "tokens_seen": 2461925376
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020538415245737214,
+ "loss": 1.2907,
+ "theoretical_loss": 3.401111977547253,
+ "tokens_seen": 2461990912
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020536810431293884,
+ "loss": 1.2758,
+ "theoretical_loss": 3.4011047945017774,
+ "tokens_seen": 2462056448
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020535205616850554,
+ "loss": 1.2473,
+ "theoretical_loss": 3.401097611701035,
+ "tokens_seen": 2462121984
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002053360080240722,
+ "loss": 1.1714,
+ "theoretical_loss": 3.401090429145011,
+ "tokens_seen": 2462187520
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002053199598796389,
+ "loss": 1.2319,
+ "theoretical_loss": 3.4010832468336907,
+ "tokens_seen": 2462253056
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002053039117352056,
+ "loss": 1.2347,
+ "theoretical_loss": 3.4010760647670595,
+ "tokens_seen": 2462318592
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002052878635907723,
+ "loss": 1.2177,
+ "theoretical_loss": 3.4010688829451015,
+ "tokens_seen": 2462384128
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020527181544633901,
+ "loss": 1.1427,
+ "theoretical_loss": 3.4010617013678033,
+ "tokens_seen": 2462449664
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020525576730190571,
+ "loss": 1.2248,
+ "theoretical_loss": 3.4010545200351485,
+ "tokens_seen": 2462515200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020523971915747242,
+ "loss": 1.2802,
+ "theoretical_loss": 3.401047338947124,
+ "tokens_seen": 2462580736
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020522367101303912,
+ "loss": 1.2035,
+ "theoretical_loss": 3.4010401581037133,
+ "tokens_seen": 2462646272
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020520762286860582,
+ "loss": 1.1558,
+ "theoretical_loss": 3.4010329775049026,
+ "tokens_seen": 2462711808
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020519157472417252,
+ "loss": 1.2238,
+ "theoretical_loss": 3.4010257971506768,
+ "tokens_seen": 2462777344
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020517552657973922,
+ "loss": 1.2378,
+ "theoretical_loss": 3.401018617041021,
+ "tokens_seen": 2462842880
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020515947843530592,
+ "loss": 1.2178,
+ "theoretical_loss": 3.4010114371759204,
+ "tokens_seen": 2462908416
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020514343029087262,
+ "loss": 1.2749,
+ "theoretical_loss": 3.40100425755536,
+ "tokens_seen": 2462973952
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020512738214643932,
+ "loss": 1.3236,
+ "theoretical_loss": 3.4009970781793255,
+ "tokens_seen": 2463039488
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020511133400200602,
+ "loss": 1.2713,
+ "theoretical_loss": 3.400989899047801,
+ "tokens_seen": 2463105024
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020509528585757272,
+ "loss": 1.1697,
+ "theoretical_loss": 3.4009827201607727,
+ "tokens_seen": 2463170560
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020507923771313942,
+ "loss": 1.1927,
+ "theoretical_loss": 3.4009755415182257,
+ "tokens_seen": 2463236096
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020506318956870612,
+ "loss": 1.2339,
+ "theoretical_loss": 3.4009683631201444,
+ "tokens_seen": 2463301632
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020504714142427282,
+ "loss": 1.268,
+ "theoretical_loss": 3.400961184966515,
+ "tokens_seen": 2463367168
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020503109327983952,
+ "loss": 1.1121,
+ "theoretical_loss": 3.400954007057322,
+ "tokens_seen": 2463432704
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020501504513540622,
+ "loss": 1.2205,
+ "theoretical_loss": 3.4009468293925504,
+ "tokens_seen": 2463498240
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020499899699097292,
+ "loss": 1.2604,
+ "theoretical_loss": 3.4009396519721857,
+ "tokens_seen": 2463563776
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020498294884653963,
+ "loss": 1.2815,
+ "theoretical_loss": 3.400932474796213,
+ "tokens_seen": 2463629312
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020496690070210633,
+ "loss": 1.314,
+ "theoretical_loss": 3.400925297864618,
+ "tokens_seen": 2463694848
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.000204950852557673,
+ "loss": 1.265,
+ "theoretical_loss": 3.400918121177385,
+ "tokens_seen": 2463760384
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002049348044132397,
+ "loss": 1.2212,
+ "theoretical_loss": 3.4009109447344996,
+ "tokens_seen": 2463825920
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002049187562688064,
+ "loss": 1.3325,
+ "theoretical_loss": 3.4009037685359473,
+ "tokens_seen": 2463891456
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002049027081243731,
+ "loss": 1.23,
+ "theoretical_loss": 3.4008965925817125,
+ "tokens_seen": 2463956992
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048866599799398,
+ "loss": 1.3348,
+ "theoretical_loss": 3.400889416871781,
+ "tokens_seen": 2464022528
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048706118355065,
+ "loss": 1.2437,
+ "theoretical_loss": 3.400882241406138,
+ "tokens_seen": 2464088064
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 824064,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3477821350097656,
+ "objective/train/theoretical_loss": 3.4008750661847684,
+ "objective/train/tokens_used": 2484613600,
+ "theoretical_loss": 3.4008750661847684,
+ "tokens_seen": 2464153600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048545636910732,
+ "loss": 1.2699,
+ "theoretical_loss": 3.4008750661847684,
+ "tokens_seen": 2464153600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048385155466399,
+ "loss": 1.3319,
+ "theoretical_loss": 3.4008678912076573,
+ "tokens_seen": 2464219136
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048224674022066,
+ "loss": 1.1806,
+ "theoretical_loss": 3.4008607164747904,
+ "tokens_seen": 2464284672
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002048064192577733,
+ "loss": 1.2831,
+ "theoretical_loss": 3.4008535419861525,
+ "tokens_seen": 2464350208
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020479037111334006,
+ "loss": 1.2485,
+ "theoretical_loss": 3.4008463677417287,
+ "tokens_seen": 2464415744
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020477432296890676,
+ "loss": 1.1256,
+ "theoretical_loss": 3.4008391937415046,
+ "tokens_seen": 2464481280
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020475827482447346,
+ "loss": 1.2369,
+ "theoretical_loss": 3.400832019985465,
+ "tokens_seen": 2464546816
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020474222668004016,
+ "loss": 1.231,
+ "theoretical_loss": 3.4008248464735953,
+ "tokens_seen": 2464612352
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020472617853560686,
+ "loss": 1.2005,
+ "theoretical_loss": 3.4008176732058804,
+ "tokens_seen": 2464677888
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020471013039117354,
+ "loss": 1.2592,
+ "theoretical_loss": 3.400810500182306,
+ "tokens_seen": 2464743424
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020469408224674024,
+ "loss": 1.3405,
+ "theoretical_loss": 3.4008033274028575,
+ "tokens_seen": 2464808960
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020467803410230694,
+ "loss": 1.2228,
+ "theoretical_loss": 3.400796154867519,
+ "tokens_seen": 2464874496
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020466198595787364,
+ "loss": 1.28,
+ "theoretical_loss": 3.4007889825762767,
+ "tokens_seen": 2464940032
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020464593781344034,
+ "loss": 1.2225,
+ "theoretical_loss": 3.400781810529115,
+ "tokens_seen": 2465005568
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020462988966900704,
+ "loss": 1.245,
+ "theoretical_loss": 3.40077463872602,
+ "tokens_seen": 2465071104
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020461384152457374,
+ "loss": 1.4661,
+ "theoretical_loss": 3.4007674671669763,
+ "tokens_seen": 2465136640
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020459779338014044,
+ "loss": 1.2308,
+ "theoretical_loss": 3.400760295851969,
+ "tokens_seen": 2465202176
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020458174523570714,
+ "loss": 1.2438,
+ "theoretical_loss": 3.400753124780984,
+ "tokens_seen": 2465267712
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020456569709127384,
+ "loss": 1.1934,
+ "theoretical_loss": 3.400745953954006,
+ "tokens_seen": 2465333248
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020454964894684054,
+ "loss": 1.2494,
+ "theoretical_loss": 3.40073878337102,
+ "tokens_seen": 2465398784
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020453360080240724,
+ "loss": 1.2292,
+ "theoretical_loss": 3.400731613032012,
+ "tokens_seen": 2465464320
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020451755265797394,
+ "loss": 1.3447,
+ "theoretical_loss": 3.4007244429369665,
+ "tokens_seen": 2465529856
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020450150451354064,
+ "loss": 1.1722,
+ "theoretical_loss": 3.400717273085869,
+ "tokens_seen": 2465595392
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020448545636910734,
+ "loss": 1.184,
+ "theoretical_loss": 3.4007101034787044,
+ "tokens_seen": 2465660928
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020446940822467405,
+ "loss": 1.2654,
+ "theoretical_loss": 3.4007029341154587,
+ "tokens_seen": 2465726464
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020445336008024075,
+ "loss": 1.3029,
+ "theoretical_loss": 3.400695764996116,
+ "tokens_seen": 2465792000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020443731193580745,
+ "loss": 1.2573,
+ "theoretical_loss": 3.4006885961206623,
+ "tokens_seen": 2465857536
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020442126379137415,
+ "loss": 1.2895,
+ "theoretical_loss": 3.400681427489083,
+ "tokens_seen": 2465923072
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020440521564694085,
+ "loss": 1.1699,
+ "theoretical_loss": 3.4006742591013626,
+ "tokens_seen": 2465988608
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020438916750250755,
+ "loss": 1.1582,
+ "theoretical_loss": 3.4006670909574868,
+ "tokens_seen": 2466054144
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020437311935807425,
+ "loss": 1.2569,
+ "theoretical_loss": 3.4006599230574404,
+ "tokens_seen": 2466119680
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020435707121364095,
+ "loss": 1.18,
+ "theoretical_loss": 3.400652755401209,
+ "tokens_seen": 2466185216
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020434102306920765,
+ "loss": 1.3147,
+ "theoretical_loss": 3.4006455879887785,
+ "tokens_seen": 2466250752
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020432497492477432,
+ "loss": 1.246,
+ "theoretical_loss": 3.400638420820133,
+ "tokens_seen": 2466316288
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020430892678034102,
+ "loss": 1.1472,
+ "theoretical_loss": 3.4006312538952574,
+ "tokens_seen": 2466381824
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020429287863590773,
+ "loss": 1.3528,
+ "theoretical_loss": 3.4006240872141387,
+ "tokens_seen": 2466447360
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020427683049147443,
+ "loss": 1.2192,
+ "theoretical_loss": 3.4006169207767605,
+ "tokens_seen": 2466512896
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020426078234704113,
+ "loss": 1.3142,
+ "theoretical_loss": 3.4006097545831087,
+ "tokens_seen": 2466578432
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020424473420260783,
+ "loss": 1.2213,
+ "theoretical_loss": 3.4006025886331686,
+ "tokens_seen": 2466643968
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020422868605817453,
+ "loss": 1.2326,
+ "theoretical_loss": 3.400595422926925,
+ "tokens_seen": 2466709504
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020421263791374123,
+ "loss": 1.267,
+ "theoretical_loss": 3.4005882574643636,
+ "tokens_seen": 2466775040
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020419658976930793,
+ "loss": 1.2992,
+ "theoretical_loss": 3.40058109224547,
+ "tokens_seen": 2466840576
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020418054162487463,
+ "loss": 1.1801,
+ "theoretical_loss": 3.4005739272702282,
+ "tokens_seen": 2466906112
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020416449348044133,
+ "loss": 1.1433,
+ "theoretical_loss": 3.400566762538624,
+ "tokens_seen": 2466971648
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020414844533600803,
+ "loss": 1.2958,
+ "theoretical_loss": 3.400559598050643,
+ "tokens_seen": 2467037184
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020413239719157473,
+ "loss": 1.1669,
+ "theoretical_loss": 3.4005524338062707,
+ "tokens_seen": 2467102720
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020411634904714143,
+ "loss": 1.2069,
+ "theoretical_loss": 3.400545269805492,
+ "tokens_seen": 2467168256
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020410030090270813,
+ "loss": 1.2906,
+ "theoretical_loss": 3.400538106048291,
+ "tokens_seen": 2467233792
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020408425275827483,
+ "loss": 1.2204,
+ "theoretical_loss": 3.400530942534655,
+ "tokens_seen": 2467299328
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020406820461384153,
+ "loss": 1.2933,
+ "theoretical_loss": 3.4005237792645677,
+ "tokens_seen": 2467364864
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 825046,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1484394073486328,
+ "objective/train/theoretical_loss": 3.400516616238015,
+ "objective/train/tokens_used": 2487890400,
+ "theoretical_loss": 3.400516616238015,
+ "tokens_seen": 2467430400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020405215646940823,
+ "loss": 1.177,
+ "theoretical_loss": 3.400516616238015,
+ "tokens_seen": 2467430400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020403610832497494,
+ "loss": 1.2294,
+ "theoretical_loss": 3.4005094534549825,
+ "tokens_seen": 2467495936
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020402006018054164,
+ "loss": 1.1848,
+ "theoretical_loss": 3.4005022909154543,
+ "tokens_seen": 2467561472
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020400401203610834,
+ "loss": 1.1985,
+ "theoretical_loss": 3.4004951286194167,
+ "tokens_seen": 2467627008
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020398796389167504,
+ "loss": 1.2679,
+ "theoretical_loss": 3.400487966566854,
+ "tokens_seen": 2467692544
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020397191574724174,
+ "loss": 1.2474,
+ "theoretical_loss": 3.400480804757753,
+ "tokens_seen": 2467758080
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020395586760280844,
+ "loss": 1.2142,
+ "theoretical_loss": 3.4004736431920977,
+ "tokens_seen": 2467823616
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002039398194583751,
+ "loss": 1.2753,
+ "theoretical_loss": 3.4004664818698735,
+ "tokens_seen": 2467889152
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002039237713139418,
+ "loss": 1.1571,
+ "theoretical_loss": 3.400459320791066,
+ "tokens_seen": 2467954688
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0002039077231695085,
+ "loss": 1.2638,
+ "theoretical_loss": 3.4004521599556603,
+ "tokens_seen": 2468020224
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020389167502507521,
+ "loss": 1.2101,
+ "theoretical_loss": 3.4004449993636414,
+ "tokens_seen": 2468085760
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020387562688064191,
+ "loss": 1.2085,
+ "theoretical_loss": 3.400437839014995,
+ "tokens_seen": 2468151296
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020385957873620862,
+ "loss": 1.3225,
+ "theoretical_loss": 3.4004306789097063,
+ "tokens_seen": 2468216832
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020384353059177532,
+ "loss": 1.1514,
+ "theoretical_loss": 3.4004235190477603,
+ "tokens_seen": 2468282368
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020382748244734202,
+ "loss": 1.201,
+ "theoretical_loss": 3.4004163594291423,
+ "tokens_seen": 2468347904
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020381143430290872,
+ "loss": 1.2713,
+ "theoretical_loss": 3.400409200053838,
+ "tokens_seen": 2468413440
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020379538615847542,
+ "loss": 1.3468,
+ "theoretical_loss": 3.4004020409218323,
+ "tokens_seen": 2468478976
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020377933801404212,
+ "loss": 1.2268,
+ "theoretical_loss": 3.400394882033111,
+ "tokens_seen": 2468544512
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020376328986960882,
+ "loss": 1.2785,
+ "theoretical_loss": 3.4003877233876585,
+ "tokens_seen": 2468610048
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020374724172517552,
+ "loss": 1.2412,
+ "theoretical_loss": 3.4003805649854604,
+ "tokens_seen": 2468675584
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020373119358074222,
+ "loss": 1.2188,
+ "theoretical_loss": 3.400373406826502,
+ "tokens_seen": 2468741120
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020371514543630892,
+ "loss": 1.2024,
+ "theoretical_loss": 3.400366248910769,
+ "tokens_seen": 2468806656
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020369909729187562,
+ "loss": 1.2805,
+ "theoretical_loss": 3.4003590912382458,
+ "tokens_seen": 2468872192
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020368304914744232,
+ "loss": 1.2869,
+ "theoretical_loss": 3.4003519338089188,
+ "tokens_seen": 2468937728
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020366700100300902,
+ "loss": 1.2198,
+ "theoretical_loss": 3.4003447766227723,
+ "tokens_seen": 2469003264
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020365095285857572,
+ "loss": 1.1152,
+ "theoretical_loss": 3.4003376196797923,
+ "tokens_seen": 2469068800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020363490471414242,
+ "loss": 1.2242,
+ "theoretical_loss": 3.400330462979963,
+ "tokens_seen": 2469134336
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020361885656970912,
+ "loss": 1.2391,
+ "theoretical_loss": 3.4003233065232714,
+ "tokens_seen": 2469199872
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020360280842527585,
+ "loss": 1.2412,
+ "theoretical_loss": 3.4003161503097017,
+ "tokens_seen": 2469265408
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020358676028084255,
+ "loss": 1.1966,
+ "theoretical_loss": 3.400308994339239,
+ "tokens_seen": 2469330944
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020357071213640925,
+ "loss": 1.2149,
+ "theoretical_loss": 3.400301838611869,
+ "tokens_seen": 2469396480
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020355466399197595,
+ "loss": 1.196,
+ "theoretical_loss": 3.400294683127577,
+ "tokens_seen": 2469462016
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020353861584754266,
+ "loss": 1.129,
+ "theoretical_loss": 3.400287527886348,
+ "tokens_seen": 2469527552
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020352256770310936,
+ "loss": 1.2521,
+ "theoretical_loss": 3.400280372888168,
+ "tokens_seen": 2469593088
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020350651955867606,
+ "loss": 1.2563,
+ "theoretical_loss": 3.400273218133021,
+ "tokens_seen": 2469658624
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020349047141424276,
+ "loss": 1.2738,
+ "theoretical_loss": 3.4002660636208937,
+ "tokens_seen": 2469724160
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020347442326980946,
+ "loss": 1.2064,
+ "theoretical_loss": 3.4002589093517703,
+ "tokens_seen": 2469789696
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020345837512537616,
+ "loss": 1.1881,
+ "theoretical_loss": 3.4002517553256366,
+ "tokens_seen": 2469855232
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020344232698094286,
+ "loss": 1.3745,
+ "theoretical_loss": 3.4002446015424783,
+ "tokens_seen": 2469920768
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020342627883650956,
+ "loss": 1.2314,
+ "theoretical_loss": 3.4002374480022803,
+ "tokens_seen": 2469986304
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020341023069207626,
+ "loss": 1.1338,
+ "theoretical_loss": 3.4002302947050276,
+ "tokens_seen": 2470051840
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020339418254764296,
+ "loss": 1.3465,
+ "theoretical_loss": 3.400223141650706,
+ "tokens_seen": 2470117376
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020337813440320966,
+ "loss": 1.217,
+ "theoretical_loss": 3.4002159888393004,
+ "tokens_seen": 2470182912
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020336208625877636,
+ "loss": 1.2215,
+ "theoretical_loss": 3.4002088362707963,
+ "tokens_seen": 2470248448
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020334603811434306,
+ "loss": 1.2032,
+ "theoretical_loss": 3.4002016839451787,
+ "tokens_seen": 2470313984
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020332998996990976,
+ "loss": 1.1877,
+ "theoretical_loss": 3.400194531862434,
+ "tokens_seen": 2470379520
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020331394182547646,
+ "loss": 1.163,
+ "theoretical_loss": 3.400187380022546,
+ "tokens_seen": 2470445056
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020329789368104314,
+ "loss": 1.348,
+ "theoretical_loss": 3.4001802284255014,
+ "tokens_seen": 2470510592
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020328184553660984,
+ "loss": 1.2937,
+ "theoretical_loss": 3.400173077071284,
+ "tokens_seen": 2470576128
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020326579739217654,
+ "loss": 1.2102,
+ "theoretical_loss": 3.4001659259598807,
+ "tokens_seen": 2470641664
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 826177,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2446273565292358,
+ "objective/train/theoretical_loss": 3.400158775091276,
+ "objective/train/tokens_used": 2491167200,
+ "theoretical_loss": 3.400158775091276,
+ "tokens_seen": 2470707200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020324974924774324,
+ "loss": 1.224,
+ "theoretical_loss": 3.400158775091276,
+ "tokens_seen": 2470707200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020323370110330994,
+ "loss": 1.2143,
+ "theoretical_loss": 3.4001516244654546,
+ "tokens_seen": 2470772736
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020321765295887664,
+ "loss": 1.1827,
+ "theoretical_loss": 3.400144474082403,
+ "tokens_seen": 2470838272
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020320160481444334,
+ "loss": 1.2172,
+ "theoretical_loss": 3.400137323942106,
+ "tokens_seen": 2470903808
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020318555667001004,
+ "loss": 1.2655,
+ "theoretical_loss": 3.4001301740445493,
+ "tokens_seen": 2470969344
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020316950852557674,
+ "loss": 1.2819,
+ "theoretical_loss": 3.4001230243897176,
+ "tokens_seen": 2471034880
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020315346038114344,
+ "loss": 1.2298,
+ "theoretical_loss": 3.400115874977596,
+ "tokens_seen": 2471100416
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020313741223671014,
+ "loss": 1.2408,
+ "theoretical_loss": 3.400108725808171,
+ "tokens_seen": 2471165952
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020312136409227684,
+ "loss": 1.2579,
+ "theoretical_loss": 3.400101576881427,
+ "tokens_seen": 2471231488
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020310531594784355,
+ "loss": 1.1748,
+ "theoretical_loss": 3.4000944281973493,
+ "tokens_seen": 2471297024
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020308926780341025,
+ "loss": 1.2703,
+ "theoretical_loss": 3.400087279755924,
+ "tokens_seen": 2471362560
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020307321965897695,
+ "loss": 1.2316,
+ "theoretical_loss": 3.4000801315571354,
+ "tokens_seen": 2471428096
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020305717151454365,
+ "loss": 1.23,
+ "theoretical_loss": 3.4000729836009698,
+ "tokens_seen": 2471493632
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020304112337011035,
+ "loss": 1.1598,
+ "theoretical_loss": 3.400065835887412,
+ "tokens_seen": 2471559168
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020302507522567705,
+ "loss": 1.1848,
+ "theoretical_loss": 3.400058688416447,
+ "tokens_seen": 2471624704
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020300902708124375,
+ "loss": 1.1619,
+ "theoretical_loss": 3.400051541188061,
+ "tokens_seen": 2471690240
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020299297893681045,
+ "loss": 1.2958,
+ "theoretical_loss": 3.4000443942022387,
+ "tokens_seen": 2471755776
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020297693079237715,
+ "loss": 1.1878,
+ "theoretical_loss": 3.4000372474589655,
+ "tokens_seen": 2471821312
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020296088264794385,
+ "loss": 1.1028,
+ "theoretical_loss": 3.400030100958227,
+ "tokens_seen": 2471886848
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020294483450351055,
+ "loss": 1.1892,
+ "theoretical_loss": 3.4000229547000087,
+ "tokens_seen": 2471952384
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020292878635907725,
+ "loss": 1.2539,
+ "theoretical_loss": 3.4000158086842953,
+ "tokens_seen": 2472017920
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020291273821464393,
+ "loss": 1.2367,
+ "theoretical_loss": 3.400008662911073,
+ "tokens_seen": 2472083456
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020289669007021063,
+ "loss": 1.1866,
+ "theoretical_loss": 3.4000015173803257,
+ "tokens_seen": 2472148992
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020288064192577733,
+ "loss": 1.2055,
+ "theoretical_loss": 3.3999943720920403,
+ "tokens_seen": 2472214528
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020286459378134403,
+ "loss": 1.3016,
+ "theoretical_loss": 3.3999872270462017,
+ "tokens_seen": 2472280064
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020284854563691073,
+ "loss": 1.1554,
+ "theoretical_loss": 3.3999800822427946,
+ "tokens_seen": 2472345600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020283249749247743,
+ "loss": 1.3193,
+ "theoretical_loss": 3.399972937681805,
+ "tokens_seen": 2472411136
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020281644934804413,
+ "loss": 1.2168,
+ "theoretical_loss": 3.399965793363218,
+ "tokens_seen": 2472476672
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020280040120361083,
+ "loss": 1.2385,
+ "theoretical_loss": 3.399958649287019,
+ "tokens_seen": 2472542208
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020278435305917753,
+ "loss": 1.2101,
+ "theoretical_loss": 3.3999515054531937,
+ "tokens_seen": 2472607744
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020276830491474423,
+ "loss": 1.1638,
+ "theoretical_loss": 3.399944361861727,
+ "tokens_seen": 2472673280
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020275225677031093,
+ "loss": 1.2297,
+ "theoretical_loss": 3.399937218512604,
+ "tokens_seen": 2472738816
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020273620862587763,
+ "loss": 1.1639,
+ "theoretical_loss": 3.399930075405811,
+ "tokens_seen": 2472804352
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020272016048144433,
+ "loss": 1.0829,
+ "theoretical_loss": 3.3999229325413323,
+ "tokens_seen": 2472869888
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020270411233701103,
+ "loss": 1.2019,
+ "theoretical_loss": 3.3999157899191537,
+ "tokens_seen": 2472935424
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020268806419257773,
+ "loss": 1.2127,
+ "theoretical_loss": 3.399908647539261,
+ "tokens_seen": 2473000960
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020267201604814444,
+ "loss": 1.2104,
+ "theoretical_loss": 3.3999015054016386,
+ "tokens_seen": 2473066496
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020265596790371114,
+ "loss": 1.0949,
+ "theoretical_loss": 3.399894363506273,
+ "tokens_seen": 2473132032
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020263991975927784,
+ "loss": 1.3268,
+ "theoretical_loss": 3.399887221853149,
+ "tokens_seen": 2473197568
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020262387161484454,
+ "loss": 1.1291,
+ "theoretical_loss": 3.3998800804422515,
+ "tokens_seen": 2473263104
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020260782347041124,
+ "loss": 1.1697,
+ "theoretical_loss": 3.3998729392735667,
+ "tokens_seen": 2473328640
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020259177532597794,
+ "loss": 1.1637,
+ "theoretical_loss": 3.3998657983470792,
+ "tokens_seen": 2473394176
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020257572718154464,
+ "loss": 1.1975,
+ "theoretical_loss": 3.3998586576627754,
+ "tokens_seen": 2473459712
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020255967903711134,
+ "loss": 1.1958,
+ "theoretical_loss": 3.3998515172206396,
+ "tokens_seen": 2473525248
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020254363089267804,
+ "loss": 1.1177,
+ "theoretical_loss": 3.3998443770206577,
+ "tokens_seen": 2473590784
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020252758274824471,
+ "loss": 1.3374,
+ "theoretical_loss": 3.399837237062815,
+ "tokens_seen": 2473656320
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020251153460381141,
+ "loss": 1.2645,
+ "theoretical_loss": 3.3998300973470967,
+ "tokens_seen": 2473721856
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020249548645937812,
+ "loss": 1.299,
+ "theoretical_loss": 3.3998229578734884,
+ "tokens_seen": 2473787392
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020247943831494482,
+ "loss": 1.2866,
+ "theoretical_loss": 3.399815818641976,
+ "tokens_seen": 2473852928
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020246339017051152,
+ "loss": 1.3517,
+ "theoretical_loss": 3.3998086796525433,
+ "tokens_seen": 2473918464
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 827110,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.092146873474121,
+ "objective/train/theoretical_loss": 3.399801540905177,
+ "objective/train/tokens_used": 2494444000,
+ "theoretical_loss": 3.399801540905177,
+ "tokens_seen": 2473984000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020244734202607822,
+ "loss": 1.1631,
+ "theoretical_loss": 3.399801540905177,
+ "tokens_seen": 2473984000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020243129388164492,
+ "loss": 1.3172,
+ "theoretical_loss": 3.3997944023998623,
+ "tokens_seen": 2474049536
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020241524573721162,
+ "loss": 1.2592,
+ "theoretical_loss": 3.399787264136584,
+ "tokens_seen": 2474115072
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020239919759277837,
+ "loss": 1.3061,
+ "theoretical_loss": 3.399780126115328,
+ "tokens_seen": 2474180608
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020238314944834507,
+ "loss": 1.2104,
+ "theoretical_loss": 3.39977298833608,
+ "tokens_seen": 2474246144
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020236710130391177,
+ "loss": 1.3113,
+ "theoretical_loss": 3.399765850798825,
+ "tokens_seen": 2474311680
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020235105315947848,
+ "loss": 1.1805,
+ "theoretical_loss": 3.399758713503548,
+ "tokens_seen": 2474377216
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020233500501504518,
+ "loss": 1.2795,
+ "theoretical_loss": 3.399751576450235,
+ "tokens_seen": 2474442752
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020231895687061188,
+ "loss": 1.1756,
+ "theoretical_loss": 3.3997444396388707,
+ "tokens_seen": 2474508288
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020230290872617858,
+ "loss": 1.277,
+ "theoretical_loss": 3.3997373030694416,
+ "tokens_seen": 2474573824
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020228686058174525,
+ "loss": 1.1854,
+ "theoretical_loss": 3.399730166741932,
+ "tokens_seen": 2474639360
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020227081243731195,
+ "loss": 1.2728,
+ "theoretical_loss": 3.3997230306563275,
+ "tokens_seen": 2474704896
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020225476429287865,
+ "loss": 1.2201,
+ "theoretical_loss": 3.3997158948126143,
+ "tokens_seen": 2474770432
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020223871614844535,
+ "loss": 1.1787,
+ "theoretical_loss": 3.3997087592107764,
+ "tokens_seen": 2474835968
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020222266800401205,
+ "loss": 1.4076,
+ "theoretical_loss": 3.3997016238508007,
+ "tokens_seen": 2474901504
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020220661985957875,
+ "loss": 1.2577,
+ "theoretical_loss": 3.3996944887326714,
+ "tokens_seen": 2474967040
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020219057171514545,
+ "loss": 1.2906,
+ "theoretical_loss": 3.3996873538563745,
+ "tokens_seen": 2475032576
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020217452357071215,
+ "loss": 1.1519,
+ "theoretical_loss": 3.399680219221896,
+ "tokens_seen": 2475098112
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020215847542627886,
+ "loss": 1.2329,
+ "theoretical_loss": 3.3996730848292196,
+ "tokens_seen": 2475163648
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020214242728184556,
+ "loss": 1.2425,
+ "theoretical_loss": 3.399665950678332,
+ "tokens_seen": 2475229184
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020212637913741226,
+ "loss": 1.3295,
+ "theoretical_loss": 3.3996588167692186,
+ "tokens_seen": 2475294720
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020211033099297896,
+ "loss": 1.2554,
+ "theoretical_loss": 3.3996516831018644,
+ "tokens_seen": 2475360256
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020209428284854566,
+ "loss": 1.1474,
+ "theoretical_loss": 3.3996445496762546,
+ "tokens_seen": 2475425792
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020207823470411236,
+ "loss": 1.2736,
+ "theoretical_loss": 3.399637416492375,
+ "tokens_seen": 2475491328
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020206218655967906,
+ "loss": 1.115,
+ "theoretical_loss": 3.399630283550211,
+ "tokens_seen": 2475556864
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020204613841524576,
+ "loss": 1.092,
+ "theoretical_loss": 3.3996231508497483,
+ "tokens_seen": 2475622400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020203009027081246,
+ "loss": 1.2549,
+ "theoretical_loss": 3.3996160183909714,
+ "tokens_seen": 2475687936
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020201404212637916,
+ "loss": 1.2399,
+ "theoretical_loss": 3.399608886173867,
+ "tokens_seen": 2475753472
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020199799398194586,
+ "loss": 1.2441,
+ "theoretical_loss": 3.399601754198419,
+ "tokens_seen": 2475819008
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020198194583751256,
+ "loss": 1.2462,
+ "theoretical_loss": 3.399594622464614,
+ "tokens_seen": 2475884544
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020196589769307926,
+ "loss": 1.1227,
+ "theoretical_loss": 3.399587490972437,
+ "tokens_seen": 2475950080
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020194984954864596,
+ "loss": 1.3241,
+ "theoretical_loss": 3.399580359721873,
+ "tokens_seen": 2476015616
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020193380140421266,
+ "loss": 1.2855,
+ "theoretical_loss": 3.3995732287129083,
+ "tokens_seen": 2476081152
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020191775325977937,
+ "loss": 1.2714,
+ "theoretical_loss": 3.3995660979455278,
+ "tokens_seen": 2476146688
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020190170511534604,
+ "loss": 1.194,
+ "theoretical_loss": 3.3995589674197166,
+ "tokens_seen": 2476212224
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020188565697091274,
+ "loss": 1.3153,
+ "theoretical_loss": 3.399551837135461,
+ "tokens_seen": 2476277760
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020186960882647944,
+ "loss": 1.2652,
+ "theoretical_loss": 3.399544707092746,
+ "tokens_seen": 2476343296
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020185356068204614,
+ "loss": 1.1842,
+ "theoretical_loss": 3.3995375772915564,
+ "tokens_seen": 2476408832
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020183751253761284,
+ "loss": 1.2272,
+ "theoretical_loss": 3.3995304477318786,
+ "tokens_seen": 2476474368
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020182146439317954,
+ "loss": 1.1926,
+ "theoretical_loss": 3.3995233184136975,
+ "tokens_seen": 2476539904
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020180541624874624,
+ "loss": 1.1573,
+ "theoretical_loss": 3.3995161893369987,
+ "tokens_seen": 2476605440
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020178936810431294,
+ "loss": 1.2365,
+ "theoretical_loss": 3.3995090605017677,
+ "tokens_seen": 2476670976
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020177331995987964,
+ "loss": 1.2127,
+ "theoretical_loss": 3.3995019319079898,
+ "tokens_seen": 2476736512
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020175727181544634,
+ "loss": 1.1979,
+ "theoretical_loss": 3.3994948035556503,
+ "tokens_seen": 2476802048
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020174122367101304,
+ "loss": 1.2216,
+ "theoretical_loss": 3.399487675444735,
+ "tokens_seen": 2476867584
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020172517552657975,
+ "loss": 1.2472,
+ "theoretical_loss": 3.3994805475752288,
+ "tokens_seen": 2476933120
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020170912738214645,
+ "loss": 1.2953,
+ "theoretical_loss": 3.3994734199471175,
+ "tokens_seen": 2476998656
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020169307923771315,
+ "loss": 1.2852,
+ "theoretical_loss": 3.399466292560387,
+ "tokens_seen": 2477064192
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020167703109327985,
+ "loss": 1.1975,
+ "theoretical_loss": 3.3994591654150215,
+ "tokens_seen": 2477129728
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020166098294884655,
+ "loss": 1.1297,
+ "theoretical_loss": 3.3994520385110074,
+ "tokens_seen": 2477195264
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 828337,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0806145668029785,
+ "objective/train/theoretical_loss": 3.39944491184833,
+ "objective/train/tokens_used": 2497720800,
+ "theoretical_loss": 3.39944491184833,
+ "tokens_seen": 2477260800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020164493480441325,
+ "loss": 1.1918,
+ "theoretical_loss": 3.39944491184833,
+ "tokens_seen": 2477260800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020162888665997995,
+ "loss": 1.1674,
+ "theoretical_loss": 3.399437785426975,
+ "tokens_seen": 2477326336
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020161283851554665,
+ "loss": 1.1511,
+ "theoretical_loss": 3.399430659246927,
+ "tokens_seen": 2477391872
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020159679037111335,
+ "loss": 1.2589,
+ "theoretical_loss": 3.3994235333081724,
+ "tokens_seen": 2477457408
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020158074222668005,
+ "loss": 1.2159,
+ "theoretical_loss": 3.3994164076106954,
+ "tokens_seen": 2477522944
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020156469408224675,
+ "loss": 1.2536,
+ "theoretical_loss": 3.3994092821544832,
+ "tokens_seen": 2477588480
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020154864593781345,
+ "loss": 1.3516,
+ "theoretical_loss": 3.3994021569395194,
+ "tokens_seen": 2477654016
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020153259779338015,
+ "loss": 1.3962,
+ "theoretical_loss": 3.399395031965791,
+ "tokens_seen": 2477719552
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020151654964894683,
+ "loss": 1.1767,
+ "theoretical_loss": 3.3993879072332827,
+ "tokens_seen": 2477785088
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020150050150451353,
+ "loss": 1.3202,
+ "theoretical_loss": 3.39938078274198,
+ "tokens_seen": 2477850624
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020148445336008023,
+ "loss": 1.183,
+ "theoretical_loss": 3.399373658491868,
+ "tokens_seen": 2477916160
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020146840521564693,
+ "loss": 1.1596,
+ "theoretical_loss": 3.399366534482933,
+ "tokens_seen": 2477981696
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020145235707121363,
+ "loss": 1.2108,
+ "theoretical_loss": 3.39935941071516,
+ "tokens_seen": 2478047232
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020143630892678033,
+ "loss": 1.1791,
+ "theoretical_loss": 3.3993522871885347,
+ "tokens_seen": 2478112768
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020142026078234703,
+ "loss": 1.2114,
+ "theoretical_loss": 3.3993451639030416,
+ "tokens_seen": 2478178304
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020140421263791373,
+ "loss": 1.1557,
+ "theoretical_loss": 3.3993380408586678,
+ "tokens_seen": 2478243840
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020138816449348043,
+ "loss": 1.2622,
+ "theoretical_loss": 3.399330918055397,
+ "tokens_seen": 2478309376
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020137211634904713,
+ "loss": 1.217,
+ "theoretical_loss": 3.399323795493216,
+ "tokens_seen": 2478374912
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020135606820461383,
+ "loss": 1.2067,
+ "theoretical_loss": 3.39931667317211,
+ "tokens_seen": 2478440448
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020134002006018053,
+ "loss": 1.0893,
+ "theoretical_loss": 3.3993095510920637,
+ "tokens_seen": 2478505984
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020132397191574723,
+ "loss": 1.2759,
+ "theoretical_loss": 3.3993024292530634,
+ "tokens_seen": 2478571520
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020130792377131393,
+ "loss": 1.3027,
+ "theoretical_loss": 3.3992953076550942,
+ "tokens_seen": 2478637056
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020129187562688064,
+ "loss": 1.2184,
+ "theoretical_loss": 3.3992881862981417,
+ "tokens_seen": 2478702592
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020127582748244734,
+ "loss": 1.175,
+ "theoretical_loss": 3.3992810651821914,
+ "tokens_seen": 2478768128
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020125977933801404,
+ "loss": 1.2264,
+ "theoretical_loss": 3.399273944307229,
+ "tokens_seen": 2478833664
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020124373119358074,
+ "loss": 1.1346,
+ "theoretical_loss": 3.3992668236732393,
+ "tokens_seen": 2478899200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020122768304914744,
+ "loss": 1.2599,
+ "theoretical_loss": 3.399259703280208,
+ "tokens_seen": 2478964736
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020121163490471414,
+ "loss": 1.3759,
+ "theoretical_loss": 3.399252583128121,
+ "tokens_seen": 2479030272
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020119558676028087,
+ "loss": 1.2659,
+ "theoretical_loss": 3.3992454632169635,
+ "tokens_seen": 2479095808
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020117953861584757,
+ "loss": 1.2509,
+ "theoretical_loss": 3.399238343546721,
+ "tokens_seen": 2479161344
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020116349047141427,
+ "loss": 1.241,
+ "theoretical_loss": 3.3992312241173788,
+ "tokens_seen": 2479226880
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020114744232698097,
+ "loss": 1.3191,
+ "theoretical_loss": 3.3992241049289227,
+ "tokens_seen": 2479292416
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020113139418254767,
+ "loss": 1.2666,
+ "theoretical_loss": 3.399216985981338,
+ "tokens_seen": 2479357952
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020111534603811437,
+ "loss": 1.2927,
+ "theoretical_loss": 3.3992098672746103,
+ "tokens_seen": 2479423488
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020109929789368107,
+ "loss": 1.1192,
+ "theoretical_loss": 3.3992027488087246,
+ "tokens_seen": 2479489024
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020108324974924777,
+ "loss": 1.2918,
+ "theoretical_loss": 3.3991956305836672,
+ "tokens_seen": 2479554560
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020106720160481447,
+ "loss": 1.2369,
+ "theoretical_loss": 3.399188512599423,
+ "tokens_seen": 2479620096
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020105115346038117,
+ "loss": 1.2813,
+ "theoretical_loss": 3.3991813948559777,
+ "tokens_seen": 2479685632
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020103510531594787,
+ "loss": 1.2316,
+ "theoretical_loss": 3.3991742773533167,
+ "tokens_seen": 2479751168
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020101905717151457,
+ "loss": 1.2014,
+ "theoretical_loss": 3.3991671600914257,
+ "tokens_seen": 2479816704
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020100300902708127,
+ "loss": 1.2179,
+ "theoretical_loss": 3.3991600430702897,
+ "tokens_seen": 2479882240
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020098696088264797,
+ "loss": 1.2137,
+ "theoretical_loss": 3.3991529262898954,
+ "tokens_seen": 2479947776
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020097091273821468,
+ "loss": 1.2418,
+ "theoretical_loss": 3.3991458097502263,
+ "tokens_seen": 2480013312
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020095486459378138,
+ "loss": 1.1184,
+ "theoretical_loss": 3.3991386934512695,
+ "tokens_seen": 2480078848
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020093881644934808,
+ "loss": 1.2318,
+ "theoretical_loss": 3.3991315773930104,
+ "tokens_seen": 2480144384
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020092276830491478,
+ "loss": 1.245,
+ "theoretical_loss": 3.3991244615754335,
+ "tokens_seen": 2480209920
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020090672016048148,
+ "loss": 1.321,
+ "theoretical_loss": 3.3991173459985253,
+ "tokens_seen": 2480275456
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020089067201604815,
+ "loss": 1.2953,
+ "theoretical_loss": 3.399110230662271,
+ "tokens_seen": 2480340992
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020087462387161485,
+ "loss": 1.2491,
+ "theoretical_loss": 3.3991031155666556,
+ "tokens_seen": 2480406528
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020085857572718155,
+ "loss": 1.2179,
+ "theoretical_loss": 3.3990960007116655,
+ "tokens_seen": 2480472064
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 829255,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2619050741195679,
+ "objective/train/theoretical_loss": 3.399088886097285,
+ "objective/train/tokens_used": 2500997600,
+ "theoretical_loss": 3.399088886097285,
+ "tokens_seen": 2480537600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020084252758274825,
+ "loss": 1.2409,
+ "theoretical_loss": 3.399088886097285,
+ "tokens_seen": 2480537600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020082647943831495,
+ "loss": 1.1882,
+ "theoretical_loss": 3.399081771723501,
+ "tokens_seen": 2480603136
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020081043129388165,
+ "loss": 1.2541,
+ "theoretical_loss": 3.3990746575902984,
+ "tokens_seen": 2480668672
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020079438314944836,
+ "loss": 1.1639,
+ "theoretical_loss": 3.399067543697662,
+ "tokens_seen": 2480734208
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020077833500501506,
+ "loss": 1.2356,
+ "theoretical_loss": 3.3990604300455787,
+ "tokens_seen": 2480799744
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020076228686058176,
+ "loss": 1.2349,
+ "theoretical_loss": 3.3990533166340327,
+ "tokens_seen": 2480865280
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020074623871614846,
+ "loss": 1.2201,
+ "theoretical_loss": 3.3990462034630102,
+ "tokens_seen": 2480930816
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020073019057171516,
+ "loss": 1.2488,
+ "theoretical_loss": 3.399039090532497,
+ "tokens_seen": 2480996352
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020071414242728186,
+ "loss": 1.2834,
+ "theoretical_loss": 3.399031977842478,
+ "tokens_seen": 2481061888
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020069809428284856,
+ "loss": 1.212,
+ "theoretical_loss": 3.3990248653929385,
+ "tokens_seen": 2481127424
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020068204613841526,
+ "loss": 1.2744,
+ "theoretical_loss": 3.3990177531838652,
+ "tokens_seen": 2481192960
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020066599799398196,
+ "loss": 1.1677,
+ "theoretical_loss": 3.3990106412152423,
+ "tokens_seen": 2481258496
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020064994984954866,
+ "loss": 1.1946,
+ "theoretical_loss": 3.3990035294870564,
+ "tokens_seen": 2481324032
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020063390170511536,
+ "loss": 1.2133,
+ "theoretical_loss": 3.398996417999292,
+ "tokens_seen": 2481389568
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020061785356068206,
+ "loss": 1.1975,
+ "theoretical_loss": 3.3989893067519352,
+ "tokens_seen": 2481455104
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020060180541624876,
+ "loss": 1.1522,
+ "theoretical_loss": 3.3989821957449715,
+ "tokens_seen": 2481520640
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020058575727181546,
+ "loss": 1.2933,
+ "theoretical_loss": 3.3989750849783866,
+ "tokens_seen": 2481586176
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020056970912738216,
+ "loss": 1.2804,
+ "theoretical_loss": 3.398967974452166,
+ "tokens_seen": 2481651712
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020055366098294886,
+ "loss": 1.2739,
+ "theoretical_loss": 3.3989608641662947,
+ "tokens_seen": 2481717248
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020053761283851557,
+ "loss": 1.2873,
+ "theoretical_loss": 3.3989537541207584,
+ "tokens_seen": 2481782784
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020052156469408227,
+ "loss": 1.1431,
+ "theoretical_loss": 3.3989466443155427,
+ "tokens_seen": 2481848320
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020050551654964894,
+ "loss": 1.2132,
+ "theoretical_loss": 3.398939534750634,
+ "tokens_seen": 2481913856
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020048946840521564,
+ "loss": 1.1458,
+ "theoretical_loss": 3.3989324254260165,
+ "tokens_seen": 2481979392
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020047342026078234,
+ "loss": 1.2597,
+ "theoretical_loss": 3.398925316341676,
+ "tokens_seen": 2482044928
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020045737211634904,
+ "loss": 1.2372,
+ "theoretical_loss": 3.398918207497599,
+ "tokens_seen": 2482110464
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020044132397191574,
+ "loss": 1.2187,
+ "theoretical_loss": 3.39891109889377,
+ "tokens_seen": 2482176000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020042527582748244,
+ "loss": 1.1225,
+ "theoretical_loss": 3.398903990530175,
+ "tokens_seen": 2482241536
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020040922768304914,
+ "loss": 1.2305,
+ "theoretical_loss": 3.3988968824067998,
+ "tokens_seen": 2482307072
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020039317953861584,
+ "loss": 1.2513,
+ "theoretical_loss": 3.398889774523629,
+ "tokens_seen": 2482372608
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020037713139418254,
+ "loss": 1.3474,
+ "theoretical_loss": 3.398882666880649,
+ "tokens_seen": 2482438144
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020036108324974925,
+ "loss": 1.227,
+ "theoretical_loss": 3.398875559477845,
+ "tokens_seen": 2482503680
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020034503510531595,
+ "loss": 1.1729,
+ "theoretical_loss": 3.3988684523152024,
+ "tokens_seen": 2482569216
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020032898696088265,
+ "loss": 1.111,
+ "theoretical_loss": 3.3988613453927075,
+ "tokens_seen": 2482634752
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020031293881644935,
+ "loss": 1.1301,
+ "theoretical_loss": 3.398854238710345,
+ "tokens_seen": 2482700288
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020029689067201605,
+ "loss": 1.2035,
+ "theoretical_loss": 3.3988471322681004,
+ "tokens_seen": 2482765824
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020028084252758275,
+ "loss": 1.1641,
+ "theoretical_loss": 3.39884002606596,
+ "tokens_seen": 2482831360
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020026479438314945,
+ "loss": 1.2287,
+ "theoretical_loss": 3.3988329201039087,
+ "tokens_seen": 2482896896
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020024874623871615,
+ "loss": 1.2242,
+ "theoretical_loss": 3.3988258143819325,
+ "tokens_seen": 2482962432
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020023269809428285,
+ "loss": 1.209,
+ "theoretical_loss": 3.398818708900017,
+ "tokens_seen": 2483027968
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020021664994984955,
+ "loss": 1.1486,
+ "theoretical_loss": 3.3988116036581473,
+ "tokens_seen": 2483093504
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020020060180541625,
+ "loss": 1.1233,
+ "theoretical_loss": 3.398804498656309,
+ "tokens_seen": 2483159040
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020018455366098295,
+ "loss": 1.1901,
+ "theoretical_loss": 3.3987973938944878,
+ "tokens_seen": 2483224576
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020016850551654965,
+ "loss": 1.2299,
+ "theoretical_loss": 3.3987902893726694,
+ "tokens_seen": 2483290112
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020015245737211635,
+ "loss": 1.2256,
+ "theoretical_loss": 3.3987831850908394,
+ "tokens_seen": 2483355648
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020013640922768305,
+ "loss": 1.2078,
+ "theoretical_loss": 3.398776081048983,
+ "tokens_seen": 2483421184
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020012036108324973,
+ "loss": 1.1614,
+ "theoretical_loss": 3.398768977247086,
+ "tokens_seen": 2483486720
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020010431293881643,
+ "loss": 1.285,
+ "theoretical_loss": 3.398761873685134,
+ "tokens_seen": 2483552256
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020008826479438313,
+ "loss": 1.3088,
+ "theoretical_loss": 3.3987547703631122,
+ "tokens_seen": 2483617792
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020007221664994983,
+ "loss": 1.2618,
+ "theoretical_loss": 3.398747667281007,
+ "tokens_seen": 2483683328
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020005616850551653,
+ "loss": 1.269,
+ "theoretical_loss": 3.398740564438803,
+ "tokens_seen": 2483748864
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 830439,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1394518613815308,
+ "objective/train/theoretical_loss": 3.3987334618364864,
+ "objective/train/tokens_used": 2504274400,
+ "theoretical_loss": 3.3987334618364864,
+ "tokens_seen": 2483814400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020004012036108323,
+ "loss": 1.201,
+ "theoretical_loss": 3.3987334618364864,
+ "tokens_seen": 2483814400
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020002407221664993,
+ "loss": 1.1851,
+ "theoretical_loss": 3.3987263594740424,
+ "tokens_seen": 2483879936
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00020000802407221663,
+ "loss": 1.2609,
+ "theoretical_loss": 3.398719257351457,
+ "tokens_seen": 2483945472
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019999197592778336,
+ "loss": 1.3455,
+ "theoretical_loss": 3.398712155468715,
+ "tokens_seen": 2484011008
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019997592778335006,
+ "loss": 1.1757,
+ "theoretical_loss": 3.3987050538258026,
+ "tokens_seen": 2484076544
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019995987963891676,
+ "loss": 1.2225,
+ "theoretical_loss": 3.3986979524227054,
+ "tokens_seen": 2484142080
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019994383149448346,
+ "loss": 1.2049,
+ "theoretical_loss": 3.398690851259409,
+ "tokens_seen": 2484207616
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019992778335005016,
+ "loss": 1.3037,
+ "theoretical_loss": 3.3986837503358984,
+ "tokens_seen": 2484273152
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019991173520561686,
+ "loss": 1.2266,
+ "theoretical_loss": 3.3986766496521597,
+ "tokens_seen": 2484338688
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019989568706118356,
+ "loss": 1.237,
+ "theoretical_loss": 3.3986695492081784,
+ "tokens_seen": 2484404224
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019987963891675026,
+ "loss": 1.2933,
+ "theoretical_loss": 3.39866244900394,
+ "tokens_seen": 2484469760
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019986359077231696,
+ "loss": 1.255,
+ "theoretical_loss": 3.3986553490394305,
+ "tokens_seen": 2484535296
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019984754262788367,
+ "loss": 1.2548,
+ "theoretical_loss": 3.398648249314635,
+ "tokens_seen": 2484600832
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019983149448345037,
+ "loss": 1.2898,
+ "theoretical_loss": 3.3986411498295386,
+ "tokens_seen": 2484666368
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019981544633901707,
+ "loss": 1.3061,
+ "theoretical_loss": 3.398634050584128,
+ "tokens_seen": 2484731904
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019979939819458374,
+ "loss": 1.1572,
+ "theoretical_loss": 3.398626951578388,
+ "tokens_seen": 2484797440
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019978335005015044,
+ "loss": 1.4248,
+ "theoretical_loss": 3.3986198528123044,
+ "tokens_seen": 2484862976
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019976730190571714,
+ "loss": 1.2282,
+ "theoretical_loss": 3.398612754285863,
+ "tokens_seen": 2484928512
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019975125376128384,
+ "loss": 1.1543,
+ "theoretical_loss": 3.398605655999049,
+ "tokens_seen": 2484994048
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019973520561685054,
+ "loss": 1.2339,
+ "theoretical_loss": 3.3985985579518485,
+ "tokens_seen": 2485059584
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019971915747241724,
+ "loss": 1.2232,
+ "theoretical_loss": 3.398591460144247,
+ "tokens_seen": 2485125120
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019970310932798394,
+ "loss": 1.113,
+ "theoretical_loss": 3.398584362576229,
+ "tokens_seen": 2485190656
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019968706118355067,
+ "loss": 1.2763,
+ "theoretical_loss": 3.398577265247782,
+ "tokens_seen": 2485256192
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019967101303911737,
+ "loss": 1.2816,
+ "theoretical_loss": 3.39857016815889,
+ "tokens_seen": 2485321728
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019965496489468407,
+ "loss": 1.2113,
+ "theoretical_loss": 3.3985630713095394,
+ "tokens_seen": 2485387264
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019963891675025077,
+ "loss": 1.1842,
+ "theoretical_loss": 3.3985559746997156,
+ "tokens_seen": 2485452800
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019962286860581747,
+ "loss": 1.1257,
+ "theoretical_loss": 3.3985488783294038,
+ "tokens_seen": 2485518336
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019960682046138418,
+ "loss": 1.1704,
+ "theoretical_loss": 3.3985417821985906,
+ "tokens_seen": 2485583872
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019959077231695088,
+ "loss": 1.2482,
+ "theoretical_loss": 3.3985346863072605,
+ "tokens_seen": 2485649408
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019957472417251758,
+ "loss": 1.175,
+ "theoretical_loss": 3.3985275906553998,
+ "tokens_seen": 2485714944
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019955867602808428,
+ "loss": 1.1798,
+ "theoretical_loss": 3.398520495242994,
+ "tokens_seen": 2485780480
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019954262788365098,
+ "loss": 1.4069,
+ "theoretical_loss": 3.3985134000700286,
+ "tokens_seen": 2485846016
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019952657973921768,
+ "loss": 1.1513,
+ "theoretical_loss": 3.398506305136489,
+ "tokens_seen": 2485911552
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019951053159478438,
+ "loss": 1.2008,
+ "theoretical_loss": 3.3984992104423615,
+ "tokens_seen": 2485977088
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019949448345035108,
+ "loss": 1.25,
+ "theoretical_loss": 3.3984921159876307,
+ "tokens_seen": 2486042624
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019947843530591775,
+ "loss": 1.2065,
+ "theoretical_loss": 3.3984850217722826,
+ "tokens_seen": 2486108160
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019946238716148445,
+ "loss": 1.234,
+ "theoretical_loss": 3.3984779277963035,
+ "tokens_seen": 2486173696
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019944633901705115,
+ "loss": 1.2046,
+ "theoretical_loss": 3.3984708340596783,
+ "tokens_seen": 2486239232
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019943029087261786,
+ "loss": 1.1899,
+ "theoretical_loss": 3.3984637405623928,
+ "tokens_seen": 2486304768
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019941424272818456,
+ "loss": 1.2836,
+ "theoretical_loss": 3.3984566473044326,
+ "tokens_seen": 2486370304
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019939819458375126,
+ "loss": 1.3254,
+ "theoretical_loss": 3.398449554285783,
+ "tokens_seen": 2486435840
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019938214643931796,
+ "loss": 1.2892,
+ "theoretical_loss": 3.3984424615064306,
+ "tokens_seen": 2486501376
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019936609829488466,
+ "loss": 1.2191,
+ "theoretical_loss": 3.39843536896636,
+ "tokens_seen": 2486566912
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019935005015045136,
+ "loss": 1.4025,
+ "theoretical_loss": 3.398428276665557,
+ "tokens_seen": 2486632448
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019933400200601806,
+ "loss": 1.2142,
+ "theoretical_loss": 3.3984211846040075,
+ "tokens_seen": 2486697984
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019931795386158476,
+ "loss": 1.1669,
+ "theoretical_loss": 3.398414092781697,
+ "tokens_seen": 2486763520
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019930190571715146,
+ "loss": 1.2416,
+ "theoretical_loss": 3.398407001198611,
+ "tokens_seen": 2486829056
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019928585757271816,
+ "loss": 1.1906,
+ "theoretical_loss": 3.398399909854736,
+ "tokens_seen": 2486894592
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019926980942828486,
+ "loss": 1.2428,
+ "theoretical_loss": 3.3983928187500565,
+ "tokens_seen": 2486960128
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019925376128385156,
+ "loss": 1.3065,
+ "theoretical_loss": 3.398385727884558,
+ "tokens_seen": 2487025664
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 831332,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1451013088226318,
+ "objective/train/theoretical_loss": 3.398378637258227,
+ "objective/train/tokens_used": 2507551200,
+ "theoretical_loss": 3.398378637258227,
+ "tokens_seen": 2487091200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019923771313941826,
+ "loss": 1.2082,
+ "theoretical_loss": 3.398378637258227,
+ "tokens_seen": 2487091200
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019922166499498496,
+ "loss": 1.1934,
+ "theoretical_loss": 3.3983715468710494,
+ "tokens_seen": 2487156736
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019920561685055166,
+ "loss": 1.2297,
+ "theoretical_loss": 3.3983644567230096,
+ "tokens_seen": 2487222272
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019918956870611836,
+ "loss": 1.2851,
+ "theoretical_loss": 3.398357366814094,
+ "tokens_seen": 2487287808
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019917352056168507,
+ "loss": 1.0724,
+ "theoretical_loss": 3.398350277144288,
+ "tokens_seen": 2487353344
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019915747241725177,
+ "loss": 1.2158,
+ "theoretical_loss": 3.3983431877135777,
+ "tokens_seen": 2487418880
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019914142427281847,
+ "loss": 1.2054,
+ "theoretical_loss": 3.398336098521948,
+ "tokens_seen": 2487484416
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019912537612838517,
+ "loss": 1.1671,
+ "theoretical_loss": 3.3983290095693848,
+ "tokens_seen": 2487549952
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019910932798395187,
+ "loss": 1.2764,
+ "theoretical_loss": 3.3983219208558744,
+ "tokens_seen": 2487615488
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019909327983951857,
+ "loss": 1.2087,
+ "theoretical_loss": 3.398314832381401,
+ "tokens_seen": 2487681024
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019907723169508527,
+ "loss": 1.1089,
+ "theoretical_loss": 3.398307744145952,
+ "tokens_seen": 2487746560
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019906118355065197,
+ "loss": 1.2529,
+ "theoretical_loss": 3.398300656149512,
+ "tokens_seen": 2487812096
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019904513540621867,
+ "loss": 1.2395,
+ "theoretical_loss": 3.3982935683920665,
+ "tokens_seen": 2487877632
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019902908726178537,
+ "loss": 1.1737,
+ "theoretical_loss": 3.398286480873602,
+ "tokens_seen": 2487943168
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019901303911735207,
+ "loss": 1.2631,
+ "theoretical_loss": 3.398279393594103,
+ "tokens_seen": 2488008704
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019899699097291877,
+ "loss": 1.2305,
+ "theoretical_loss": 3.3982723065535563,
+ "tokens_seen": 2488074240
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019898094282848547,
+ "loss": 1.2078,
+ "theoretical_loss": 3.3982652197519467,
+ "tokens_seen": 2488139776
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019896489468405217,
+ "loss": 1.2546,
+ "theoretical_loss": 3.3982581331892603,
+ "tokens_seen": 2488205312
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019894884653961887,
+ "loss": 1.2737,
+ "theoretical_loss": 3.3982510468654823,
+ "tokens_seen": 2488270848
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019893279839518557,
+ "loss": 1.2507,
+ "theoretical_loss": 3.398243960780599,
+ "tokens_seen": 2488336384
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019891675025075228,
+ "loss": 1.278,
+ "theoretical_loss": 3.3982368749345957,
+ "tokens_seen": 2488401920
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019890070210631898,
+ "loss": 1.2553,
+ "theoretical_loss": 3.3982297893274582,
+ "tokens_seen": 2488467456
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019888465396188568,
+ "loss": 1.1529,
+ "theoretical_loss": 3.398222703959172,
+ "tokens_seen": 2488532992
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019886860581745238,
+ "loss": 1.1808,
+ "theoretical_loss": 3.3982156188297226,
+ "tokens_seen": 2488598528
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019885255767301908,
+ "loss": 1.1966,
+ "theoretical_loss": 3.398208533939096,
+ "tokens_seen": 2488664064
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019883650952858578,
+ "loss": 1.239,
+ "theoretical_loss": 3.3982014492872774,
+ "tokens_seen": 2488729600
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019882046138415248,
+ "loss": 1.2778,
+ "theoretical_loss": 3.398194364874253,
+ "tokens_seen": 2488795136
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019880441323971918,
+ "loss": 1.1353,
+ "theoretical_loss": 3.398187280700008,
+ "tokens_seen": 2488860672
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019878836509528585,
+ "loss": 1.2361,
+ "theoretical_loss": 3.398180196764529,
+ "tokens_seen": 2488926208
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019877231695085255,
+ "loss": 1.2747,
+ "theoretical_loss": 3.398173113067801,
+ "tokens_seen": 2488991744
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019875626880641925,
+ "loss": 1.1917,
+ "theoretical_loss": 3.3981660296098086,
+ "tokens_seen": 2489057280
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019874022066198596,
+ "loss": 1.1691,
+ "theoretical_loss": 3.3981589463905393,
+ "tokens_seen": 2489122816
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019872417251755266,
+ "loss": 1.3089,
+ "theoretical_loss": 3.398151863409978,
+ "tokens_seen": 2489188352
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019870812437311936,
+ "loss": 1.2664,
+ "theoretical_loss": 3.39814478066811,
+ "tokens_seen": 2489253888
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019869207622868606,
+ "loss": 1.2682,
+ "theoretical_loss": 3.3981376981649216,
+ "tokens_seen": 2489319424
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019867602808425276,
+ "loss": 1.1665,
+ "theoretical_loss": 3.3981306159003983,
+ "tokens_seen": 2489384960
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019865997993981946,
+ "loss": 1.3145,
+ "theoretical_loss": 3.398123533874525,
+ "tokens_seen": 2489450496
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019864393179538616,
+ "loss": 1.2427,
+ "theoretical_loss": 3.398116452087289,
+ "tokens_seen": 2489516032
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019862788365095286,
+ "loss": 1.1563,
+ "theoretical_loss": 3.3981093705386742,
+ "tokens_seen": 2489581568
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019861183550651956,
+ "loss": 1.1608,
+ "theoretical_loss": 3.3981022892286674,
+ "tokens_seen": 2489647104
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019859578736208626,
+ "loss": 1.2877,
+ "theoretical_loss": 3.398095208157254,
+ "tokens_seen": 2489712640
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019857973921765296,
+ "loss": 1.1756,
+ "theoretical_loss": 3.39808812732442,
+ "tokens_seen": 2489778176
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019856369107321966,
+ "loss": 1.2563,
+ "theoretical_loss": 3.3980810467301508,
+ "tokens_seen": 2489843712
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019854764292878636,
+ "loss": 1.2139,
+ "theoretical_loss": 3.3980739663744313,
+ "tokens_seen": 2489909248
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019853159478435306,
+ "loss": 1.1579,
+ "theoretical_loss": 3.398066886257248,
+ "tokens_seen": 2489974784
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019851554663991976,
+ "loss": 1.1793,
+ "theoretical_loss": 3.398059806378587,
+ "tokens_seen": 2490040320
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019849949849548646,
+ "loss": 1.1124,
+ "theoretical_loss": 3.3980527267384337,
+ "tokens_seen": 2490105856
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.0001984834503510532,
+ "loss": 1.311,
+ "theoretical_loss": 3.398045647336773,
+ "tokens_seen": 2490171392
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019846740220661987,
+ "loss": 1.3087,
+ "theoretical_loss": 3.3980385681735914,
+ "tokens_seen": 2490236928
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019845135406218657,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3980314892488743,
+ "tokens_seen": 2490302464
+ },
+ {
+ "epoch": 0.75,
+ "objective/train/docs_used": 832559,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2697808742523193,
+ "objective/train/theoretical_loss": 3.3980244105626074,
+ "objective/train/tokens_used": 2510828000,
+ "theoretical_loss": 3.3980244105626074,
+ "tokens_seen": 2490368000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019843530591775327,
+ "loss": 1.2602,
+ "theoretical_loss": 3.3980244105626074,
+ "tokens_seen": 2490368000
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019841925777331997,
+ "loss": 1.2989,
+ "theoretical_loss": 3.3980173321147764,
+ "tokens_seen": 2490433536
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019840320962888667,
+ "loss": 1.1108,
+ "theoretical_loss": 3.398010253905367,
+ "tokens_seen": 2490499072
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019838716148445337,
+ "loss": 1.324,
+ "theoretical_loss": 3.398003175934365,
+ "tokens_seen": 2490564608
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019837111334002007,
+ "loss": 1.2147,
+ "theoretical_loss": 3.397996098201756,
+ "tokens_seen": 2490630144
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019835506519558677,
+ "loss": 1.1787,
+ "theoretical_loss": 3.3979890207075263,
+ "tokens_seen": 2490695680
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019833901705115347,
+ "loss": 1.265,
+ "theoretical_loss": 3.3979819434516605,
+ "tokens_seen": 2490761216
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019832296890672017,
+ "loss": 1.2064,
+ "theoretical_loss": 3.3979748664341445,
+ "tokens_seen": 2490826752
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019830692076228687,
+ "loss": 1.1862,
+ "theoretical_loss": 3.397967789654965,
+ "tokens_seen": 2490892288
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019829087261785357,
+ "loss": 1.2933,
+ "theoretical_loss": 3.3979607131141067,
+ "tokens_seen": 2490957824
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019827482447342027,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3979536368115557,
+ "tokens_seen": 2491023360
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019825877632898697,
+ "loss": 1.2084,
+ "theoretical_loss": 3.3979465607472976,
+ "tokens_seen": 2491088896
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019824272818455367,
+ "loss": 1.1839,
+ "theoretical_loss": 3.3979394849213183,
+ "tokens_seen": 2491154432
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019822668004012038,
+ "loss": 1.2297,
+ "theoretical_loss": 3.397932409333603,
+ "tokens_seen": 2491219968
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019821063189568708,
+ "loss": 1.3544,
+ "theoretical_loss": 3.397925333984138,
+ "tokens_seen": 2491285504
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019819458375125378,
+ "loss": 1.2595,
+ "theoretical_loss": 3.3979182588729087,
+ "tokens_seen": 2491351040
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019817853560682048,
+ "loss": 1.1311,
+ "theoretical_loss": 3.397911183999901,
+ "tokens_seen": 2491416576
+ },
+ {
+ "epoch": 0.75,
+ "learning_rate": 0.00019816248746238718,
+ "loss": 1.2664,
+ "theoretical_loss": 3.3979041093651006,
+ "tokens_seen": 2491482112
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019814643931795388,
+ "loss": 1.2052,
+ "theoretical_loss": 3.397897034968493,
+ "tokens_seen": 2491547648
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019813039117352058,
+ "loss": 1.2593,
+ "theoretical_loss": 3.3978899608100637,
+ "tokens_seen": 2491613184
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019811434302908728,
+ "loss": 1.2833,
+ "theoretical_loss": 3.3978828868897994,
+ "tokens_seen": 2491678720
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019809829488465398,
+ "loss": 1.2358,
+ "theoretical_loss": 3.3978758132076847,
+ "tokens_seen": 2491744256
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019808224674022065,
+ "loss": 1.2502,
+ "theoretical_loss": 3.397868739763706,
+ "tokens_seen": 2491809792
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019806619859578735,
+ "loss": 1.1732,
+ "theoretical_loss": 3.3978616665578487,
+ "tokens_seen": 2491875328
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019805015045135406,
+ "loss": 1.1261,
+ "theoretical_loss": 3.3978545935900986,
+ "tokens_seen": 2491940864
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019803410230692076,
+ "loss": 1.1845,
+ "theoretical_loss": 3.3978475208604415,
+ "tokens_seen": 2492006400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019801805416248746,
+ "loss": 1.1471,
+ "theoretical_loss": 3.397840448368863,
+ "tokens_seen": 2492071936
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019800200601805416,
+ "loss": 1.2106,
+ "theoretical_loss": 3.397833376115349,
+ "tokens_seen": 2492137472
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019798595787362086,
+ "loss": 1.3668,
+ "theoretical_loss": 3.397826304099885,
+ "tokens_seen": 2492203008
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019796990972918756,
+ "loss": 1.2273,
+ "theoretical_loss": 3.397819232322457,
+ "tokens_seen": 2492268544
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019795386158475426,
+ "loss": 1.2069,
+ "theoretical_loss": 3.3978121607830505,
+ "tokens_seen": 2492334080
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019793781344032096,
+ "loss": 1.2428,
+ "theoretical_loss": 3.3978050894816514,
+ "tokens_seen": 2492399616
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019792176529588766,
+ "loss": 1.2217,
+ "theoretical_loss": 3.3977980184182455,
+ "tokens_seen": 2492465152
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019790571715145436,
+ "loss": 1.2452,
+ "theoretical_loss": 3.397790947592818,
+ "tokens_seen": 2492530688
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978896690070211,
+ "loss": 1.2358,
+ "theoretical_loss": 3.397783877005355,
+ "tokens_seen": 2492596224
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978736208625878,
+ "loss": 1.1849,
+ "theoretical_loss": 3.3977768066558425,
+ "tokens_seen": 2492661760
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978575727181545,
+ "loss": 1.2653,
+ "theoretical_loss": 3.397769736544266,
+ "tokens_seen": 2492727296
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978415245737212,
+ "loss": 1.1974,
+ "theoretical_loss": 3.3977626666706113,
+ "tokens_seen": 2492792832
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978254764292879,
+ "loss": 1.1352,
+ "theoretical_loss": 3.3977555970348634,
+ "tokens_seen": 2492858368
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001978094282848546,
+ "loss": 1.3429,
+ "theoretical_loss": 3.3977485276370096,
+ "tokens_seen": 2492923904
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001977933801404213,
+ "loss": 1.4025,
+ "theoretical_loss": 3.397741458477034,
+ "tokens_seen": 2492989440
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000197777331995988,
+ "loss": 1.208,
+ "theoretical_loss": 3.3977343895549237,
+ "tokens_seen": 2493054976
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019776128385155467,
+ "loss": 1.2541,
+ "theoretical_loss": 3.397727320870663,
+ "tokens_seen": 2493120512
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019774523570712137,
+ "loss": 1.3009,
+ "theoretical_loss": 3.3977202524242394,
+ "tokens_seen": 2493186048
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019772918756268807,
+ "loss": 1.2071,
+ "theoretical_loss": 3.3977131842156374,
+ "tokens_seen": 2493251584
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019771313941825477,
+ "loss": 1.1655,
+ "theoretical_loss": 3.397706116244843,
+ "tokens_seen": 2493317120
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019769709127382147,
+ "loss": 1.1185,
+ "theoretical_loss": 3.397699048511842,
+ "tokens_seen": 2493382656
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019768104312938817,
+ "loss": 1.2865,
+ "theoretical_loss": 3.39769198101662,
+ "tokens_seen": 2493448192
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019766499498495487,
+ "loss": 1.1666,
+ "theoretical_loss": 3.3976849137591634,
+ "tokens_seen": 2493513728
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019764894684052157,
+ "loss": 1.2264,
+ "theoretical_loss": 3.397677846739457,
+ "tokens_seen": 2493579264
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 833727,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1852948665618896,
+ "objective/train/theoretical_loss": 3.397670779957487,
+ "objective/train/tokens_used": 2514104800,
+ "theoretical_loss": 3.397670779957487,
+ "tokens_seen": 2493644800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019763289869608827,
+ "loss": 1.258,
+ "theoretical_loss": 3.397670779957487,
+ "tokens_seen": 2493644800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019761685055165497,
+ "loss": 1.1733,
+ "theoretical_loss": 3.3976637134132393,
+ "tokens_seen": 2493710336
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019760080240722167,
+ "loss": 1.1952,
+ "theoretical_loss": 3.3976566471067,
+ "tokens_seen": 2493775872
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019758475426278837,
+ "loss": 1.2515,
+ "theoretical_loss": 3.3976495810378537,
+ "tokens_seen": 2493841408
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019756870611835507,
+ "loss": 1.2697,
+ "theoretical_loss": 3.397642515206687,
+ "tokens_seen": 2493906944
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019755265797392178,
+ "loss": 1.2078,
+ "theoretical_loss": 3.3976354496131855,
+ "tokens_seen": 2493972480
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019753660982948848,
+ "loss": 1.2375,
+ "theoretical_loss": 3.3976283842573354,
+ "tokens_seen": 2494038016
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019752056168505518,
+ "loss": 1.1955,
+ "theoretical_loss": 3.3976213191391214,
+ "tokens_seen": 2494103552
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019750451354062188,
+ "loss": 1.1719,
+ "theoretical_loss": 3.3976142542585306,
+ "tokens_seen": 2494169088
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019748846539618858,
+ "loss": 1.1389,
+ "theoretical_loss": 3.3976071896155475,
+ "tokens_seen": 2494234624
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019747241725175528,
+ "loss": 1.1905,
+ "theoretical_loss": 3.3976001252101584,
+ "tokens_seen": 2494300160
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019745636910732198,
+ "loss": 1.2328,
+ "theoretical_loss": 3.3975930610423495,
+ "tokens_seen": 2494365696
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019744032096288868,
+ "loss": 1.1742,
+ "theoretical_loss": 3.397585997112106,
+ "tokens_seen": 2494431232
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019742427281845538,
+ "loss": 1.1732,
+ "theoretical_loss": 3.3975789334194135,
+ "tokens_seen": 2494496768
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019740822467402208,
+ "loss": 1.1412,
+ "theoretical_loss": 3.3975718699642585,
+ "tokens_seen": 2494562304
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019739217652958878,
+ "loss": 1.2652,
+ "theoretical_loss": 3.397564806746626,
+ "tokens_seen": 2494627840
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019737612838515545,
+ "loss": 1.19,
+ "theoretical_loss": 3.3975577437665025,
+ "tokens_seen": 2494693376
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019736008024072216,
+ "loss": 1.1678,
+ "theoretical_loss": 3.3975506810238736,
+ "tokens_seen": 2494758912
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019734403209628886,
+ "loss": 1.2406,
+ "theoretical_loss": 3.3975436185187244,
+ "tokens_seen": 2494824448
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019732798395185556,
+ "loss": 1.2577,
+ "theoretical_loss": 3.3975365562510413,
+ "tokens_seen": 2494889984
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019731193580742226,
+ "loss": 1.2974,
+ "theoretical_loss": 3.39752949422081,
+ "tokens_seen": 2494955520
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019729588766298899,
+ "loss": 1.3592,
+ "theoretical_loss": 3.3975224324280164,
+ "tokens_seen": 2495021056
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019727983951855569,
+ "loss": 1.236,
+ "theoretical_loss": 3.3975153708726458,
+ "tokens_seen": 2495086592
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001972637913741224,
+ "loss": 1.2978,
+ "theoretical_loss": 3.3975083095546843,
+ "tokens_seen": 2495152128
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001972477432296891,
+ "loss": 1.185,
+ "theoretical_loss": 3.3975012484741183,
+ "tokens_seen": 2495217664
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001972316950852558,
+ "loss": 1.2044,
+ "theoretical_loss": 3.397494187630932,
+ "tokens_seen": 2495283200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001972156469408225,
+ "loss": 1.201,
+ "theoretical_loss": 3.397487127025113,
+ "tokens_seen": 2495348736
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971995987963892,
+ "loss": 1.2367,
+ "theoretical_loss": 3.397480066656646,
+ "tokens_seen": 2495414272
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971835506519559,
+ "loss": 1.2433,
+ "theoretical_loss": 3.3974730065255168,
+ "tokens_seen": 2495479808
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971675025075226,
+ "loss": 1.142,
+ "theoretical_loss": 3.397465946631711,
+ "tokens_seen": 2495545344
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971514543630893,
+ "loss": 1.1151,
+ "theoretical_loss": 3.397458886975216,
+ "tokens_seen": 2495610880
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000197135406218656,
+ "loss": 1.2363,
+ "theoretical_loss": 3.397451827556015,
+ "tokens_seen": 2495676416
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971193580742227,
+ "loss": 1.2027,
+ "theoretical_loss": 3.397444768374096,
+ "tokens_seen": 2495741952
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001971033099297894,
+ "loss": 1.2011,
+ "theoretical_loss": 3.397437709429444,
+ "tokens_seen": 2495807488
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001970872617853561,
+ "loss": 1.1527,
+ "theoretical_loss": 3.3974306507220446,
+ "tokens_seen": 2495873024
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019707121364092277,
+ "loss": 1.2068,
+ "theoretical_loss": 3.397423592251884,
+ "tokens_seen": 2495938560
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019705516549648947,
+ "loss": 1.249,
+ "theoretical_loss": 3.397416534018947,
+ "tokens_seen": 2496004096
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019703911735205617,
+ "loss": 1.3041,
+ "theoretical_loss": 3.397409476023221,
+ "tokens_seen": 2496069632
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019702306920762287,
+ "loss": 1.1836,
+ "theoretical_loss": 3.397402418264691,
+ "tokens_seen": 2496135168
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019700702106318957,
+ "loss": 1.3214,
+ "theoretical_loss": 3.3973953607433423,
+ "tokens_seen": 2496200704
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019699097291875627,
+ "loss": 1.2023,
+ "theoretical_loss": 3.397388303459161,
+ "tokens_seen": 2496266240
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019697492477432297,
+ "loss": 1.2813,
+ "theoretical_loss": 3.3973812464121336,
+ "tokens_seen": 2496331776
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019695887662988967,
+ "loss": 1.1223,
+ "theoretical_loss": 3.397374189602245,
+ "tokens_seen": 2496397312
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019694282848545637,
+ "loss": 1.0884,
+ "theoretical_loss": 3.3973671330294817,
+ "tokens_seen": 2496462848
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019692678034102307,
+ "loss": 1.1843,
+ "theoretical_loss": 3.397360076693829,
+ "tokens_seen": 2496528384
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019691073219658977,
+ "loss": 1.194,
+ "theoretical_loss": 3.397353020595273,
+ "tokens_seen": 2496593920
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019689468405215647,
+ "loss": 1.2526,
+ "theoretical_loss": 3.397345964733799,
+ "tokens_seen": 2496659456
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019687863590772317,
+ "loss": 1.1783,
+ "theoretical_loss": 3.3973389091093935,
+ "tokens_seen": 2496724992
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019686258776328988,
+ "loss": 1.2051,
+ "theoretical_loss": 3.3973318537220423,
+ "tokens_seen": 2496790528
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019684653961885658,
+ "loss": 1.1191,
+ "theoretical_loss": 3.3973247985717308,
+ "tokens_seen": 2496856064
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 834675,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.094529628753662,
+ "objective/train/theoretical_loss": 3.397317743658445,
+ "objective/train/tokens_used": 2517381600,
+ "theoretical_loss": 3.397317743658445,
+ "tokens_seen": 2496921600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019683049147442328,
+ "loss": 1.1351,
+ "theoretical_loss": 3.397317743658445,
+ "tokens_seen": 2496921600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019681444332998998,
+ "loss": 1.2244,
+ "theoretical_loss": 3.3973106889821705,
+ "tokens_seen": 2496987136
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019679839518555668,
+ "loss": 1.1887,
+ "theoretical_loss": 3.3973036345428933,
+ "tokens_seen": 2497052672
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019678234704112338,
+ "loss": 1.2425,
+ "theoretical_loss": 3.3972965803405994,
+ "tokens_seen": 2497118208
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019676629889669008,
+ "loss": 1.1659,
+ "theoretical_loss": 3.3972895263752743,
+ "tokens_seen": 2497183744
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019675025075225678,
+ "loss": 1.2185,
+ "theoretical_loss": 3.397282472646904,
+ "tokens_seen": 2497249280
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019673420260782348,
+ "loss": 1.2426,
+ "theoretical_loss": 3.3972754191554744,
+ "tokens_seen": 2497314816
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019671815446339018,
+ "loss": 1.2389,
+ "theoretical_loss": 3.3972683659009713,
+ "tokens_seen": 2497380352
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019670210631895688,
+ "loss": 1.163,
+ "theoretical_loss": 3.39726131288338,
+ "tokens_seen": 2497445888
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019668605817452358,
+ "loss": 1.2172,
+ "theoretical_loss": 3.3972542601026867,
+ "tokens_seen": 2497511424
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019667001003009028,
+ "loss": 1.2056,
+ "theoretical_loss": 3.3972472075588778,
+ "tokens_seen": 2497576960
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019665396188565698,
+ "loss": 1.2784,
+ "theoretical_loss": 3.397240155251938,
+ "tokens_seen": 2497642496
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019663791374122368,
+ "loss": 1.1979,
+ "theoretical_loss": 3.397233103181854,
+ "tokens_seen": 2497708032
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019662186559679038,
+ "loss": 1.179,
+ "theoretical_loss": 3.3972260513486114,
+ "tokens_seen": 2497773568
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019660581745235709,
+ "loss": 1.2527,
+ "theoretical_loss": 3.397218999752196,
+ "tokens_seen": 2497839104
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019658976930792379,
+ "loss": 1.2825,
+ "theoretical_loss": 3.3972119483925938,
+ "tokens_seen": 2497904640
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001965737211634905,
+ "loss": 1.1905,
+ "theoretical_loss": 3.3972048972697904,
+ "tokens_seen": 2497970176
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001965576730190572,
+ "loss": 1.1757,
+ "theoretical_loss": 3.3971978463837713,
+ "tokens_seen": 2498035712
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001965416248746239,
+ "loss": 1.2178,
+ "theoretical_loss": 3.3971907957345233,
+ "tokens_seen": 2498101248
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001965255767301906,
+ "loss": 1.3259,
+ "theoretical_loss": 3.3971837453220317,
+ "tokens_seen": 2498166784
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001965095285857573,
+ "loss": 1.363,
+ "theoretical_loss": 3.397176695146282,
+ "tokens_seen": 2498232320
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000196493480441324,
+ "loss": 1.257,
+ "theoretical_loss": 3.39716964520726,
+ "tokens_seen": 2498297856
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001964774322968907,
+ "loss": 1.2151,
+ "theoretical_loss": 3.3971625955049527,
+ "tokens_seen": 2498363392
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001964613841524574,
+ "loss": 1.2527,
+ "theoretical_loss": 3.3971555460393446,
+ "tokens_seen": 2498428928
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001964453360080241,
+ "loss": 1.2298,
+ "theoretical_loss": 3.397148496810422,
+ "tokens_seen": 2498494464
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001964292878635908,
+ "loss": 1.2827,
+ "theoretical_loss": 3.3971414478181714,
+ "tokens_seen": 2498560000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001964132397191575,
+ "loss": 1.26,
+ "theoretical_loss": 3.3971343990625775,
+ "tokens_seen": 2498625536
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001963971915747242,
+ "loss": 1.1719,
+ "theoretical_loss": 3.3971273505436272,
+ "tokens_seen": 2498691072
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001963811434302909,
+ "loss": 1.2759,
+ "theoretical_loss": 3.3971203022613055,
+ "tokens_seen": 2498756608
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019636509528585757,
+ "loss": 1.3374,
+ "theoretical_loss": 3.397113254215599,
+ "tokens_seen": 2498822144
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019634904714142427,
+ "loss": 1.1912,
+ "theoretical_loss": 3.3971062064064927,
+ "tokens_seen": 2498887680
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019633299899699097,
+ "loss": 1.118,
+ "theoretical_loss": 3.397099158833973,
+ "tokens_seen": 2498953216
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019631695085255767,
+ "loss": 1.2384,
+ "theoretical_loss": 3.3970921114980257,
+ "tokens_seen": 2499018752
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019630090270812437,
+ "loss": 1.2525,
+ "theoretical_loss": 3.3970850643986368,
+ "tokens_seen": 2499084288
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019628485456369107,
+ "loss": 1.24,
+ "theoretical_loss": 3.397078017535792,
+ "tokens_seen": 2499149824
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019626880641925777,
+ "loss": 1.2153,
+ "theoretical_loss": 3.397070970909477,
+ "tokens_seen": 2499215360
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019625275827482447,
+ "loss": 1.213,
+ "theoretical_loss": 3.397063924519678,
+ "tokens_seen": 2499280896
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019623671013039117,
+ "loss": 1.2189,
+ "theoretical_loss": 3.3970568783663806,
+ "tokens_seen": 2499346432
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019622066198595787,
+ "loss": 1.1459,
+ "theoretical_loss": 3.3970498324495706,
+ "tokens_seen": 2499411968
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019620461384152457,
+ "loss": 1.2174,
+ "theoretical_loss": 3.397042786769234,
+ "tokens_seen": 2499477504
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019618856569709127,
+ "loss": 1.1497,
+ "theoretical_loss": 3.397035741325357,
+ "tokens_seen": 2499543040
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019617251755265798,
+ "loss": 1.1609,
+ "theoretical_loss": 3.397028696117925,
+ "tokens_seen": 2499608576
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019615646940822468,
+ "loss": 1.2077,
+ "theoretical_loss": 3.3970216511469236,
+ "tokens_seen": 2499674112
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019614042126379138,
+ "loss": 1.171,
+ "theoretical_loss": 3.3970146064123394,
+ "tokens_seen": 2499739648
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019612437311935808,
+ "loss": 1.1714,
+ "theoretical_loss": 3.3970075619141578,
+ "tokens_seen": 2499805184
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019610832497492478,
+ "loss": 1.1589,
+ "theoretical_loss": 3.3970005176523648,
+ "tokens_seen": 2499870720
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001960922768304915,
+ "loss": 1.2444,
+ "theoretical_loss": 3.3969934736269463,
+ "tokens_seen": 2499936256
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001960762286860582,
+ "loss": 1.2292,
+ "theoretical_loss": 3.396986429837888,
+ "tokens_seen": 2500001792
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001960601805416249,
+ "loss": 1.3088,
+ "theoretical_loss": 3.396979386285176,
+ "tokens_seen": 2500067328
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019604413239719158,
+ "loss": 1.1654,
+ "theoretical_loss": 3.396972342968796,
+ "tokens_seen": 2500132864
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 835943,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2011704444885254,
+ "objective/train/theoretical_loss": 3.396965299888734,
+ "objective/train/tokens_used": 2520658400,
+ "theoretical_loss": 3.396965299888734,
+ "tokens_seen": 2500198400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019602808425275828,
+ "loss": 1.1695,
+ "theoretical_loss": 3.396965299888734,
+ "tokens_seen": 2500198400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019601203610832498,
+ "loss": 1.1229,
+ "theoretical_loss": 3.396958257044976,
+ "tokens_seen": 2500263936
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019599598796389168,
+ "loss": 1.2418,
+ "theoretical_loss": 3.3969512144375074,
+ "tokens_seen": 2500329472
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019597993981945838,
+ "loss": 1.1921,
+ "theoretical_loss": 3.3969441720663145,
+ "tokens_seen": 2500395008
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019596389167502508,
+ "loss": 1.2576,
+ "theoretical_loss": 3.3969371299313833,
+ "tokens_seen": 2500460544
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019594784353059178,
+ "loss": 1.2226,
+ "theoretical_loss": 3.3969300880326996,
+ "tokens_seen": 2500526080
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019593179538615848,
+ "loss": 1.3133,
+ "theoretical_loss": 3.3969230463702487,
+ "tokens_seen": 2500591616
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019591574724172519,
+ "loss": 1.2329,
+ "theoretical_loss": 3.396916004944017,
+ "tokens_seen": 2500657152
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019589969909729189,
+ "loss": 1.1005,
+ "theoretical_loss": 3.39690896375399,
+ "tokens_seen": 2500722688
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001958836509528586,
+ "loss": 1.3547,
+ "theoretical_loss": 3.3969019228001547,
+ "tokens_seen": 2500788224
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001958676028084253,
+ "loss": 1.3067,
+ "theoretical_loss": 3.3968948820824956,
+ "tokens_seen": 2500853760
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000195851554663992,
+ "loss": 1.2653,
+ "theoretical_loss": 3.396887841600999,
+ "tokens_seen": 2500919296
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001958355065195587,
+ "loss": 1.2943,
+ "theoretical_loss": 3.3968808013556515,
+ "tokens_seen": 2500984832
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001958194583751254,
+ "loss": 1.3889,
+ "theoretical_loss": 3.3968737613464386,
+ "tokens_seen": 2501050368
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001958034102306921,
+ "loss": 1.1525,
+ "theoretical_loss": 3.3968667215733452,
+ "tokens_seen": 2501115904
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957873620862588,
+ "loss": 1.3904,
+ "theoretical_loss": 3.396859682036359,
+ "tokens_seen": 2501181440
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957713139418255,
+ "loss": 1.1764,
+ "theoretical_loss": 3.396852642735464,
+ "tokens_seen": 2501246976
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957552657973922,
+ "loss": 1.2492,
+ "theoretical_loss": 3.3968456036706476,
+ "tokens_seen": 2501312512
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957392176529589,
+ "loss": 1.342,
+ "theoretical_loss": 3.396838564841895,
+ "tokens_seen": 2501378048
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957231695085256,
+ "loss": 1.2659,
+ "theoretical_loss": 3.3968315262491924,
+ "tokens_seen": 2501443584
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001957071213640923,
+ "loss": 1.2222,
+ "theoretical_loss": 3.396824487892525,
+ "tokens_seen": 2501509120
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000195691073219659,
+ "loss": 1.245,
+ "theoretical_loss": 3.3968174497718797,
+ "tokens_seen": 2501574656
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001956750250752257,
+ "loss": 1.2326,
+ "theoretical_loss": 3.396810411887242,
+ "tokens_seen": 2501640192
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019565897693079237,
+ "loss": 1.1844,
+ "theoretical_loss": 3.3968033742385972,
+ "tokens_seen": 2501705728
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019564292878635907,
+ "loss": 1.1537,
+ "theoretical_loss": 3.396796336825932,
+ "tokens_seen": 2501771264
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019562688064192577,
+ "loss": 1.23,
+ "theoretical_loss": 3.396789299649232,
+ "tokens_seen": 2501836800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019561083249749247,
+ "loss": 1.1277,
+ "theoretical_loss": 3.3967822627084834,
+ "tokens_seen": 2501902336
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019559478435305917,
+ "loss": 1.2505,
+ "theoretical_loss": 3.3967752260036717,
+ "tokens_seen": 2501967872
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019557873620862587,
+ "loss": 1.2875,
+ "theoretical_loss": 3.396768189534783,
+ "tokens_seen": 2502033408
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019556268806419257,
+ "loss": 1.1977,
+ "theoretical_loss": 3.3967611533018034,
+ "tokens_seen": 2502098944
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019554663991975927,
+ "loss": 1.1657,
+ "theoretical_loss": 3.3967541173047184,
+ "tokens_seen": 2502164480
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019553059177532597,
+ "loss": 1.1731,
+ "theoretical_loss": 3.3967470815435137,
+ "tokens_seen": 2502230016
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019551454363089267,
+ "loss": 1.2082,
+ "theoretical_loss": 3.3967400460181763,
+ "tokens_seen": 2502295552
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019549849548645937,
+ "loss": 1.3192,
+ "theoretical_loss": 3.396733010728691,
+ "tokens_seen": 2502361088
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954824473420261,
+ "loss": 1.1778,
+ "theoretical_loss": 3.3967259756750443,
+ "tokens_seen": 2502426624
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954663991975928,
+ "loss": 1.2459,
+ "theoretical_loss": 3.3967189408572223,
+ "tokens_seen": 2502492160
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954503510531595,
+ "loss": 1.1783,
+ "theoretical_loss": 3.39671190627521,
+ "tokens_seen": 2502557696
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954343029087262,
+ "loss": 1.2286,
+ "theoretical_loss": 3.396704871928994,
+ "tokens_seen": 2502623232
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954182547642929,
+ "loss": 1.3401,
+ "theoretical_loss": 3.3966978378185604,
+ "tokens_seen": 2502688768
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001954022066198596,
+ "loss": 1.2457,
+ "theoretical_loss": 3.3966908039438946,
+ "tokens_seen": 2502754304
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001953861584754263,
+ "loss": 1.139,
+ "theoretical_loss": 3.396683770304983,
+ "tokens_seen": 2502819840
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000195370110330993,
+ "loss": 1.2442,
+ "theoretical_loss": 3.396676736901811,
+ "tokens_seen": 2502885376
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001953540621865597,
+ "loss": 1.2214,
+ "theoretical_loss": 3.396669703734365,
+ "tokens_seen": 2502950912
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019533801404212638,
+ "loss": 1.3004,
+ "theoretical_loss": 3.3966626708026313,
+ "tokens_seen": 2503016448
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019532196589769308,
+ "loss": 1.2304,
+ "theoretical_loss": 3.3966556381065947,
+ "tokens_seen": 2503081984
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019530591775325978,
+ "loss": 1.2286,
+ "theoretical_loss": 3.3966486056462415,
+ "tokens_seen": 2503147520
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019528986960882648,
+ "loss": 1.1371,
+ "theoretical_loss": 3.396641573421558,
+ "tokens_seen": 2503213056
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019527382146439318,
+ "loss": 1.3062,
+ "theoretical_loss": 3.3966345414325305,
+ "tokens_seen": 2503278592
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019525777331995988,
+ "loss": 1.178,
+ "theoretical_loss": 3.396627509679144,
+ "tokens_seen": 2503344128
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019524172517552659,
+ "loss": 1.2415,
+ "theoretical_loss": 3.3966204781613847,
+ "tokens_seen": 2503409664
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 836801,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1851744651794434,
+ "objective/train/theoretical_loss": 3.396613446879239,
+ "objective/train/tokens_used": 2523935200,
+ "theoretical_loss": 3.396613446879239,
+ "tokens_seen": 2503475200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019522567703109329,
+ "loss": 1.1437,
+ "theoretical_loss": 3.396613446879239,
+ "tokens_seen": 2503475200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019520962888665999,
+ "loss": 1.1687,
+ "theoretical_loss": 3.3966064158326925,
+ "tokens_seen": 2503540736
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951935807422267,
+ "loss": 1.1268,
+ "theoretical_loss": 3.396599385021731,
+ "tokens_seen": 2503606272
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951775325977934,
+ "loss": 1.3568,
+ "theoretical_loss": 3.3965923544463408,
+ "tokens_seen": 2503671808
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951614844533601,
+ "loss": 1.2383,
+ "theoretical_loss": 3.3965853241065074,
+ "tokens_seen": 2503737344
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951454363089268,
+ "loss": 1.1945,
+ "theoretical_loss": 3.396578294002217,
+ "tokens_seen": 2503802880
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951293881644935,
+ "loss": 1.2047,
+ "theoretical_loss": 3.396571264133456,
+ "tokens_seen": 2503868416
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001951133400200602,
+ "loss": 1.2576,
+ "theoretical_loss": 3.396564234500209,
+ "tokens_seen": 2503933952
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950972918756269,
+ "loss": 1.2785,
+ "theoretical_loss": 3.3965572051024635,
+ "tokens_seen": 2503999488
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950812437311936,
+ "loss": 1.3584,
+ "theoretical_loss": 3.3965501759402046,
+ "tokens_seen": 2504065024
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950651955867603,
+ "loss": 1.264,
+ "theoretical_loss": 3.3965431470134186,
+ "tokens_seen": 2504130560
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000195049147442327,
+ "loss": 1.2903,
+ "theoretical_loss": 3.396536118322091,
+ "tokens_seen": 2504196096
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950330992978937,
+ "loss": 1.2513,
+ "theoretical_loss": 3.396529089866208,
+ "tokens_seen": 2504261632
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950170511534604,
+ "loss": 1.1821,
+ "theoretical_loss": 3.3965220616457557,
+ "tokens_seen": 2504327168
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001950010030090271,
+ "loss": 1.1958,
+ "theoretical_loss": 3.3965150336607195,
+ "tokens_seen": 2504392704
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001949849548645938,
+ "loss": 1.145,
+ "theoretical_loss": 3.3965080059110866,
+ "tokens_seen": 2504458240
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001949689067201605,
+ "loss": 1.3016,
+ "theoretical_loss": 3.3965009783968414,
+ "tokens_seen": 2504523776
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019495285857572717,
+ "loss": 1.1432,
+ "theoretical_loss": 3.396493951117971,
+ "tokens_seen": 2504589312
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019493681043129387,
+ "loss": 1.1899,
+ "theoretical_loss": 3.396486924074461,
+ "tokens_seen": 2504654848
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019492076228686057,
+ "loss": 1.2631,
+ "theoretical_loss": 3.396479897266297,
+ "tokens_seen": 2504720384
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019490471414242727,
+ "loss": 1.2164,
+ "theoretical_loss": 3.3964728706934655,
+ "tokens_seen": 2504785920
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000194888665997994,
+ "loss": 1.2142,
+ "theoretical_loss": 3.396465844355952,
+ "tokens_seen": 2504851456
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001948726178535607,
+ "loss": 1.1886,
+ "theoretical_loss": 3.396458818253743,
+ "tokens_seen": 2504916992
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001948565697091274,
+ "loss": 1.2463,
+ "theoretical_loss": 3.3964517923868236,
+ "tokens_seen": 2504982528
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001948405215646941,
+ "loss": 1.2308,
+ "theoretical_loss": 3.396444766755181,
+ "tokens_seen": 2505048064
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001948244734202608,
+ "loss": 1.279,
+ "theoretical_loss": 3.3964377413588,
+ "tokens_seen": 2505113600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001948084252758275,
+ "loss": 1.2787,
+ "theoretical_loss": 3.396430716197667,
+ "tokens_seen": 2505179136
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001947923771313942,
+ "loss": 1.3077,
+ "theoretical_loss": 3.3964236912717682,
+ "tokens_seen": 2505244672
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001947763289869609,
+ "loss": 1.1953,
+ "theoretical_loss": 3.3964166665810898,
+ "tokens_seen": 2505310208
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001947602808425276,
+ "loss": 1.2642,
+ "theoretical_loss": 3.396409642125617,
+ "tokens_seen": 2505375744
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001947442326980943,
+ "loss": 1.2242,
+ "theoretical_loss": 3.396402617905336,
+ "tokens_seen": 2505441280
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000194728184553661,
+ "loss": 1.1512,
+ "theoretical_loss": 3.396395593920233,
+ "tokens_seen": 2505506816
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001947121364092277,
+ "loss": 1.2333,
+ "theoretical_loss": 3.396388570170294,
+ "tokens_seen": 2505572352
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001946960882647944,
+ "loss": 1.1901,
+ "theoretical_loss": 3.3963815466555047,
+ "tokens_seen": 2505637888
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001946800401203611,
+ "loss": 1.3146,
+ "theoretical_loss": 3.3963745233758513,
+ "tokens_seen": 2505703424
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001946639919759278,
+ "loss": 1.1958,
+ "theoretical_loss": 3.3963675003313196,
+ "tokens_seen": 2505768960
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019464794383149448,
+ "loss": 1.1709,
+ "theoretical_loss": 3.396360477521896,
+ "tokens_seen": 2505834496
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019463189568706118,
+ "loss": 1.1571,
+ "theoretical_loss": 3.396353454947566,
+ "tokens_seen": 2505900032
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019461584754262788,
+ "loss": 1.147,
+ "theoretical_loss": 3.3963464326083157,
+ "tokens_seen": 2505965568
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019459979939819458,
+ "loss": 1.2497,
+ "theoretical_loss": 3.3963394105041314,
+ "tokens_seen": 2506031104
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019458375125376128,
+ "loss": 1.3761,
+ "theoretical_loss": 3.396332388634998,
+ "tokens_seen": 2506096640
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019456770310932798,
+ "loss": 1.1774,
+ "theoretical_loss": 3.3963253670009035,
+ "tokens_seen": 2506162176
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019455165496489469,
+ "loss": 1.2169,
+ "theoretical_loss": 3.396318345601832,
+ "tokens_seen": 2506227712
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019453560682046139,
+ "loss": 1.2107,
+ "theoretical_loss": 3.3963113244377703,
+ "tokens_seen": 2506293248
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001945195586760281,
+ "loss": 1.3085,
+ "theoretical_loss": 3.396304303508704,
+ "tokens_seen": 2506358784
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001945035105315948,
+ "loss": 1.1103,
+ "theoretical_loss": 3.39629728281462,
+ "tokens_seen": 2506424320
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001944874623871615,
+ "loss": 1.1505,
+ "theoretical_loss": 3.396290262355503,
+ "tokens_seen": 2506489856
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001944714142427282,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3962832421313403,
+ "tokens_seen": 2506555392
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001944553660982949,
+ "loss": 1.2668,
+ "theoretical_loss": 3.3962762221421166,
+ "tokens_seen": 2506620928
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001944393179538616,
+ "loss": 1.2191,
+ "theoretical_loss": 3.396269202387819,
+ "tokens_seen": 2506686464
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 837983,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3862534761428833,
+ "objective/train/theoretical_loss": 3.396262182868433,
+ "objective/train/tokens_used": 2527212000,
+ "theoretical_loss": 3.396262182868433,
+ "tokens_seen": 2506752000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001944232698094283,
+ "loss": 1.3293,
+ "theoretical_loss": 3.396262182868433,
+ "tokens_seen": 2506752000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000194407221664995,
+ "loss": 1.1991,
+ "theoretical_loss": 3.3962551635839446,
+ "tokens_seen": 2506817536
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943911735205617,
+ "loss": 1.2173,
+ "theoretical_loss": 3.3962481445343395,
+ "tokens_seen": 2506883072
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943751253761284,
+ "loss": 1.1538,
+ "theoretical_loss": 3.3962411257196043,
+ "tokens_seen": 2506948608
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943590772316951,
+ "loss": 1.1887,
+ "theoretical_loss": 3.396234107139725,
+ "tokens_seen": 2507014144
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943430290872618,
+ "loss": 1.2427,
+ "theoretical_loss": 3.396227088794687,
+ "tokens_seen": 2507079680
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943269809428285,
+ "loss": 1.1937,
+ "theoretical_loss": 3.3962200706844765,
+ "tokens_seen": 2507145216
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001943109327983952,
+ "loss": 1.2303,
+ "theoretical_loss": 3.3962130528090797,
+ "tokens_seen": 2507210752
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001942948846539619,
+ "loss": 1.3851,
+ "theoretical_loss": 3.3962060351684826,
+ "tokens_seen": 2507276288
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001942788365095286,
+ "loss": 1.1938,
+ "theoretical_loss": 3.3961990177626715,
+ "tokens_seen": 2507341824
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001942627883650953,
+ "loss": 1.2786,
+ "theoretical_loss": 3.3961920005916317,
+ "tokens_seen": 2507407360
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000194246740220662,
+ "loss": 1.2737,
+ "theoretical_loss": 3.39618498365535,
+ "tokens_seen": 2507472896
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001942306920762287,
+ "loss": 1.2037,
+ "theoretical_loss": 3.3961779669538115,
+ "tokens_seen": 2507538432
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001942146439317954,
+ "loss": 1.1921,
+ "theoretical_loss": 3.3961709504870026,
+ "tokens_seen": 2507603968
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941985957873621,
+ "loss": 1.2391,
+ "theoretical_loss": 3.39616393425491,
+ "tokens_seen": 2507669504
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941825476429288,
+ "loss": 1.2024,
+ "theoretical_loss": 3.396156918257519,
+ "tokens_seen": 2507735040
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941664994984955,
+ "loss": 1.3097,
+ "theoretical_loss": 3.396149902494815,
+ "tokens_seen": 2507800576
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941504513540622,
+ "loss": 1.2206,
+ "theoretical_loss": 3.3961428869667856,
+ "tokens_seen": 2507866112
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941344032096289,
+ "loss": 1.1699,
+ "theoretical_loss": 3.396135871673416,
+ "tokens_seen": 2507931648
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941183550651956,
+ "loss": 1.3081,
+ "theoretical_loss": 3.3961288566146917,
+ "tokens_seen": 2507997184
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001941023069207623,
+ "loss": 1.2172,
+ "theoretical_loss": 3.3961218417905994,
+ "tokens_seen": 2508062720
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000194086258776329,
+ "loss": 1.161,
+ "theoretical_loss": 3.396114827201125,
+ "tokens_seen": 2508128256
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001940702106318957,
+ "loss": 1.2611,
+ "theoretical_loss": 3.396107812846254,
+ "tokens_seen": 2508193792
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001940541624874624,
+ "loss": 1.1761,
+ "theoretical_loss": 3.3961007987259735,
+ "tokens_seen": 2508259328
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001940381143430291,
+ "loss": 1.0773,
+ "theoretical_loss": 3.396093784840269,
+ "tokens_seen": 2508324864
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001940220661985958,
+ "loss": 1.1955,
+ "theoretical_loss": 3.396086771189126,
+ "tokens_seen": 2508390400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001940060180541625,
+ "loss": 1.2193,
+ "theoretical_loss": 3.396079757772531,
+ "tokens_seen": 2508455936
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001939899699097292,
+ "loss": 1.1528,
+ "theoretical_loss": 3.39607274459047,
+ "tokens_seen": 2508521472
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001939739217652959,
+ "loss": 1.1683,
+ "theoretical_loss": 3.396065731642929,
+ "tokens_seen": 2508587008
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001939578736208626,
+ "loss": 1.2637,
+ "theoretical_loss": 3.3960587189298943,
+ "tokens_seen": 2508652544
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019394182547642928,
+ "loss": 1.1625,
+ "theoretical_loss": 3.3960517064513516,
+ "tokens_seen": 2508718080
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019392577733199598,
+ "loss": 1.2908,
+ "theoretical_loss": 3.396044694207287,
+ "tokens_seen": 2508783616
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019390972918756268,
+ "loss": 1.2296,
+ "theoretical_loss": 3.3960376821976865,
+ "tokens_seen": 2508849152
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019389368104312938,
+ "loss": 1.2768,
+ "theoretical_loss": 3.3960306704225363,
+ "tokens_seen": 2508914688
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019387763289869608,
+ "loss": 1.2666,
+ "theoretical_loss": 3.3960236588818224,
+ "tokens_seen": 2508980224
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019386158475426279,
+ "loss": 1.1791,
+ "theoretical_loss": 3.3960166475755305,
+ "tokens_seen": 2509045760
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019384553660982949,
+ "loss": 1.2245,
+ "theoretical_loss": 3.396009636503647,
+ "tokens_seen": 2509111296
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001938294884653962,
+ "loss": 1.2237,
+ "theoretical_loss": 3.3960026256661577,
+ "tokens_seen": 2509176832
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001938134403209629,
+ "loss": 1.1963,
+ "theoretical_loss": 3.395995615063049,
+ "tokens_seen": 2509242368
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937973921765296,
+ "loss": 1.2684,
+ "theoretical_loss": 3.3959886046943066,
+ "tokens_seen": 2509307904
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937813440320963,
+ "loss": 1.2072,
+ "theoretical_loss": 3.3959815945599168,
+ "tokens_seen": 2509373440
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000193765295887663,
+ "loss": 1.2022,
+ "theoretical_loss": 3.3959745846598657,
+ "tokens_seen": 2509438976
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937492477432297,
+ "loss": 1.2486,
+ "theoretical_loss": 3.395967574994139,
+ "tokens_seen": 2509504512
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937331995987964,
+ "loss": 1.1792,
+ "theoretical_loss": 3.395960565562723,
+ "tokens_seen": 2509570048
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937171514543631,
+ "loss": 1.2311,
+ "theoretical_loss": 3.3959535563656034,
+ "tokens_seen": 2509635584
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001937011033099298,
+ "loss": 1.2832,
+ "theoretical_loss": 3.3959465474027666,
+ "tokens_seen": 2509701120
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019368505516549652,
+ "loss": 1.1687,
+ "theoretical_loss": 3.395939538674199,
+ "tokens_seen": 2509766656
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019366900702106322,
+ "loss": 1.2921,
+ "theoretical_loss": 3.3959325301798855,
+ "tokens_seen": 2509832192
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019365295887662992,
+ "loss": 1.2443,
+ "theoretical_loss": 3.3959255219198132,
+ "tokens_seen": 2509897728
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019363691073219662,
+ "loss": 1.1912,
+ "theoretical_loss": 3.3959185138939683,
+ "tokens_seen": 2509963264
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 838918,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.175501823425293,
+ "objective/train/theoretical_loss": 3.395911506102336,
+ "objective/train/tokens_used": 2530488800,
+ "theoretical_loss": 3.395911506102336,
+ "tokens_seen": 2510028800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001936208625877633,
+ "loss": 1.205,
+ "theoretical_loss": 3.395911506102336,
+ "tokens_seen": 2510028800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019360481444333,
+ "loss": 1.3494,
+ "theoretical_loss": 3.3959044985449025,
+ "tokens_seen": 2510094336
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935887662988967,
+ "loss": 1.2559,
+ "theoretical_loss": 3.395897491221654,
+ "tokens_seen": 2510159872
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935727181544634,
+ "loss": 1.216,
+ "theoretical_loss": 3.395890484132577,
+ "tokens_seen": 2510225408
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935566700100301,
+ "loss": 1.323,
+ "theoretical_loss": 3.3958834772776574,
+ "tokens_seen": 2510290944
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935406218655968,
+ "loss": 1.1334,
+ "theoretical_loss": 3.395876470656881,
+ "tokens_seen": 2510356480
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935245737211635,
+ "loss": 1.224,
+ "theoretical_loss": 3.3958694642702336,
+ "tokens_seen": 2510422016
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001935085255767302,
+ "loss": 1.2977,
+ "theoretical_loss": 3.3958624581177017,
+ "tokens_seen": 2510487552
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001934924774322969,
+ "loss": 1.2504,
+ "theoretical_loss": 3.3958554521992714,
+ "tokens_seen": 2510553088
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001934764292878636,
+ "loss": 1.3186,
+ "theoretical_loss": 3.395848446514929,
+ "tokens_seen": 2510618624
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001934603811434303,
+ "loss": 1.2696,
+ "theoretical_loss": 3.39584144106466,
+ "tokens_seen": 2510684160
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000193444332998997,
+ "loss": 1.2094,
+ "theoretical_loss": 3.3958344358484505,
+ "tokens_seen": 2510749696
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001934282848545637,
+ "loss": 1.2751,
+ "theoretical_loss": 3.395827430866287,
+ "tokens_seen": 2510815232
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001934122367101304,
+ "loss": 1.2525,
+ "theoretical_loss": 3.3958204261181546,
+ "tokens_seen": 2510880768
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933961885656971,
+ "loss": 1.2455,
+ "theoretical_loss": 3.395813421604041,
+ "tokens_seen": 2510946304
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933801404212638,
+ "loss": 1.2042,
+ "theoretical_loss": 3.395806417323931,
+ "tokens_seen": 2511011840
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933640922768305,
+ "loss": 1.2715,
+ "theoretical_loss": 3.395799413277811,
+ "tokens_seen": 2511077376
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933480441323972,
+ "loss": 1.2831,
+ "theoretical_loss": 3.3957924094656673,
+ "tokens_seen": 2511142912
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933319959879639,
+ "loss": 1.1423,
+ "theoretical_loss": 3.3957854058874855,
+ "tokens_seen": 2511208448
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001933159478435306,
+ "loss": 1.2279,
+ "theoretical_loss": 3.3957784025432525,
+ "tokens_seen": 2511273984
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001932998996990973,
+ "loss": 1.3087,
+ "theoretical_loss": 3.3957713994329533,
+ "tokens_seen": 2511339520
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000193283851554664,
+ "loss": 1.3316,
+ "theoretical_loss": 3.395764396556575,
+ "tokens_seen": 2511405056
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001932678034102307,
+ "loss": 1.3056,
+ "theoretical_loss": 3.395757393914103,
+ "tokens_seen": 2511470592
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001932517552657974,
+ "loss": 1.1784,
+ "theoretical_loss": 3.3957503915055236,
+ "tokens_seen": 2511536128
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019323570712136408,
+ "loss": 1.2264,
+ "theoretical_loss": 3.395743389330823,
+ "tokens_seen": 2511601664
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019321965897693078,
+ "loss": 1.27,
+ "theoretical_loss": 3.395736387389987,
+ "tokens_seen": 2511667200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019320361083249748,
+ "loss": 1.3165,
+ "theoretical_loss": 3.395729385683002,
+ "tokens_seen": 2511732736
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019318756268806419,
+ "loss": 1.3195,
+ "theoretical_loss": 3.3957223842098543,
+ "tokens_seen": 2511798272
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019317151454363089,
+ "loss": 1.3362,
+ "theoretical_loss": 3.395715382970529,
+ "tokens_seen": 2511863808
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019315546639919759,
+ "loss": 1.2177,
+ "theoretical_loss": 3.3957083819650133,
+ "tokens_seen": 2511929344
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001931394182547643,
+ "loss": 1.2041,
+ "theoretical_loss": 3.395701381193293,
+ "tokens_seen": 2511994880
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000193123370110331,
+ "loss": 1.2994,
+ "theoretical_loss": 3.3956943806553532,
+ "tokens_seen": 2512060416
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001931073219658977,
+ "loss": 1.309,
+ "theoretical_loss": 3.3956873803511813,
+ "tokens_seen": 2512125952
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019309127382146442,
+ "loss": 1.2845,
+ "theoretical_loss": 3.395680380280763,
+ "tokens_seen": 2512191488
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019307522567703112,
+ "loss": 1.2844,
+ "theoretical_loss": 3.395673380444084,
+ "tokens_seen": 2512257024
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019305917753259782,
+ "loss": 1.3458,
+ "theoretical_loss": 3.3956663808411314,
+ "tokens_seen": 2512322560
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019304312938816452,
+ "loss": 1.1635,
+ "theoretical_loss": 3.39565938147189,
+ "tokens_seen": 2512388096
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019302708124373122,
+ "loss": 1.272,
+ "theoretical_loss": 3.3956523823363467,
+ "tokens_seen": 2512453632
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019301103309929792,
+ "loss": 1.2422,
+ "theoretical_loss": 3.3956453834344873,
+ "tokens_seen": 2512519168
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019299498495486462,
+ "loss": 1.1422,
+ "theoretical_loss": 3.3956383847662983,
+ "tokens_seen": 2512584704
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019297893681043132,
+ "loss": 1.1637,
+ "theoretical_loss": 3.395631386331765,
+ "tokens_seen": 2512650240
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019296288866599802,
+ "loss": 1.2324,
+ "theoretical_loss": 3.3956243881308747,
+ "tokens_seen": 2512715776
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019294684052156472,
+ "loss": 1.1093,
+ "theoretical_loss": 3.395617390163612,
+ "tokens_seen": 2512781312
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001929307923771314,
+ "loss": 1.2073,
+ "theoretical_loss": 3.3956103924299645,
+ "tokens_seen": 2512846848
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001929147442326981,
+ "loss": 1.218,
+ "theoretical_loss": 3.3956033949299176,
+ "tokens_seen": 2512912384
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928986960882648,
+ "loss": 1.263,
+ "theoretical_loss": 3.395596397663457,
+ "tokens_seen": 2512977920
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928826479438315,
+ "loss": 1.2031,
+ "theoretical_loss": 3.39558940063057,
+ "tokens_seen": 2513043456
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928665997993982,
+ "loss": 1.1723,
+ "theoretical_loss": 3.3955824038312414,
+ "tokens_seen": 2513108992
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928505516549649,
+ "loss": 1.3444,
+ "theoretical_loss": 3.395575407265458,
+ "tokens_seen": 2513174528
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928345035105316,
+ "loss": 1.2819,
+ "theoretical_loss": 3.395568410933206,
+ "tokens_seen": 2513240064
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 840191,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3639895915985107,
+ "objective/train/theoretical_loss": 3.395561414834471,
+ "objective/train/tokens_used": 2533765600,
+ "theoretical_loss": 3.395561414834471,
+ "tokens_seen": 2513305600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001928184553660983,
+ "loss": 1.3505,
+ "theoretical_loss": 3.395561414834471,
+ "tokens_seen": 2513305600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000192802407221665,
+ "loss": 1.2204,
+ "theoretical_loss": 3.39555441896924,
+ "tokens_seen": 2513371136
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927863590772317,
+ "loss": 1.2122,
+ "theoretical_loss": 3.395547423337498,
+ "tokens_seen": 2513436672
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927703109327984,
+ "loss": 1.1819,
+ "theoretical_loss": 3.395540427939232,
+ "tokens_seen": 2513502208
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927542627883651,
+ "loss": 1.2136,
+ "theoretical_loss": 3.3955334327744273,
+ "tokens_seen": 2513567744
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927382146439318,
+ "loss": 1.3192,
+ "theoretical_loss": 3.395526437843071,
+ "tokens_seen": 2513633280
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927221664994985,
+ "loss": 1.2008,
+ "theoretical_loss": 3.3955194431451488,
+ "tokens_seen": 2513698816
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001927061183550652,
+ "loss": 1.2369,
+ "theoretical_loss": 3.3955124486806465,
+ "tokens_seen": 2513764352
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001926900702106319,
+ "loss": 1.201,
+ "theoretical_loss": 3.3955054544495504,
+ "tokens_seen": 2513829888
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001926740220661986,
+ "loss": 1.2625,
+ "theoretical_loss": 3.3954984604518472,
+ "tokens_seen": 2513895424
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001926579739217653,
+ "loss": 1.3409,
+ "theoretical_loss": 3.395491466687522,
+ "tokens_seen": 2513960960
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000192641925777332,
+ "loss": 1.3522,
+ "theoretical_loss": 3.3954844731565617,
+ "tokens_seen": 2514026496
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001926258776328987,
+ "loss": 1.2753,
+ "theoretical_loss": 3.395477479858952,
+ "tokens_seen": 2514092032
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001926098294884654,
+ "loss": 1.3759,
+ "theoretical_loss": 3.3954704867946797,
+ "tokens_seen": 2514157568
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001925937813440321,
+ "loss": 1.1793,
+ "theoretical_loss": 3.3954634939637303,
+ "tokens_seen": 2514223104
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001925777331995988,
+ "loss": 1.1635,
+ "theoretical_loss": 3.3954565013660902,
+ "tokens_seen": 2514288640
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001925616850551655,
+ "loss": 1.2322,
+ "theoretical_loss": 3.3954495090017454,
+ "tokens_seen": 2514354176
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019254563691073218,
+ "loss": 1.2233,
+ "theoretical_loss": 3.395442516870682,
+ "tokens_seen": 2514419712
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019252958876629888,
+ "loss": 1.0636,
+ "theoretical_loss": 3.395435524972886,
+ "tokens_seen": 2514485248
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019251354062186558,
+ "loss": 1.3243,
+ "theoretical_loss": 3.395428533308344,
+ "tokens_seen": 2514550784
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019249749247743229,
+ "loss": 1.2092,
+ "theoretical_loss": 3.3954215418770417,
+ "tokens_seen": 2514616320
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000192481444332999,
+ "loss": 1.2317,
+ "theoretical_loss": 3.3954145506789657,
+ "tokens_seen": 2514681856
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019246539618856571,
+ "loss": 1.1753,
+ "theoretical_loss": 3.3954075597141014,
+ "tokens_seen": 2514747392
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019244934804413241,
+ "loss": 1.1243,
+ "theoretical_loss": 3.395400568982436,
+ "tokens_seen": 2514812928
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019243329989969911,
+ "loss": 1.3581,
+ "theoretical_loss": 3.395393578483955,
+ "tokens_seen": 2514878464
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019241725175526582,
+ "loss": 1.1478,
+ "theoretical_loss": 3.3953865882186443,
+ "tokens_seen": 2514944000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019240120361083252,
+ "loss": 1.1222,
+ "theoretical_loss": 3.3953795981864903,
+ "tokens_seen": 2515009536
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019238515546639922,
+ "loss": 1.1794,
+ "theoretical_loss": 3.395372608387479,
+ "tokens_seen": 2515075072
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019236910732196592,
+ "loss": 1.2134,
+ "theoretical_loss": 3.3953656188215975,
+ "tokens_seen": 2515140608
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019235305917753262,
+ "loss": 1.1846,
+ "theoretical_loss": 3.3953586294888307,
+ "tokens_seen": 2515206144
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019233701103309932,
+ "loss": 1.3127,
+ "theoretical_loss": 3.3953516403891655,
+ "tokens_seen": 2515271680
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019232096288866602,
+ "loss": 1.1587,
+ "theoretical_loss": 3.3953446515225876,
+ "tokens_seen": 2515337216
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019230491474423272,
+ "loss": 1.3073,
+ "theoretical_loss": 3.3953376628890832,
+ "tokens_seen": 2515402752
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019228886659979942,
+ "loss": 1.1928,
+ "theoretical_loss": 3.3953306744886387,
+ "tokens_seen": 2515468288
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019227281845536612,
+ "loss": 1.2513,
+ "theoretical_loss": 3.3953236863212406,
+ "tokens_seen": 2515533824
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019225677031093282,
+ "loss": 1.1662,
+ "theoretical_loss": 3.3953166983868743,
+ "tokens_seen": 2515599360
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019224072216649952,
+ "loss": 1.2908,
+ "theoretical_loss": 3.395309710685526,
+ "tokens_seen": 2515664896
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001922246740220662,
+ "loss": 1.2539,
+ "theoretical_loss": 3.3953027232171826,
+ "tokens_seen": 2515730432
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001922086258776329,
+ "loss": 1.2325,
+ "theoretical_loss": 3.3952957359818297,
+ "tokens_seen": 2515795968
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001921925777331996,
+ "loss": 1.2379,
+ "theoretical_loss": 3.3952887489794534,
+ "tokens_seen": 2515861504
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001921765295887663,
+ "loss": 1.2414,
+ "theoretical_loss": 3.39528176221004,
+ "tokens_seen": 2515927040
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000192160481444333,
+ "loss": 1.2462,
+ "theoretical_loss": 3.395274775673576,
+ "tokens_seen": 2515992576
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001921444332998997,
+ "loss": 1.3523,
+ "theoretical_loss": 3.395267789370047,
+ "tokens_seen": 2516058112
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001921283851554664,
+ "loss": 1.2588,
+ "theoretical_loss": 3.39526080329944,
+ "tokens_seen": 2516123648
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001921123370110331,
+ "loss": 1.2847,
+ "theoretical_loss": 3.39525381746174,
+ "tokens_seen": 2516189184
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920962888665998,
+ "loss": 1.3111,
+ "theoretical_loss": 3.3952468318569338,
+ "tokens_seen": 2516254720
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920802407221665,
+ "loss": 1.2607,
+ "theoretical_loss": 3.3952398464850075,
+ "tokens_seen": 2516320256
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920641925777332,
+ "loss": 1.1896,
+ "theoretical_loss": 3.3952328613459475,
+ "tokens_seen": 2516385792
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920481444332999,
+ "loss": 1.2671,
+ "theoretical_loss": 3.39522587643974,
+ "tokens_seen": 2516451328
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920320962888666,
+ "loss": 1.1638,
+ "theoretical_loss": 3.395218891766371,
+ "tokens_seen": 2516516864
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 841116,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0737740993499756,
+ "objective/train/theoretical_loss": 3.395211907325826,
+ "objective/train/tokens_used": 2537042400,
+ "theoretical_loss": 3.395211907325826,
+ "tokens_seen": 2516582400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001920160481444333,
+ "loss": 1.1446,
+ "theoretical_loss": 3.395211907325826,
+ "tokens_seen": 2516582400
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000192,
+ "loss": 1.2054,
+ "theoretical_loss": 3.395204923118092,
+ "tokens_seen": 2516647936
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919839518555667,
+ "loss": 1.3156,
+ "theoretical_loss": 3.395197939143155,
+ "tokens_seen": 2516713472
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919679037111334,
+ "loss": 1.2779,
+ "theoretical_loss": 3.3951909554010014,
+ "tokens_seen": 2516779008
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919518555667001,
+ "loss": 1.2375,
+ "theoretical_loss": 3.395183971891617,
+ "tokens_seen": 2516844544
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919358074222668,
+ "loss": 1.238,
+ "theoretical_loss": 3.3951769886149883,
+ "tokens_seen": 2516910080
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919197592778335,
+ "loss": 1.309,
+ "theoretical_loss": 3.3951700055711007,
+ "tokens_seen": 2516975616
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001919037111334002,
+ "loss": 1.3241,
+ "theoretical_loss": 3.3951630227599416,
+ "tokens_seen": 2517041152
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001918876629889669,
+ "loss": 1.3252,
+ "theoretical_loss": 3.3951560401814964,
+ "tokens_seen": 2517106688
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001918716148445336,
+ "loss": 1.2644,
+ "theoretical_loss": 3.3951490578357513,
+ "tokens_seen": 2517172224
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001918555667001003,
+ "loss": 1.2739,
+ "theoretical_loss": 3.395142075722693,
+ "tokens_seen": 2517237760
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000191839518555667,
+ "loss": 1.2622,
+ "theoretical_loss": 3.395135093842307,
+ "tokens_seen": 2517303296
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001918234704112337,
+ "loss": 1.2625,
+ "theoretical_loss": 3.39512811219458,
+ "tokens_seen": 2517368832
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001918074222668004,
+ "loss": 1.2306,
+ "theoretical_loss": 3.395121130779498,
+ "tokens_seen": 2517434368
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001917913741223671,
+ "loss": 1.1693,
+ "theoretical_loss": 3.3951141495970467,
+ "tokens_seen": 2517499904
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019177532597793381,
+ "loss": 1.2514,
+ "theoretical_loss": 3.3951071686472134,
+ "tokens_seen": 2517565440
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019175927783350051,
+ "loss": 1.2609,
+ "theoretical_loss": 3.395100187929984,
+ "tokens_seen": 2517630976
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019174322968906722,
+ "loss": 1.3038,
+ "theoretical_loss": 3.3950932074453433,
+ "tokens_seen": 2517696512
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019172718154463392,
+ "loss": 1.3479,
+ "theoretical_loss": 3.3950862271932793,
+ "tokens_seen": 2517762048
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019171113340020062,
+ "loss": 1.2301,
+ "theoretical_loss": 3.3950792471737774,
+ "tokens_seen": 2517827584
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019169508525576732,
+ "loss": 1.2599,
+ "theoretical_loss": 3.3950722673868237,
+ "tokens_seen": 2517893120
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019167903711133402,
+ "loss": 1.2017,
+ "theoretical_loss": 3.395065287832405,
+ "tokens_seen": 2517958656
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019166298896690072,
+ "loss": 1.2822,
+ "theoretical_loss": 3.3950583085105066,
+ "tokens_seen": 2518024192
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019164694082246742,
+ "loss": 1.1419,
+ "theoretical_loss": 3.3950513294211158,
+ "tokens_seen": 2518089728
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019163089267803412,
+ "loss": 1.1867,
+ "theoretical_loss": 3.3950443505642176,
+ "tokens_seen": 2518155264
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019161484453360082,
+ "loss": 1.1914,
+ "theoretical_loss": 3.395037371939799,
+ "tokens_seen": 2518220800
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019159879638916752,
+ "loss": 1.2774,
+ "theoretical_loss": 3.395030393547846,
+ "tokens_seen": 2518286336
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019158274824473422,
+ "loss": 1.2222,
+ "theoretical_loss": 3.3950234153883443,
+ "tokens_seen": 2518351872
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019156670010030092,
+ "loss": 1.2515,
+ "theoretical_loss": 3.3950164374612815,
+ "tokens_seen": 2518417408
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019155065195586762,
+ "loss": 1.3107,
+ "theoretical_loss": 3.3950094597666425,
+ "tokens_seen": 2518482944
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019153460381143432,
+ "loss": 1.2046,
+ "theoretical_loss": 3.3950024823044136,
+ "tokens_seen": 2518548480
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000191518555667001,
+ "loss": 1.2034,
+ "theoretical_loss": 3.394995505074582,
+ "tokens_seen": 2518614016
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001915025075225677,
+ "loss": 1.247,
+ "theoretical_loss": 3.394988528077133,
+ "tokens_seen": 2518679552
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914864593781344,
+ "loss": 1.2045,
+ "theoretical_loss": 3.3949815513120525,
+ "tokens_seen": 2518745088
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914704112337011,
+ "loss": 1.2915,
+ "theoretical_loss": 3.394974574779328,
+ "tokens_seen": 2518810624
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914543630892678,
+ "loss": 1.337,
+ "theoretical_loss": 3.394967598478945,
+ "tokens_seen": 2518876160
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914383149448345,
+ "loss": 1.2344,
+ "theoretical_loss": 3.3949606224108893,
+ "tokens_seen": 2518941696
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914222668004012,
+ "loss": 1.2545,
+ "theoretical_loss": 3.3949536465751478,
+ "tokens_seen": 2519007232
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001914062186559679,
+ "loss": 1.2373,
+ "theoretical_loss": 3.394946670971706,
+ "tokens_seen": 2519072768
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001913901705115346,
+ "loss": 1.2537,
+ "theoretical_loss": 3.3949396956005513,
+ "tokens_seen": 2519138304
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001913741223671013,
+ "loss": 1.2587,
+ "theoretical_loss": 3.394932720461669,
+ "tokens_seen": 2519203840
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000191358074222668,
+ "loss": 1.3021,
+ "theoretical_loss": 3.394925745555045,
+ "tokens_seen": 2519269376
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001913420260782347,
+ "loss": 1.2178,
+ "theoretical_loss": 3.3949187708806665,
+ "tokens_seen": 2519334912
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001913259779338014,
+ "loss": 1.3025,
+ "theoretical_loss": 3.394911796438519,
+ "tokens_seen": 2519400448
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001913099297893681,
+ "loss": 1.3094,
+ "theoretical_loss": 3.3949048222285896,
+ "tokens_seen": 2519465984
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001912938816449348,
+ "loss": 1.2495,
+ "theoretical_loss": 3.394897848250863,
+ "tokens_seen": 2519531520
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019127783350050153,
+ "loss": 1.0934,
+ "theoretical_loss": 3.394890874505327,
+ "tokens_seen": 2519597056
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019126178535606823,
+ "loss": 1.1425,
+ "theoretical_loss": 3.3948839009919674,
+ "tokens_seen": 2519662592
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019124573721163493,
+ "loss": 1.2714,
+ "theoretical_loss": 3.3948769277107695,
+ "tokens_seen": 2519728128
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019122968906720164,
+ "loss": 1.1877,
+ "theoretical_loss": 3.3948699546617207,
+ "tokens_seen": 2519793664
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 842376,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2077178955078125,
+ "objective/train/theoretical_loss": 3.394862981844807,
+ "objective/train/tokens_used": 2540319200,
+ "theoretical_loss": 3.394862981844807,
+ "tokens_seen": 2519859200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001912136409227683,
+ "loss": 1.2096,
+ "theoretical_loss": 3.394862981844807,
+ "tokens_seen": 2519859200
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000191197592778335,
+ "loss": 1.208,
+ "theoretical_loss": 3.394856009260014,
+ "tokens_seen": 2519924736
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911815446339017,
+ "loss": 1.4849,
+ "theoretical_loss": 3.3948490369073285,
+ "tokens_seen": 2519990272
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911654964894684,
+ "loss": 1.2556,
+ "theoretical_loss": 3.3948420647867366,
+ "tokens_seen": 2520055808
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911494483450351,
+ "loss": 1.2083,
+ "theoretical_loss": 3.3948350928982247,
+ "tokens_seen": 2520121344
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911334002006018,
+ "loss": 1.2162,
+ "theoretical_loss": 3.394828121241779,
+ "tokens_seen": 2520186880
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911173520561685,
+ "loss": 1.1963,
+ "theoretical_loss": 3.3948211498173846,
+ "tokens_seen": 2520252416
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001911013039117352,
+ "loss": 1.2108,
+ "theoretical_loss": 3.3948141786250297,
+ "tokens_seen": 2520317952
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019108525576730191,
+ "loss": 1.3027,
+ "theoretical_loss": 3.3948072076646993,
+ "tokens_seen": 2520383488
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019106920762286861,
+ "loss": 1.2871,
+ "theoretical_loss": 3.39480023693638,
+ "tokens_seen": 2520449024
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019105315947843532,
+ "loss": 1.2297,
+ "theoretical_loss": 3.394793266440058,
+ "tokens_seen": 2520514560
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019103711133400202,
+ "loss": 1.2092,
+ "theoretical_loss": 3.39478629617572,
+ "tokens_seen": 2520580096
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019102106318956872,
+ "loss": 1.1718,
+ "theoretical_loss": 3.3947793261433508,
+ "tokens_seen": 2520645632
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019100501504513542,
+ "loss": 1.3035,
+ "theoretical_loss": 3.3947723563429384,
+ "tokens_seen": 2520711168
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019098896690070212,
+ "loss": 1.216,
+ "theoretical_loss": 3.394765386774468,
+ "tokens_seen": 2520776704
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019097291875626882,
+ "loss": 1.2002,
+ "theoretical_loss": 3.394758417437926,
+ "tokens_seen": 2520842240
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019095687061183552,
+ "loss": 1.1021,
+ "theoretical_loss": 3.3947514483332992,
+ "tokens_seen": 2520907776
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019094082246740222,
+ "loss": 1.3205,
+ "theoretical_loss": 3.394744479460573,
+ "tokens_seen": 2520973312
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019092477432296892,
+ "loss": 1.1865,
+ "theoretical_loss": 3.394737510819734,
+ "tokens_seen": 2521038848
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019090872617853562,
+ "loss": 1.2165,
+ "theoretical_loss": 3.394730542410769,
+ "tokens_seen": 2521104384
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019089267803410232,
+ "loss": 1.2311,
+ "theoretical_loss": 3.394723574233664,
+ "tokens_seen": 2521169920
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019087662988966902,
+ "loss": 1.2046,
+ "theoretical_loss": 3.3947166062884047,
+ "tokens_seen": 2521235456
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019086058174523572,
+ "loss": 1.2477,
+ "theoretical_loss": 3.3947096385749775,
+ "tokens_seen": 2521300992
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019084453360080242,
+ "loss": 1.2479,
+ "theoretical_loss": 3.3947026710933694,
+ "tokens_seen": 2521366528
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001908284854563691,
+ "loss": 1.2128,
+ "theoretical_loss": 3.394695703843566,
+ "tokens_seen": 2521432064
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001908124373119358,
+ "loss": 1.2739,
+ "theoretical_loss": 3.3946887368255534,
+ "tokens_seen": 2521497600
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907963891675025,
+ "loss": 1.2322,
+ "theoretical_loss": 3.394681770039319,
+ "tokens_seen": 2521563136
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907803410230692,
+ "loss": 1.2407,
+ "theoretical_loss": 3.3946748034848473,
+ "tokens_seen": 2521628672
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907642928786359,
+ "loss": 1.2945,
+ "theoretical_loss": 3.3946678371621255,
+ "tokens_seen": 2521694208
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907482447342026,
+ "loss": 1.2716,
+ "theoretical_loss": 3.3946608710711406,
+ "tokens_seen": 2521759744
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907321965897693,
+ "loss": 1.2766,
+ "theoretical_loss": 3.394653905211878,
+ "tokens_seen": 2521825280
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.000190716148445336,
+ "loss": 1.3418,
+ "theoretical_loss": 3.3946469395843235,
+ "tokens_seen": 2521890816
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001907001003009027,
+ "loss": 1.2245,
+ "theoretical_loss": 3.3946399741884647,
+ "tokens_seen": 2521956352
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019068405215646943,
+ "loss": 1.1483,
+ "theoretical_loss": 3.394633009024287,
+ "tokens_seen": 2522021888
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019066800401203613,
+ "loss": 1.2167,
+ "theoretical_loss": 3.3946260440917766,
+ "tokens_seen": 2522087424
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019065195586760283,
+ "loss": 1.3095,
+ "theoretical_loss": 3.39461907939092,
+ "tokens_seen": 2522152960
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019063590772316953,
+ "loss": 1.2455,
+ "theoretical_loss": 3.3946121149217037,
+ "tokens_seen": 2522218496
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019061985957873623,
+ "loss": 1.1952,
+ "theoretical_loss": 3.3946051506841135,
+ "tokens_seen": 2522284032
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019060381143430293,
+ "loss": 1.1452,
+ "theoretical_loss": 3.3945981866781363,
+ "tokens_seen": 2522349568
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019058776328986963,
+ "loss": 1.3721,
+ "theoretical_loss": 3.394591222903758,
+ "tokens_seen": 2522415104
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019057171514543633,
+ "loss": 1.2393,
+ "theoretical_loss": 3.3945842593609643,
+ "tokens_seen": 2522480640
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019055566700100303,
+ "loss": 1.3651,
+ "theoretical_loss": 3.394577296049743,
+ "tokens_seen": 2522546176
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019053961885656974,
+ "loss": 1.2264,
+ "theoretical_loss": 3.394570332970079,
+ "tokens_seen": 2522611712
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019052357071213644,
+ "loss": 1.2683,
+ "theoretical_loss": 3.3945633701219586,
+ "tokens_seen": 2522677248
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001905075225677031,
+ "loss": 1.3467,
+ "theoretical_loss": 3.394556407505369,
+ "tokens_seen": 2522742784
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904914744232698,
+ "loss": 1.231,
+ "theoretical_loss": 3.3945494451202967,
+ "tokens_seen": 2522808320
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904754262788365,
+ "loss": 1.2425,
+ "theoretical_loss": 3.3945424829667266,
+ "tokens_seen": 2522873856
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904593781344032,
+ "loss": 1.1649,
+ "theoretical_loss": 3.3945355210446455,
+ "tokens_seen": 2522939392
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904433299899699,
+ "loss": 1.2585,
+ "theoretical_loss": 3.39452855935404,
+ "tokens_seen": 2523004928
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904272818455366,
+ "loss": 1.2358,
+ "theoretical_loss": 3.3945215978948964,
+ "tokens_seen": 2523070464
+ },
+ {
+ "epoch": 0.76,
+ "objective/train/docs_used": 843632,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2696731090545654,
+ "objective/train/theoretical_loss": 3.394514636667201,
+ "objective/train/tokens_used": 2543596000,
+ "theoretical_loss": 3.394514636667201,
+ "tokens_seen": 2523136000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001904112337011033,
+ "loss": 1.3735,
+ "theoretical_loss": 3.394514636667201,
+ "tokens_seen": 2523136000
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019039518555667001,
+ "loss": 1.3104,
+ "theoretical_loss": 3.39450767567094,
+ "tokens_seen": 2523201536
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019037913741223671,
+ "loss": 1.2269,
+ "theoretical_loss": 3.3945007149060995,
+ "tokens_seen": 2523267072
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019036308926780342,
+ "loss": 1.1875,
+ "theoretical_loss": 3.3944937543726663,
+ "tokens_seen": 2523332608
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019034704112337012,
+ "loss": 1.2556,
+ "theoretical_loss": 3.394486794070626,
+ "tokens_seen": 2523398144
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019033099297893682,
+ "loss": 1.1804,
+ "theoretical_loss": 3.394479833999965,
+ "tokens_seen": 2523463680
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019031494483450352,
+ "loss": 1.2697,
+ "theoretical_loss": 3.3944728741606705,
+ "tokens_seen": 2523529216
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019029889669007022,
+ "loss": 1.2328,
+ "theoretical_loss": 3.394465914552728,
+ "tokens_seen": 2523594752
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019028284854563692,
+ "loss": 1.1921,
+ "theoretical_loss": 3.3944589551761237,
+ "tokens_seen": 2523660288
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019026680040120362,
+ "loss": 1.2014,
+ "theoretical_loss": 3.394451996030844,
+ "tokens_seen": 2523725824
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019025075225677032,
+ "loss": 1.2345,
+ "theoretical_loss": 3.394445037116876,
+ "tokens_seen": 2523791360
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019023470411233702,
+ "loss": 1.2752,
+ "theoretical_loss": 3.394438078434205,
+ "tokens_seen": 2523856896
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019021865596790372,
+ "loss": 1.2722,
+ "theoretical_loss": 3.3944311199828174,
+ "tokens_seen": 2523922432
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019020260782347042,
+ "loss": 1.1993,
+ "theoretical_loss": 3.3944241617627005,
+ "tokens_seen": 2523987968
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019018655967903712,
+ "loss": 1.2526,
+ "theoretical_loss": 3.3944172037738394,
+ "tokens_seen": 2524053504
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019017051153460382,
+ "loss": 1.2317,
+ "theoretical_loss": 3.394410246016221,
+ "tokens_seen": 2524119040
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019015446339017052,
+ "loss": 1.1943,
+ "theoretical_loss": 3.3944032884898316,
+ "tokens_seen": 2524184576
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.00019013841524573722,
+ "loss": 1.25,
+ "theoretical_loss": 3.394396331194657,
+ "tokens_seen": 2524250112
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001901223671013039,
+ "loss": 1.2104,
+ "theoretical_loss": 3.3943893741306845,
+ "tokens_seen": 2524315648
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001901063189568706,
+ "loss": 1.2524,
+ "theoretical_loss": 3.3943824172978996,
+ "tokens_seen": 2524381184
+ },
+ {
+ "epoch": 0.76,
+ "learning_rate": 0.0001900902708124373,
+ "loss": 1.2413,
+ "theoretical_loss": 3.3943754606962893,
+ "tokens_seen": 2524446720
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00019007422266800403,
+ "loss": 1.1799,
+ "theoretical_loss": 3.394368504325839,
+ "tokens_seen": 2524512256
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00019005817452357073,
+ "loss": 1.185,
+ "theoretical_loss": 3.3943615481865352,
+ "tokens_seen": 2524577792
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00019004212637913743,
+ "loss": 1.2198,
+ "theoretical_loss": 3.394354592278365,
+ "tokens_seen": 2524643328
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00019002607823470413,
+ "loss": 1.2875,
+ "theoretical_loss": 3.3943476366013146,
+ "tokens_seen": 2524708864
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00019001003009027083,
+ "loss": 1.2503,
+ "theoretical_loss": 3.394340681155369,
+ "tokens_seen": 2524774400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018999398194583753,
+ "loss": 1.2372,
+ "theoretical_loss": 3.394333725940516,
+ "tokens_seen": 2524839936
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018997793380140423,
+ "loss": 1.2054,
+ "theoretical_loss": 3.394326770956742,
+ "tokens_seen": 2524905472
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018996188565697093,
+ "loss": 1.2249,
+ "theoretical_loss": 3.3943198162040322,
+ "tokens_seen": 2524971008
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018994583751253763,
+ "loss": 1.1985,
+ "theoretical_loss": 3.3943128616823737,
+ "tokens_seen": 2525036544
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018992978936810433,
+ "loss": 1.3384,
+ "theoretical_loss": 3.394305907391752,
+ "tokens_seen": 2525102080
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018991374122367103,
+ "loss": 1.2015,
+ "theoretical_loss": 3.3942989533321546,
+ "tokens_seen": 2525167616
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018989769307923773,
+ "loss": 1.3059,
+ "theoretical_loss": 3.394291999503567,
+ "tokens_seen": 2525233152
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018988164493480443,
+ "loss": 1.2896,
+ "theoretical_loss": 3.394285045905976,
+ "tokens_seen": 2525298688
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018986559679037114,
+ "loss": 1.2271,
+ "theoretical_loss": 3.3942780925393676,
+ "tokens_seen": 2525364224
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018984954864593784,
+ "loss": 1.2369,
+ "theoretical_loss": 3.394271139403728,
+ "tokens_seen": 2525429760
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018983350050150454,
+ "loss": 1.2166,
+ "theoretical_loss": 3.3942641864990444,
+ "tokens_seen": 2525495296
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018981745235707124,
+ "loss": 1.2171,
+ "theoretical_loss": 3.394257233825302,
+ "tokens_seen": 2525560832
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001898014042126379,
+ "loss": 1.1671,
+ "theoretical_loss": 3.3942502813824875,
+ "tokens_seen": 2525626368
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001897853560682046,
+ "loss": 1.2539,
+ "theoretical_loss": 3.394243329170588,
+ "tokens_seen": 2525691904
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001897693079237713,
+ "loss": 1.2905,
+ "theoretical_loss": 3.3942363771895887,
+ "tokens_seen": 2525757440
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.000189753259779338,
+ "loss": 1.2241,
+ "theoretical_loss": 3.3942294254394767,
+ "tokens_seen": 2525822976
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001897372116349047,
+ "loss": 1.2278,
+ "theoretical_loss": 3.3942224739202382,
+ "tokens_seen": 2525888512
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018972116349047141,
+ "loss": 1.2543,
+ "theoretical_loss": 3.3942155226318595,
+ "tokens_seen": 2525954048
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018970511534603811,
+ "loss": 1.2532,
+ "theoretical_loss": 3.3942085715743264,
+ "tokens_seen": 2526019584
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018968906720160481,
+ "loss": 1.2555,
+ "theoretical_loss": 3.3942016207476264,
+ "tokens_seen": 2526085120
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018967301905717152,
+ "loss": 1.2781,
+ "theoretical_loss": 3.394194670151745,
+ "tokens_seen": 2526150656
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018965697091273822,
+ "loss": 1.2557,
+ "theoretical_loss": 3.3941877197866686,
+ "tokens_seen": 2526216192
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018964092276830492,
+ "loss": 1.2569,
+ "theoretical_loss": 3.394180769652384,
+ "tokens_seen": 2526281728
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018962487462387162,
+ "loss": 1.1863,
+ "theoretical_loss": 3.3941738197488767,
+ "tokens_seen": 2526347264
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 844616,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1497843265533447,
+ "objective/train/theoretical_loss": 3.394166870076134,
+ "objective/train/tokens_used": 2546872800,
+ "theoretical_loss": 3.394166870076134,
+ "tokens_seen": 2526412800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018960882647943832,
+ "loss": 1.1828,
+ "theoretical_loss": 3.394166870076134,
+ "tokens_seen": 2526412800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018959277833500502,
+ "loss": 1.2301,
+ "theoretical_loss": 3.3941599206341415,
+ "tokens_seen": 2526478336
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018957673019057172,
+ "loss": 1.1905,
+ "theoretical_loss": 3.394152971422886,
+ "tokens_seen": 2526543872
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018956068204613842,
+ "loss": 1.2673,
+ "theoretical_loss": 3.3941460224423543,
+ "tokens_seen": 2526609408
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018954463390170512,
+ "loss": 1.2414,
+ "theoretical_loss": 3.3941390736925316,
+ "tokens_seen": 2526674944
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018952858575727182,
+ "loss": 1.3804,
+ "theoretical_loss": 3.394132125173405,
+ "tokens_seen": 2526740480
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018951253761283852,
+ "loss": 1.127,
+ "theoretical_loss": 3.3941251768849607,
+ "tokens_seen": 2526806016
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018949648946840522,
+ "loss": 1.3409,
+ "theoretical_loss": 3.3941182288271854,
+ "tokens_seen": 2526871552
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018948044132397192,
+ "loss": 1.2422,
+ "theoretical_loss": 3.394111281000065,
+ "tokens_seen": 2526937088
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018946439317953862,
+ "loss": 1.2815,
+ "theoretical_loss": 3.3941043334035856,
+ "tokens_seen": 2527002624
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018944834503510532,
+ "loss": 1.3702,
+ "theoretical_loss": 3.3940973860377346,
+ "tokens_seen": 2527068160
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018943229689067203,
+ "loss": 1.1912,
+ "theoretical_loss": 3.3940904389024973,
+ "tokens_seen": 2527133696
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018941624874623873,
+ "loss": 1.1665,
+ "theoretical_loss": 3.3940834919978604,
+ "tokens_seen": 2527199232
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018940020060180543,
+ "loss": 1.1886,
+ "theoretical_loss": 3.3940765453238106,
+ "tokens_seen": 2527264768
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018938415245737213,
+ "loss": 1.2574,
+ "theoretical_loss": 3.3940695988803338,
+ "tokens_seen": 2527330304
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018936810431293883,
+ "loss": 1.2512,
+ "theoretical_loss": 3.394062652667417,
+ "tokens_seen": 2527395840
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018935205616850553,
+ "loss": 1.0968,
+ "theoretical_loss": 3.394055706685046,
+ "tokens_seen": 2527461376
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018933600802407223,
+ "loss": 1.1788,
+ "theoretical_loss": 3.394048760933207,
+ "tokens_seen": 2527526912
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018931995987963893,
+ "loss": 1.1762,
+ "theoretical_loss": 3.394041815411887,
+ "tokens_seen": 2527592448
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018930391173520563,
+ "loss": 1.2626,
+ "theoretical_loss": 3.394034870121072,
+ "tokens_seen": 2527657984
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018928786359077233,
+ "loss": 1.1391,
+ "theoretical_loss": 3.3940279250607484,
+ "tokens_seen": 2527723520
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018927181544633903,
+ "loss": 1.2426,
+ "theoretical_loss": 3.394020980230903,
+ "tokens_seen": 2527789056
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018925576730190573,
+ "loss": 1.1303,
+ "theoretical_loss": 3.394014035631521,
+ "tokens_seen": 2527854592
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018923971915747243,
+ "loss": 1.2078,
+ "theoretical_loss": 3.3940070912625906,
+ "tokens_seen": 2527920128
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018922367101303913,
+ "loss": 1.2496,
+ "theoretical_loss": 3.394000147124096,
+ "tokens_seen": 2527985664
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018920762286860583,
+ "loss": 1.2562,
+ "theoretical_loss": 3.393993203216026,
+ "tokens_seen": 2528051200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018919157472417253,
+ "loss": 1.289,
+ "theoretical_loss": 3.393986259538365,
+ "tokens_seen": 2528116736
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018917552657973924,
+ "loss": 1.3237,
+ "theoretical_loss": 3.3939793160911,
+ "tokens_seen": 2528182272
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018915947843530594,
+ "loss": 1.1863,
+ "theoretical_loss": 3.3939723728742175,
+ "tokens_seen": 2528247808
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018914343029087264,
+ "loss": 1.3418,
+ "theoretical_loss": 3.393965429887704,
+ "tokens_seen": 2528313344
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018912738214643934,
+ "loss": 1.2635,
+ "theoretical_loss": 3.393958487131546,
+ "tokens_seen": 2528378880
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018911133400200604,
+ "loss": 1.1688,
+ "theoretical_loss": 3.3939515446057293,
+ "tokens_seen": 2528444416
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001890952858575727,
+ "loss": 1.2017,
+ "theoretical_loss": 3.393944602310241,
+ "tokens_seen": 2528509952
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001890792377131394,
+ "loss": 1.2106,
+ "theoretical_loss": 3.393937660245067,
+ "tokens_seen": 2528575488
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001890631895687061,
+ "loss": 1.222,
+ "theoretical_loss": 3.3939307184101932,
+ "tokens_seen": 2528641024
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001890471414242728,
+ "loss": 1.112,
+ "theoretical_loss": 3.3939237768056074,
+ "tokens_seen": 2528706560
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018903109327983951,
+ "loss": 1.2588,
+ "theoretical_loss": 3.3939168354312947,
+ "tokens_seen": 2528772096
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018901504513540621,
+ "loss": 1.1692,
+ "theoretical_loss": 3.3939098942872423,
+ "tokens_seen": 2528837632
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018899899699097292,
+ "loss": 1.1448,
+ "theoretical_loss": 3.393902953373436,
+ "tokens_seen": 2528903168
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018898294884653962,
+ "loss": 1.2601,
+ "theoretical_loss": 3.3938960126898623,
+ "tokens_seen": 2528968704
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018896690070210632,
+ "loss": 1.3064,
+ "theoretical_loss": 3.3938890722365085,
+ "tokens_seen": 2529034240
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018895085255767302,
+ "loss": 1.1788,
+ "theoretical_loss": 3.39388213201336,
+ "tokens_seen": 2529099776
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018893480441323972,
+ "loss": 1.2701,
+ "theoretical_loss": 3.393875192020403,
+ "tokens_seen": 2529165312
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018891875626880642,
+ "loss": 1.1069,
+ "theoretical_loss": 3.393868252257625,
+ "tokens_seen": 2529230848
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018890270812437312,
+ "loss": 1.1892,
+ "theoretical_loss": 3.393861312725011,
+ "tokens_seen": 2529296384
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018888665997993985,
+ "loss": 1.112,
+ "theoretical_loss": 3.393854373422549,
+ "tokens_seen": 2529361920
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018887061183550655,
+ "loss": 1.1283,
+ "theoretical_loss": 3.3938474343502243,
+ "tokens_seen": 2529427456
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018885456369107325,
+ "loss": 1.2236,
+ "theoretical_loss": 3.393840495508023,
+ "tokens_seen": 2529492992
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018883851554663995,
+ "loss": 1.2219,
+ "theoretical_loss": 3.393833556895933,
+ "tokens_seen": 2529558528
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018882246740220665,
+ "loss": 1.2756,
+ "theoretical_loss": 3.3938266185139394,
+ "tokens_seen": 2529624064
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 845397,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1955511569976807,
+ "objective/train/theoretical_loss": 3.3938196803620286,
+ "objective/train/tokens_used": 2550149600,
+ "theoretical_loss": 3.3938196803620286,
+ "tokens_seen": 2529689600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018880641925777335,
+ "loss": 1.2375,
+ "theoretical_loss": 3.3938196803620286,
+ "tokens_seen": 2529689600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018879037111334002,
+ "loss": 1.178,
+ "theoretical_loss": 3.393812742440188,
+ "tokens_seen": 2529755136
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018877432296890672,
+ "loss": 1.2921,
+ "theoretical_loss": 3.3938058047484034,
+ "tokens_seen": 2529820672
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018875827482447342,
+ "loss": 1.1552,
+ "theoretical_loss": 3.3937988672866606,
+ "tokens_seen": 2529886208
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018874222668004013,
+ "loss": 1.2997,
+ "theoretical_loss": 3.3937919300549475,
+ "tokens_seen": 2529951744
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018872617853560683,
+ "loss": 1.185,
+ "theoretical_loss": 3.393784993053249,
+ "tokens_seen": 2530017280
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018871013039117353,
+ "loss": 1.23,
+ "theoretical_loss": 3.393778056281553,
+ "tokens_seen": 2530082816
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018869408224674023,
+ "loss": 1.305,
+ "theoretical_loss": 3.3937711197398444,
+ "tokens_seen": 2530148352
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018867803410230693,
+ "loss": 1.2321,
+ "theoretical_loss": 3.3937641834281105,
+ "tokens_seen": 2530213888
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018866198595787363,
+ "loss": 1.3437,
+ "theoretical_loss": 3.3937572473463375,
+ "tokens_seen": 2530279424
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018864593781344033,
+ "loss": 1.2402,
+ "theoretical_loss": 3.393750311494512,
+ "tokens_seen": 2530344960
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018862988966900703,
+ "loss": 1.1274,
+ "theoretical_loss": 3.39374337587262,
+ "tokens_seen": 2530410496
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018861384152457373,
+ "loss": 1.3974,
+ "theoretical_loss": 3.3937364404806485,
+ "tokens_seen": 2530476032
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018859779338014043,
+ "loss": 1.1792,
+ "theoretical_loss": 3.3937295053185834,
+ "tokens_seen": 2530541568
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018858174523570713,
+ "loss": 1.2095,
+ "theoretical_loss": 3.3937225703864113,
+ "tokens_seen": 2530607104
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018856569709127383,
+ "loss": 1.177,
+ "theoretical_loss": 3.393715635684119,
+ "tokens_seen": 2530672640
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018854964894684053,
+ "loss": 1.245,
+ "theoretical_loss": 3.3937087012116924,
+ "tokens_seen": 2530738176
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018853360080240723,
+ "loss": 1.2846,
+ "theoretical_loss": 3.393701766969118,
+ "tokens_seen": 2530803712
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018851755265797393,
+ "loss": 1.2408,
+ "theoretical_loss": 3.3936948329563825,
+ "tokens_seen": 2530869248
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018850150451354063,
+ "loss": 1.2842,
+ "theoretical_loss": 3.3936878991734725,
+ "tokens_seen": 2530934784
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018848545636910734,
+ "loss": 1.1367,
+ "theoretical_loss": 3.3936809656203737,
+ "tokens_seen": 2531000320
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018846940822467404,
+ "loss": 1.1785,
+ "theoretical_loss": 3.393674032297073,
+ "tokens_seen": 2531065856
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018845336008024074,
+ "loss": 1.2866,
+ "theoretical_loss": 3.3936670992035567,
+ "tokens_seen": 2531131392
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018843731193580744,
+ "loss": 1.2279,
+ "theoretical_loss": 3.393660166339812,
+ "tokens_seen": 2531196928
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018842126379137414,
+ "loss": 1.1593,
+ "theoretical_loss": 3.3936532337058236,
+ "tokens_seen": 2531262464
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001884052156469408,
+ "loss": 1.2492,
+ "theoretical_loss": 3.3936463013015796,
+ "tokens_seen": 2531328000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001883891675025075,
+ "loss": 1.23,
+ "theoretical_loss": 3.393639369127066,
+ "tokens_seen": 2531393536
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001883731193580742,
+ "loss": 1.2131,
+ "theoretical_loss": 3.393632437182269,
+ "tokens_seen": 2531459072
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001883570712136409,
+ "loss": 1.308,
+ "theoretical_loss": 3.393625505467175,
+ "tokens_seen": 2531524608
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018834102306920761,
+ "loss": 1.2405,
+ "theoretical_loss": 3.3936185739817706,
+ "tokens_seen": 2531590144
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018832497492477431,
+ "loss": 1.2531,
+ "theoretical_loss": 3.393611642726042,
+ "tokens_seen": 2531655680
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018830892678034102,
+ "loss": 1.3061,
+ "theoretical_loss": 3.393604711699976,
+ "tokens_seen": 2531721216
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018829287863590772,
+ "loss": 1.2617,
+ "theoretical_loss": 3.393597780903559,
+ "tokens_seen": 2531786752
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018827683049147444,
+ "loss": 1.2688,
+ "theoretical_loss": 3.393590850336777,
+ "tokens_seen": 2531852288
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018826078234704114,
+ "loss": 1.1987,
+ "theoretical_loss": 3.393583919999617,
+ "tokens_seen": 2531917824
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018824473420260784,
+ "loss": 1.2373,
+ "theoretical_loss": 3.393576989892065,
+ "tokens_seen": 2531983360
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018822868605817455,
+ "loss": 1.2014,
+ "theoretical_loss": 3.3935700600141083,
+ "tokens_seen": 2532048896
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018821263791374125,
+ "loss": 1.133,
+ "theoretical_loss": 3.3935631303657323,
+ "tokens_seen": 2532114432
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018819658976930795,
+ "loss": 1.2373,
+ "theoretical_loss": 3.3935562009469242,
+ "tokens_seen": 2532179968
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018818054162487465,
+ "loss": 1.2476,
+ "theoretical_loss": 3.39354927175767,
+ "tokens_seen": 2532245504
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018816449348044135,
+ "loss": 1.1011,
+ "theoretical_loss": 3.393542342797956,
+ "tokens_seen": 2532311040
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018814844533600805,
+ "loss": 1.2366,
+ "theoretical_loss": 3.3935354140677694,
+ "tokens_seen": 2532376576
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018813239719157475,
+ "loss": 1.1558,
+ "theoretical_loss": 3.3935284855670957,
+ "tokens_seen": 2532442112
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018811634904714145,
+ "loss": 1.2294,
+ "theoretical_loss": 3.393521557295922,
+ "tokens_seen": 2532507648
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018810030090270815,
+ "loss": 1.2017,
+ "theoretical_loss": 3.393514629254235,
+ "tokens_seen": 2532573184
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018808425275827482,
+ "loss": 1.2824,
+ "theoretical_loss": 3.3935077014420205,
+ "tokens_seen": 2532638720
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018806820461384152,
+ "loss": 1.2697,
+ "theoretical_loss": 3.393500773859265,
+ "tokens_seen": 2532704256
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018805215646940823,
+ "loss": 1.3512,
+ "theoretical_loss": 3.393493846505956,
+ "tokens_seen": 2532769792
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018803610832497493,
+ "loss": 1.2202,
+ "theoretical_loss": 3.3934869193820782,
+ "tokens_seen": 2532835328
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018802006018054163,
+ "loss": 1.2925,
+ "theoretical_loss": 3.39347999248762,
+ "tokens_seen": 2532900864
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 846552,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3343746662139893,
+ "objective/train/theoretical_loss": 3.3934730658225662,
+ "objective/train/tokens_used": 2553426400,
+ "theoretical_loss": 3.3934730658225662,
+ "tokens_seen": 2532966400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018800401203610833,
+ "loss": 1.2571,
+ "theoretical_loss": 3.3934730658225662,
+ "tokens_seen": 2532966400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018798796389167503,
+ "loss": 1.2587,
+ "theoretical_loss": 3.3934661393869043,
+ "tokens_seen": 2533031936
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018797191574724173,
+ "loss": 1.2321,
+ "theoretical_loss": 3.3934592131806207,
+ "tokens_seen": 2533097472
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018795586760280843,
+ "loss": 1.1838,
+ "theoretical_loss": 3.393452287203701,
+ "tokens_seen": 2533163008
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018793981945837513,
+ "loss": 1.3736,
+ "theoretical_loss": 3.393445361456133,
+ "tokens_seen": 2533228544
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018792377131394183,
+ "loss": 1.2323,
+ "theoretical_loss": 3.393438435937902,
+ "tokens_seen": 2533294080
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018790772316950853,
+ "loss": 1.2497,
+ "theoretical_loss": 3.3934315106489947,
+ "tokens_seen": 2533359616
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018789167502507523,
+ "loss": 1.2638,
+ "theoretical_loss": 3.393424585589398,
+ "tokens_seen": 2533425152
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018787562688064193,
+ "loss": 1.2749,
+ "theoretical_loss": 3.3934176607590985,
+ "tokens_seen": 2533490688
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018785957873620863,
+ "loss": 1.2644,
+ "theoretical_loss": 3.393410736158082,
+ "tokens_seen": 2533556224
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018784353059177533,
+ "loss": 1.2245,
+ "theoretical_loss": 3.3934038117863357,
+ "tokens_seen": 2533621760
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018782748244734203,
+ "loss": 1.2003,
+ "theoretical_loss": 3.393396887643845,
+ "tokens_seen": 2533687296
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018781143430290873,
+ "loss": 1.1759,
+ "theoretical_loss": 3.393389963730598,
+ "tokens_seen": 2533752832
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018779538615847544,
+ "loss": 1.2726,
+ "theoretical_loss": 3.3933830400465794,
+ "tokens_seen": 2533818368
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018777933801404214,
+ "loss": 1.2588,
+ "theoretical_loss": 3.393376116591777,
+ "tokens_seen": 2533883904
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018776328986960884,
+ "loss": 1.1765,
+ "theoretical_loss": 3.393369193366177,
+ "tokens_seen": 2533949440
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018774724172517554,
+ "loss": 1.1964,
+ "theoretical_loss": 3.3933622703697655,
+ "tokens_seen": 2534014976
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018773119358074224,
+ "loss": 1.1999,
+ "theoretical_loss": 3.3933553476025295,
+ "tokens_seen": 2534080512
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018771514543630894,
+ "loss": 1.3008,
+ "theoretical_loss": 3.393348425064455,
+ "tokens_seen": 2534146048
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001876990972918756,
+ "loss": 1.1588,
+ "theoretical_loss": 3.3933415027555287,
+ "tokens_seen": 2534211584
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018768304914744234,
+ "loss": 1.149,
+ "theoretical_loss": 3.393334580675737,
+ "tokens_seen": 2534277120
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018766700100300904,
+ "loss": 1.172,
+ "theoretical_loss": 3.3933276588250667,
+ "tokens_seen": 2534342656
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018765095285857574,
+ "loss": 1.197,
+ "theoretical_loss": 3.3933207372035037,
+ "tokens_seen": 2534408192
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018763490471414244,
+ "loss": 1.3048,
+ "theoretical_loss": 3.3933138158110348,
+ "tokens_seen": 2534473728
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018761885656970914,
+ "loss": 1.2108,
+ "theoretical_loss": 3.393306894647647,
+ "tokens_seen": 2534539264
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018760280842527584,
+ "loss": 1.3318,
+ "theoretical_loss": 3.393299973713326,
+ "tokens_seen": 2534604800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018758676028084254,
+ "loss": 1.2412,
+ "theoretical_loss": 3.3932930530080587,
+ "tokens_seen": 2534670336
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018757071213640924,
+ "loss": 1.1867,
+ "theoretical_loss": 3.3932861325318315,
+ "tokens_seen": 2534735872
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018755466399197595,
+ "loss": 1.2762,
+ "theoretical_loss": 3.3932792122846314,
+ "tokens_seen": 2534801408
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018753861584754265,
+ "loss": 1.2688,
+ "theoretical_loss": 3.3932722922664436,
+ "tokens_seen": 2534866944
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018752256770310935,
+ "loss": 1.2418,
+ "theoretical_loss": 3.393265372477256,
+ "tokens_seen": 2534932480
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018750651955867605,
+ "loss": 1.1165,
+ "theoretical_loss": 3.3932584529170544,
+ "tokens_seen": 2534998016
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018749047141424275,
+ "loss": 1.1885,
+ "theoretical_loss": 3.3932515335858255,
+ "tokens_seen": 2535063552
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018747442326980945,
+ "loss": 1.2653,
+ "theoretical_loss": 3.3932446144835557,
+ "tokens_seen": 2535129088
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018745837512537615,
+ "loss": 1.2512,
+ "theoretical_loss": 3.3932376956102313,
+ "tokens_seen": 2535194624
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018744232698094285,
+ "loss": 1.2289,
+ "theoretical_loss": 3.393230776965839,
+ "tokens_seen": 2535260160
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018742627883650955,
+ "loss": 1.2001,
+ "theoretical_loss": 3.393223858550366,
+ "tokens_seen": 2535325696
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018741023069207625,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3932169403637977,
+ "tokens_seen": 2535391232
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018739418254764295,
+ "loss": 1.2015,
+ "theoretical_loss": 3.3932100224061212,
+ "tokens_seen": 2535456768
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018737813440320962,
+ "loss": 1.1933,
+ "theoretical_loss": 3.393203104677323,
+ "tokens_seen": 2535522304
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018736208625877633,
+ "loss": 1.2338,
+ "theoretical_loss": 3.3931961871773892,
+ "tokens_seen": 2535587840
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018734603811434303,
+ "loss": 1.2327,
+ "theoretical_loss": 3.3931892699063066,
+ "tokens_seen": 2535653376
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018732998996990973,
+ "loss": 1.2881,
+ "theoretical_loss": 3.393182352864062,
+ "tokens_seen": 2535718912
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018731394182547643,
+ "loss": 1.3371,
+ "theoretical_loss": 3.3931754360506416,
+ "tokens_seen": 2535784448
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018729789368104313,
+ "loss": 1.2341,
+ "theoretical_loss": 3.393168519466032,
+ "tokens_seen": 2535849984
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018728184553660983,
+ "loss": 1.2079,
+ "theoretical_loss": 3.3931616031102196,
+ "tokens_seen": 2535915520
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018726579739217653,
+ "loss": 1.1949,
+ "theoretical_loss": 3.3931546869831912,
+ "tokens_seen": 2535981056
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018724974924774323,
+ "loss": 1.1935,
+ "theoretical_loss": 3.3931477710849327,
+ "tokens_seen": 2536046592
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018723370110330993,
+ "loss": 1.1775,
+ "theoretical_loss": 3.393140855415431,
+ "tokens_seen": 2536112128
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018721765295887663,
+ "loss": 1.1763,
+ "theoretical_loss": 3.393133939974673,
+ "tokens_seen": 2536177664
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 847558,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3550597429275513,
+ "objective/train/theoretical_loss": 3.393127024762645,
+ "objective/train/tokens_used": 2556703200,
+ "theoretical_loss": 3.393127024762645,
+ "tokens_seen": 2536243200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018720160481444333,
+ "loss": 1.3461,
+ "theoretical_loss": 3.393127024762645,
+ "tokens_seen": 2536243200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018718555667001003,
+ "loss": 1.2017,
+ "theoretical_loss": 3.393120109779333,
+ "tokens_seen": 2536308736
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018716950852557673,
+ "loss": 1.212,
+ "theoretical_loss": 3.393113195024724,
+ "tokens_seen": 2536374272
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018715346038114343,
+ "loss": 1.2324,
+ "theoretical_loss": 3.3931062804988046,
+ "tokens_seen": 2536439808
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018713741223671013,
+ "loss": 1.2445,
+ "theoretical_loss": 3.393099366201561,
+ "tokens_seen": 2536505344
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018712136409227684,
+ "loss": 1.2912,
+ "theoretical_loss": 3.39309245213298,
+ "tokens_seen": 2536570880
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018710531594784354,
+ "loss": 1.2254,
+ "theoretical_loss": 3.3930855382930485,
+ "tokens_seen": 2536636416
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018708926780341024,
+ "loss": 1.2006,
+ "theoretical_loss": 3.393078624681752,
+ "tokens_seen": 2536701952
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018707321965897694,
+ "loss": 1.2528,
+ "theoretical_loss": 3.3930717112990783,
+ "tokens_seen": 2536767488
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018705717151454364,
+ "loss": 1.1579,
+ "theoretical_loss": 3.3930647981450126,
+ "tokens_seen": 2536833024
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018704112337011034,
+ "loss": 1.1762,
+ "theoretical_loss": 3.393057885219542,
+ "tokens_seen": 2536898560
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018702507522567704,
+ "loss": 1.2945,
+ "theoretical_loss": 3.3930509725226536,
+ "tokens_seen": 2536964096
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018700902708124374,
+ "loss": 1.1739,
+ "theoretical_loss": 3.3930440600543332,
+ "tokens_seen": 2537029632
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018699297893681044,
+ "loss": 1.2822,
+ "theoretical_loss": 3.3930371478145673,
+ "tokens_seen": 2537095168
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018697693079237714,
+ "loss": 1.2705,
+ "theoretical_loss": 3.3930302358033435,
+ "tokens_seen": 2537160704
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018696088264794384,
+ "loss": 1.2908,
+ "theoretical_loss": 3.393023324020647,
+ "tokens_seen": 2537226240
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018694483450351054,
+ "loss": 1.2306,
+ "theoretical_loss": 3.3930164124664652,
+ "tokens_seen": 2537291776
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018692878635907724,
+ "loss": 1.1837,
+ "theoretical_loss": 3.393009501140784,
+ "tokens_seen": 2537357312
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018691273821464394,
+ "loss": 1.3636,
+ "theoretical_loss": 3.3930025900435905,
+ "tokens_seen": 2537422848
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018689669007021064,
+ "loss": 1.2431,
+ "theoretical_loss": 3.392995679174871,
+ "tokens_seen": 2537488384
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018688064192577734,
+ "loss": 1.2426,
+ "theoretical_loss": 3.3929887685346123,
+ "tokens_seen": 2537553920
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018686459378134405,
+ "loss": 1.2503,
+ "theoretical_loss": 3.3929818581228006,
+ "tokens_seen": 2537619456
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018684854563691075,
+ "loss": 1.1494,
+ "theoretical_loss": 3.3929749479394227,
+ "tokens_seen": 2537684992
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018683249749247745,
+ "loss": 1.2439,
+ "theoretical_loss": 3.392968037984465,
+ "tokens_seen": 2537750528
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018681644934804415,
+ "loss": 1.258,
+ "theoretical_loss": 3.3929611282579137,
+ "tokens_seen": 2537816064
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018680040120361085,
+ "loss": 1.2285,
+ "theoretical_loss": 3.392954218759756,
+ "tokens_seen": 2537881600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018678435305917755,
+ "loss": 1.2453,
+ "theoretical_loss": 3.3929473094899785,
+ "tokens_seen": 2537947136
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018676830491474425,
+ "loss": 1.2704,
+ "theoretical_loss": 3.392940400448567,
+ "tokens_seen": 2538012672
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018675225677031095,
+ "loss": 1.2871,
+ "theoretical_loss": 3.392933491635509,
+ "tokens_seen": 2538078208
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018673620862587765,
+ "loss": 1.2006,
+ "theoretical_loss": 3.39292658305079,
+ "tokens_seen": 2538143744
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018672016048144435,
+ "loss": 1.2579,
+ "theoretical_loss": 3.3929196746943973,
+ "tokens_seen": 2538209280
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018670411233701105,
+ "loss": 1.2428,
+ "theoretical_loss": 3.3929127665663175,
+ "tokens_seen": 2538274816
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018668806419257773,
+ "loss": 1.0839,
+ "theoretical_loss": 3.392905858666537,
+ "tokens_seen": 2538340352
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018667201604814443,
+ "loss": 1.2616,
+ "theoretical_loss": 3.392898950995042,
+ "tokens_seen": 2538405888
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018665596790371113,
+ "loss": 1.1977,
+ "theoretical_loss": 3.3928920435518197,
+ "tokens_seen": 2538471424
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018663991975927783,
+ "loss": 1.2824,
+ "theoretical_loss": 3.392885136336856,
+ "tokens_seen": 2538536960
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018662387161484453,
+ "loss": 1.1505,
+ "theoretical_loss": 3.3928782293501376,
+ "tokens_seen": 2538602496
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018660782347041123,
+ "loss": 1.2781,
+ "theoretical_loss": 3.3928713225916516,
+ "tokens_seen": 2538668032
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018659177532597793,
+ "loss": 1.2953,
+ "theoretical_loss": 3.3928644160613843,
+ "tokens_seen": 2538733568
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018657572718154463,
+ "loss": 1.2341,
+ "theoretical_loss": 3.392857509759322,
+ "tokens_seen": 2538799104
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018655967903711133,
+ "loss": 1.2102,
+ "theoretical_loss": 3.3928506036854515,
+ "tokens_seen": 2538864640
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018654363089267803,
+ "loss": 1.1903,
+ "theoretical_loss": 3.3928436978397594,
+ "tokens_seen": 2538930176
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018652758274824473,
+ "loss": 1.2362,
+ "theoretical_loss": 3.3928367922222322,
+ "tokens_seen": 2538995712
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018651153460381143,
+ "loss": 1.1412,
+ "theoretical_loss": 3.392829886832857,
+ "tokens_seen": 2539061248
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018649548645937813,
+ "loss": 1.2274,
+ "theoretical_loss": 3.392822981671619,
+ "tokens_seen": 2539126784
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018647943831494486,
+ "loss": 1.2565,
+ "theoretical_loss": 3.3928160767385056,
+ "tokens_seen": 2539192320
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018646339017051156,
+ "loss": 1.1697,
+ "theoretical_loss": 3.3928091720335036,
+ "tokens_seen": 2539257856
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018644734202607826,
+ "loss": 1.1158,
+ "theoretical_loss": 3.3928022675565996,
+ "tokens_seen": 2539323392
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018643129388164496,
+ "loss": 1.3279,
+ "theoretical_loss": 3.39279536330778,
+ "tokens_seen": 2539388928
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018641524573721166,
+ "loss": 1.3171,
+ "theoretical_loss": 3.392788459287031,
+ "tokens_seen": 2539454464
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 848831,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.207207202911377,
+ "objective/train/theoretical_loss": 3.3927815554943397,
+ "objective/train/tokens_used": 2559980000,
+ "theoretical_loss": 3.3927815554943397,
+ "tokens_seen": 2539520000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018639919759277836,
+ "loss": 1.1975,
+ "theoretical_loss": 3.3927815554943397,
+ "tokens_seen": 2539520000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018638314944834506,
+ "loss": 1.1012,
+ "theoretical_loss": 3.3927746519296926,
+ "tokens_seen": 2539585536
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018636710130391174,
+ "loss": 1.3212,
+ "theoretical_loss": 3.3927677485930756,
+ "tokens_seen": 2539651072
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018635105315947844,
+ "loss": 1.4065,
+ "theoretical_loss": 3.3927608454844767,
+ "tokens_seen": 2539716608
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018633500501504514,
+ "loss": 1.2855,
+ "theoretical_loss": 3.392753942603881,
+ "tokens_seen": 2539782144
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018631895687061184,
+ "loss": 1.2198,
+ "theoretical_loss": 3.392747039951276,
+ "tokens_seen": 2539847680
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018630290872617854,
+ "loss": 1.2604,
+ "theoretical_loss": 3.392740137526648,
+ "tokens_seen": 2539913216
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018628686058174524,
+ "loss": 1.2049,
+ "theoretical_loss": 3.3927332353299837,
+ "tokens_seen": 2539978752
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018627081243731194,
+ "loss": 1.2307,
+ "theoretical_loss": 3.39272633336127,
+ "tokens_seen": 2540044288
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018625476429287864,
+ "loss": 1.3169,
+ "theoretical_loss": 3.3927194316204923,
+ "tokens_seen": 2540109824
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018623871614844534,
+ "loss": 1.1355,
+ "theoretical_loss": 3.392712530107638,
+ "tokens_seen": 2540175360
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018622266800401204,
+ "loss": 1.2241,
+ "theoretical_loss": 3.392705628822694,
+ "tokens_seen": 2540240896
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018620661985957874,
+ "loss": 1.1714,
+ "theoretical_loss": 3.3926987277656466,
+ "tokens_seen": 2540306432
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018619057171514544,
+ "loss": 1.1871,
+ "theoretical_loss": 3.3926918269364825,
+ "tokens_seen": 2540371968
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018617452357071215,
+ "loss": 1.1783,
+ "theoretical_loss": 3.3926849263351877,
+ "tokens_seen": 2540437504
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018615847542627885,
+ "loss": 1.1965,
+ "theoretical_loss": 3.3926780259617497,
+ "tokens_seen": 2540503040
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018614242728184555,
+ "loss": 1.1625,
+ "theoretical_loss": 3.3926711258161544,
+ "tokens_seen": 2540568576
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018612637913741225,
+ "loss": 1.1924,
+ "theoretical_loss": 3.392664225898389,
+ "tokens_seen": 2540634112
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018611033099297895,
+ "loss": 1.2037,
+ "theoretical_loss": 3.392657326208439,
+ "tokens_seen": 2540699648
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018609428284854565,
+ "loss": 1.2518,
+ "theoretical_loss": 3.3926504267462927,
+ "tokens_seen": 2540765184
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018607823470411235,
+ "loss": 1.2704,
+ "theoretical_loss": 3.392643527511935,
+ "tokens_seen": 2540830720
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018606218655967905,
+ "loss": 1.268,
+ "theoretical_loss": 3.3926366285053535,
+ "tokens_seen": 2540896256
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018604613841524575,
+ "loss": 1.2098,
+ "theoretical_loss": 3.3926297297265346,
+ "tokens_seen": 2540961792
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018603009027081245,
+ "loss": 1.2339,
+ "theoretical_loss": 3.392622831175465,
+ "tokens_seen": 2541027328
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018601404212637915,
+ "loss": 1.3431,
+ "theoretical_loss": 3.3926159328521313,
+ "tokens_seen": 2541092864
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018599799398194585,
+ "loss": 1.2883,
+ "theoretical_loss": 3.3926090347565196,
+ "tokens_seen": 2541158400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018598194583751253,
+ "loss": 1.1883,
+ "theoretical_loss": 3.3926021368886174,
+ "tokens_seen": 2541223936
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018596589769307923,
+ "loss": 1.2107,
+ "theoretical_loss": 3.3925952392484104,
+ "tokens_seen": 2541289472
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018594984954864593,
+ "loss": 1.1663,
+ "theoretical_loss": 3.3925883418358858,
+ "tokens_seen": 2541355008
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018593380140421263,
+ "loss": 1.2308,
+ "theoretical_loss": 3.39258144465103,
+ "tokens_seen": 2541420544
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018591775325977933,
+ "loss": 1.2113,
+ "theoretical_loss": 3.3925745476938296,
+ "tokens_seen": 2541486080
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018590170511534603,
+ "loss": 1.3116,
+ "theoretical_loss": 3.3925676509642715,
+ "tokens_seen": 2541551616
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018588565697091276,
+ "loss": 1.2396,
+ "theoretical_loss": 3.3925607544623415,
+ "tokens_seen": 2541617152
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018586960882647946,
+ "loss": 1.0856,
+ "theoretical_loss": 3.3925538581880272,
+ "tokens_seen": 2541682688
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018585356068204616,
+ "loss": 1.2278,
+ "theoretical_loss": 3.392546962141315,
+ "tokens_seen": 2541748224
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018583751253761286,
+ "loss": 1.2154,
+ "theoretical_loss": 3.392540066322191,
+ "tokens_seen": 2541813760
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018582146439317956,
+ "loss": 1.2223,
+ "theoretical_loss": 3.3925331707306423,
+ "tokens_seen": 2541879296
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018580541624874626,
+ "loss": 1.222,
+ "theoretical_loss": 3.392526275366656,
+ "tokens_seen": 2541944832
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018578936810431296,
+ "loss": 1.228,
+ "theoretical_loss": 3.392519380230217,
+ "tokens_seen": 2542010368
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018577331995987966,
+ "loss": 1.3536,
+ "theoretical_loss": 3.3925124853213138,
+ "tokens_seen": 2542075904
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018575727181544636,
+ "loss": 1.24,
+ "theoretical_loss": 3.392505590639932,
+ "tokens_seen": 2542141440
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018574122367101306,
+ "loss": 1.245,
+ "theoretical_loss": 3.3924986961860584,
+ "tokens_seen": 2542206976
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018572517552657976,
+ "loss": 1.309,
+ "theoretical_loss": 3.39249180195968,
+ "tokens_seen": 2542272512
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018570912738214646,
+ "loss": 1.2042,
+ "theoretical_loss": 3.392484907960783,
+ "tokens_seen": 2542338048
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018569307923771316,
+ "loss": 1.1865,
+ "theoretical_loss": 3.3924780141893542,
+ "tokens_seen": 2542403584
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018567703109327987,
+ "loss": 1.2568,
+ "theoretical_loss": 3.3924711206453804,
+ "tokens_seen": 2542469120
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018566098294884654,
+ "loss": 1.2161,
+ "theoretical_loss": 3.3924642273288477,
+ "tokens_seen": 2542534656
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018564493480441324,
+ "loss": 1.2403,
+ "theoretical_loss": 3.3924573342397433,
+ "tokens_seen": 2542600192
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018562888665997994,
+ "loss": 1.1861,
+ "theoretical_loss": 3.3924504413780534,
+ "tokens_seen": 2542665728
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018561283851554664,
+ "loss": 1.2911,
+ "theoretical_loss": 3.3924435487437647,
+ "tokens_seen": 2542731264
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 849771,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.192575454711914,
+ "objective/train/theoretical_loss": 3.3924366563368644,
+ "objective/train/tokens_used": 2563256800,
+ "theoretical_loss": 3.3924366563368644,
+ "tokens_seen": 2542796800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018559679037111334,
+ "loss": 1.1147,
+ "theoretical_loss": 3.3924366563368644,
+ "tokens_seen": 2542796800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018558074222668004,
+ "loss": 1.171,
+ "theoretical_loss": 3.3924297641573387,
+ "tokens_seen": 2542862336
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018556469408224674,
+ "loss": 1.2004,
+ "theoretical_loss": 3.392422872205174,
+ "tokens_seen": 2542927872
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018554864593781344,
+ "loss": 1.1737,
+ "theoretical_loss": 3.392415980480357,
+ "tokens_seen": 2542993408
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018553259779338014,
+ "loss": 1.2306,
+ "theoretical_loss": 3.392409088982875,
+ "tokens_seen": 2543058944
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018551654964894684,
+ "loss": 1.2507,
+ "theoretical_loss": 3.3924021977127143,
+ "tokens_seen": 2543124480
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018550050150451355,
+ "loss": 1.2864,
+ "theoretical_loss": 3.392395306669861,
+ "tokens_seen": 2543190016
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018548445336008025,
+ "loss": 1.1697,
+ "theoretical_loss": 3.3923884158543025,
+ "tokens_seen": 2543255552
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018546840521564695,
+ "loss": 1.1489,
+ "theoretical_loss": 3.3923815252660248,
+ "tokens_seen": 2543321088
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018545235707121365,
+ "loss": 1.1877,
+ "theoretical_loss": 3.3923746349050155,
+ "tokens_seen": 2543386624
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018543630892678035,
+ "loss": 1.2929,
+ "theoretical_loss": 3.39236774477126,
+ "tokens_seen": 2543452160
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018542026078234705,
+ "loss": 1.2211,
+ "theoretical_loss": 3.3923608548647457,
+ "tokens_seen": 2543517696
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018540421263791375,
+ "loss": 1.2752,
+ "theoretical_loss": 3.392353965185459,
+ "tokens_seen": 2543583232
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018538816449348045,
+ "loss": 1.2962,
+ "theoretical_loss": 3.3923470757333867,
+ "tokens_seen": 2543648768
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018537211634904715,
+ "loss": 1.2712,
+ "theoretical_loss": 3.3923401865085157,
+ "tokens_seen": 2543714304
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018535606820461385,
+ "loss": 1.1882,
+ "theoretical_loss": 3.3923332975108322,
+ "tokens_seen": 2543779840
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018534002006018055,
+ "loss": 1.185,
+ "theoretical_loss": 3.392326408740323,
+ "tokens_seen": 2543845376
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018532397191574725,
+ "loss": 1.3459,
+ "theoretical_loss": 3.392319520196975,
+ "tokens_seen": 2543910912
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018530792377131395,
+ "loss": 1.2366,
+ "theoretical_loss": 3.3923126318807744,
+ "tokens_seen": 2543976448
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018529187562688065,
+ "loss": 1.1826,
+ "theoretical_loss": 3.392305743791708,
+ "tokens_seen": 2544041984
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018527582748244735,
+ "loss": 1.3458,
+ "theoretical_loss": 3.392298855929763,
+ "tokens_seen": 2544107520
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018525977933801405,
+ "loss": 1.2044,
+ "theoretical_loss": 3.3922919682949253,
+ "tokens_seen": 2544173056
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018524373119358076,
+ "loss": 1.2424,
+ "theoretical_loss": 3.392285080887182,
+ "tokens_seen": 2544238592
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018522768304914746,
+ "loss": 1.2333,
+ "theoretical_loss": 3.3922781937065194,
+ "tokens_seen": 2544304128
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018521163490471416,
+ "loss": 1.1204,
+ "theoretical_loss": 3.392271306752925,
+ "tokens_seen": 2544369664
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018519558676028086,
+ "loss": 1.2526,
+ "theoretical_loss": 3.3922644200263843,
+ "tokens_seen": 2544435200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018517953861584756,
+ "loss": 1.2265,
+ "theoretical_loss": 3.3922575335268847,
+ "tokens_seen": 2544500736
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018516349047141426,
+ "loss": 1.1826,
+ "theoretical_loss": 3.3922506472544125,
+ "tokens_seen": 2544566272
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018514744232698096,
+ "loss": 1.2365,
+ "theoretical_loss": 3.392243761208955,
+ "tokens_seen": 2544631808
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018513139418254766,
+ "loss": 1.2403,
+ "theoretical_loss": 3.3922368753904983,
+ "tokens_seen": 2544697344
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018511534603811436,
+ "loss": 1.1491,
+ "theoretical_loss": 3.3922299897990293,
+ "tokens_seen": 2544762880
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018509929789368106,
+ "loss": 1.2645,
+ "theoretical_loss": 3.3922231044345343,
+ "tokens_seen": 2544828416
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018508324974924776,
+ "loss": 1.2381,
+ "theoretical_loss": 3.3922162192970005,
+ "tokens_seen": 2544893952
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018506720160481446,
+ "loss": 1.2347,
+ "theoretical_loss": 3.3922093343864144,
+ "tokens_seen": 2544959488
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018505115346038116,
+ "loss": 1.2003,
+ "theoretical_loss": 3.3922024497027623,
+ "tokens_seen": 2545025024
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018503510531594786,
+ "loss": 1.3243,
+ "theoretical_loss": 3.3921955652460314,
+ "tokens_seen": 2545090560
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018501905717151456,
+ "loss": 1.3079,
+ "theoretical_loss": 3.3921886810162083,
+ "tokens_seen": 2545156096
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018500300902708126,
+ "loss": 1.0825,
+ "theoretical_loss": 3.3921817970132793,
+ "tokens_seen": 2545221632
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018498696088264797,
+ "loss": 1.3195,
+ "theoretical_loss": 3.3921749132372314,
+ "tokens_seen": 2545287168
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018497091273821464,
+ "loss": 1.176,
+ "theoretical_loss": 3.3921680296880514,
+ "tokens_seen": 2545352704
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018495486459378134,
+ "loss": 1.219,
+ "theoretical_loss": 3.392161146365726,
+ "tokens_seen": 2545418240
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018493881644934804,
+ "loss": 1.2417,
+ "theoretical_loss": 3.3921542632702413,
+ "tokens_seen": 2545483776
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018492276830491474,
+ "loss": 1.2832,
+ "theoretical_loss": 3.3921473804015845,
+ "tokens_seen": 2545549312
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018490672016048144,
+ "loss": 1.1562,
+ "theoretical_loss": 3.392140497759742,
+ "tokens_seen": 2545614848
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018489067201604814,
+ "loss": 1.1892,
+ "theoretical_loss": 3.3921336153447004,
+ "tokens_seen": 2545680384
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018487462387161484,
+ "loss": 1.2632,
+ "theoretical_loss": 3.392126733156447,
+ "tokens_seen": 2545745920
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018485857572718154,
+ "loss": 1.2464,
+ "theoretical_loss": 3.392119851194968,
+ "tokens_seen": 2545811456
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018484252758274824,
+ "loss": 1.2623,
+ "theoretical_loss": 3.3921129694602503,
+ "tokens_seen": 2545876992
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018482647943831494,
+ "loss": 1.3974,
+ "theoretical_loss": 3.39210608795228,
+ "tokens_seen": 2545942528
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018481043129388165,
+ "loss": 1.3174,
+ "theoretical_loss": 3.3920992066710447,
+ "tokens_seen": 2546008064
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 851047,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1998200416564941,
+ "objective/train/theoretical_loss": 3.392092325616531,
+ "objective/train/tokens_used": 2566533600,
+ "theoretical_loss": 3.392092325616531,
+ "tokens_seen": 2546073600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018479438314944835,
+ "loss": 1.2766,
+ "theoretical_loss": 3.392092325616531,
+ "tokens_seen": 2546073600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018477833500501505,
+ "loss": 1.309,
+ "theoretical_loss": 3.3920854447887248,
+ "tokens_seen": 2546139136
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018476228686058175,
+ "loss": 1.2097,
+ "theoretical_loss": 3.3920785641876137,
+ "tokens_seen": 2546204672
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018474623871614845,
+ "loss": 1.261,
+ "theoretical_loss": 3.3920716838131835,
+ "tokens_seen": 2546270208
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018473019057171515,
+ "loss": 1.246,
+ "theoretical_loss": 3.3920648036654213,
+ "tokens_seen": 2546335744
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018471414242728185,
+ "loss": 1.2942,
+ "theoretical_loss": 3.3920579237443143,
+ "tokens_seen": 2546401280
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018469809428284855,
+ "loss": 1.22,
+ "theoretical_loss": 3.3920510440498486,
+ "tokens_seen": 2546466816
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018468204613841528,
+ "loss": 1.2648,
+ "theoretical_loss": 3.392044164582011,
+ "tokens_seen": 2546532352
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018466599799398198,
+ "loss": 1.3019,
+ "theoretical_loss": 3.392037285340788,
+ "tokens_seen": 2546597888
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018464994984954865,
+ "loss": 1.2753,
+ "theoretical_loss": 3.392030406326167,
+ "tokens_seen": 2546663424
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018463390170511535,
+ "loss": 1.3624,
+ "theoretical_loss": 3.392023527538134,
+ "tokens_seen": 2546728960
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018461785356068205,
+ "loss": 1.1839,
+ "theoretical_loss": 3.3920166489766763,
+ "tokens_seen": 2546794496
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018460180541624875,
+ "loss": 1.1924,
+ "theoretical_loss": 3.39200977064178,
+ "tokens_seen": 2546860032
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018458575727181545,
+ "loss": 1.2276,
+ "theoretical_loss": 3.3920028925334322,
+ "tokens_seen": 2546925568
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018456970912738215,
+ "loss": 1.303,
+ "theoretical_loss": 3.3919960146516193,
+ "tokens_seen": 2546991104
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018455366098294886,
+ "loss": 1.3268,
+ "theoretical_loss": 3.3919891369963286,
+ "tokens_seen": 2547056640
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018453761283851556,
+ "loss": 1.3704,
+ "theoretical_loss": 3.391982259567546,
+ "tokens_seen": 2547122176
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018452156469408226,
+ "loss": 1.3116,
+ "theoretical_loss": 3.391975382365259,
+ "tokens_seen": 2547187712
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018450551654964896,
+ "loss": 1.187,
+ "theoretical_loss": 3.3919685053894537,
+ "tokens_seen": 2547253248
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018448946840521566,
+ "loss": 1.2531,
+ "theoretical_loss": 3.391961628640117,
+ "tokens_seen": 2547318784
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018447342026078236,
+ "loss": 1.1989,
+ "theoretical_loss": 3.391954752117236,
+ "tokens_seen": 2547384320
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018445737211634906,
+ "loss": 1.2643,
+ "theoretical_loss": 3.3919478758207973,
+ "tokens_seen": 2547449856
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018444132397191576,
+ "loss": 1.185,
+ "theoretical_loss": 3.391940999750787,
+ "tokens_seen": 2547515392
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018442527582748246,
+ "loss": 1.253,
+ "theoretical_loss": 3.391934123907192,
+ "tokens_seen": 2547580928
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018440922768304916,
+ "loss": 1.2845,
+ "theoretical_loss": 3.3919272482899996,
+ "tokens_seen": 2547646464
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018439317953861586,
+ "loss": 1.4003,
+ "theoretical_loss": 3.3919203728991962,
+ "tokens_seen": 2547712000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018437713139418256,
+ "loss": 1.268,
+ "theoretical_loss": 3.3919134977347687,
+ "tokens_seen": 2547777536
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018436108324974926,
+ "loss": 1.1742,
+ "theoretical_loss": 3.391906622796703,
+ "tokens_seen": 2547843072
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018434503510531596,
+ "loss": 1.2059,
+ "theoretical_loss": 3.391899748084987,
+ "tokens_seen": 2547908608
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018432898696088266,
+ "loss": 1.2231,
+ "theoretical_loss": 3.3918928735996063,
+ "tokens_seen": 2547974144
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018431293881644936,
+ "loss": 1.2193,
+ "theoretical_loss": 3.391885999340549,
+ "tokens_seen": 2548039680
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018429689067201607,
+ "loss": 1.2512,
+ "theoretical_loss": 3.3918791253078004,
+ "tokens_seen": 2548105216
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018428084252758277,
+ "loss": 1.3363,
+ "theoretical_loss": 3.391872251501348,
+ "tokens_seen": 2548170752
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018426479438314944,
+ "loss": 1.3097,
+ "theoretical_loss": 3.3918653779211785,
+ "tokens_seen": 2548236288
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018424874623871614,
+ "loss": 1.224,
+ "theoretical_loss": 3.3918585045672787,
+ "tokens_seen": 2548301824
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018423269809428284,
+ "loss": 1.3264,
+ "theoretical_loss": 3.3918516314396348,
+ "tokens_seen": 2548367360
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018421664994984954,
+ "loss": 1.3602,
+ "theoretical_loss": 3.391844758538234,
+ "tokens_seen": 2548432896
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018420060180541624,
+ "loss": 1.2881,
+ "theoretical_loss": 3.391837885863063,
+ "tokens_seen": 2548498432
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018418455366098294,
+ "loss": 1.2621,
+ "theoretical_loss": 3.3918310134141083,
+ "tokens_seen": 2548563968
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018416850551654964,
+ "loss": 1.2291,
+ "theoretical_loss": 3.391824141191357,
+ "tokens_seen": 2548629504
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018415245737211634,
+ "loss": 1.2733,
+ "theoretical_loss": 3.3918172691947954,
+ "tokens_seen": 2548695040
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018413640922768304,
+ "loss": 1.184,
+ "theoretical_loss": 3.3918103974244107,
+ "tokens_seen": 2548760576
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018412036108324975,
+ "loss": 1.2642,
+ "theoretical_loss": 3.3918035258801895,
+ "tokens_seen": 2548826112
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018410431293881645,
+ "loss": 1.2345,
+ "theoretical_loss": 3.391796654562118,
+ "tokens_seen": 2548891648
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018408826479438315,
+ "loss": 1.2785,
+ "theoretical_loss": 3.391789783470184,
+ "tokens_seen": 2548957184
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018407221664994987,
+ "loss": 1.183,
+ "theoretical_loss": 3.391782912604373,
+ "tokens_seen": 2549022720
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018405616850551658,
+ "loss": 1.2723,
+ "theoretical_loss": 3.3917760419646727,
+ "tokens_seen": 2549088256
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018404012036108328,
+ "loss": 1.3081,
+ "theoretical_loss": 3.39176917155107,
+ "tokens_seen": 2549153792
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018402407221664998,
+ "loss": 1.1422,
+ "theoretical_loss": 3.3917623013635505,
+ "tokens_seen": 2549219328
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018400802407221668,
+ "loss": 1.132,
+ "theoretical_loss": 3.391755431402102,
+ "tokens_seen": 2549284864
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 852130,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1926579475402832,
+ "objective/train/theoretical_loss": 3.3917485616667107,
+ "objective/train/tokens_used": 2569810400,
+ "theoretical_loss": 3.3917485616667107,
+ "tokens_seen": 2549350400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018399197592778338,
+ "loss": 1.2137,
+ "theoretical_loss": 3.3917485616667107,
+ "tokens_seen": 2549350400
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018397592778335008,
+ "loss": 1.3151,
+ "theoretical_loss": 3.3917416921573635,
+ "tokens_seen": 2549415936
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018395987963891678,
+ "loss": 1.2872,
+ "theoretical_loss": 3.391734822874047,
+ "tokens_seen": 2549481472
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018394383149448345,
+ "loss": 1.3052,
+ "theoretical_loss": 3.3917279538167486,
+ "tokens_seen": 2549547008
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018392778335005015,
+ "loss": 1.2793,
+ "theoretical_loss": 3.391721084985454,
+ "tokens_seen": 2549612544
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018391173520561685,
+ "loss": 1.2703,
+ "theoretical_loss": 3.391714216380151,
+ "tokens_seen": 2549678080
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018389568706118355,
+ "loss": 1.2326,
+ "theoretical_loss": 3.3917073480008257,
+ "tokens_seen": 2549743616
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018387963891675025,
+ "loss": 1.1803,
+ "theoretical_loss": 3.3917004798474655,
+ "tokens_seen": 2549809152
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018386359077231696,
+ "loss": 1.2649,
+ "theoretical_loss": 3.3916936119200565,
+ "tokens_seen": 2549874688
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018384754262788366,
+ "loss": 1.3245,
+ "theoretical_loss": 3.3916867442185854,
+ "tokens_seen": 2549940224
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018383149448345036,
+ "loss": 1.3108,
+ "theoretical_loss": 3.3916798767430394,
+ "tokens_seen": 2550005760
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018381544633901706,
+ "loss": 1.2477,
+ "theoretical_loss": 3.391673009493405,
+ "tokens_seen": 2550071296
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018379939819458376,
+ "loss": 1.1879,
+ "theoretical_loss": 3.3916661424696692,
+ "tokens_seen": 2550136832
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018378335005015046,
+ "loss": 1.2243,
+ "theoretical_loss": 3.3916592756718185,
+ "tokens_seen": 2550202368
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018376730190571716,
+ "loss": 1.2074,
+ "theoretical_loss": 3.39165240909984,
+ "tokens_seen": 2550267904
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018375125376128386,
+ "loss": 1.1012,
+ "theoretical_loss": 3.39164554275372,
+ "tokens_seen": 2550333440
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018373520561685056,
+ "loss": 1.2154,
+ "theoretical_loss": 3.3916386766334456,
+ "tokens_seen": 2550398976
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018371915747241726,
+ "loss": 1.2439,
+ "theoretical_loss": 3.3916318107390038,
+ "tokens_seen": 2550464512
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018370310932798396,
+ "loss": 1.2744,
+ "theoretical_loss": 3.3916249450703804,
+ "tokens_seen": 2550530048
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018368706118355066,
+ "loss": 1.2515,
+ "theoretical_loss": 3.3916180796275635,
+ "tokens_seen": 2550595584
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018367101303911736,
+ "loss": 1.2048,
+ "theoretical_loss": 3.391611214410539,
+ "tokens_seen": 2550661120
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018365496489468406,
+ "loss": 1.1926,
+ "theoretical_loss": 3.3916043494192936,
+ "tokens_seen": 2550726656
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018363891675025076,
+ "loss": 1.2411,
+ "theoretical_loss": 3.391597484653815,
+ "tokens_seen": 2550792192
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018362286860581747,
+ "loss": 1.2559,
+ "theoretical_loss": 3.3915906201140884,
+ "tokens_seen": 2550857728
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018360682046138417,
+ "loss": 1.3021,
+ "theoretical_loss": 3.391583755800102,
+ "tokens_seen": 2550923264
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018359077231695087,
+ "loss": 1.1949,
+ "theoretical_loss": 3.391576891711842,
+ "tokens_seen": 2550988800
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018357472417251757,
+ "loss": 1.2003,
+ "theoretical_loss": 3.391570027849295,
+ "tokens_seen": 2551054336
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018355867602808424,
+ "loss": 1.2282,
+ "theoretical_loss": 3.3915631642124486,
+ "tokens_seen": 2551119872
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018354262788365094,
+ "loss": 1.2162,
+ "theoretical_loss": 3.391556300801289,
+ "tokens_seen": 2551185408
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018352657973921764,
+ "loss": 1.2095,
+ "theoretical_loss": 3.3915494376158026,
+ "tokens_seen": 2551250944
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018351053159478434,
+ "loss": 1.1886,
+ "theoretical_loss": 3.391542574655977,
+ "tokens_seen": 2551316480
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018349448345035104,
+ "loss": 1.2627,
+ "theoretical_loss": 3.391535711921798,
+ "tokens_seen": 2551382016
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018347843530591777,
+ "loss": 1.2215,
+ "theoretical_loss": 3.391528849413253,
+ "tokens_seen": 2551447552
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018346238716148447,
+ "loss": 1.2091,
+ "theoretical_loss": 3.3915219871303295,
+ "tokens_seen": 2551513088
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018344633901705117,
+ "loss": 1.2368,
+ "theoretical_loss": 3.391515125073013,
+ "tokens_seen": 2551578624
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018343029087261787,
+ "loss": 1.2635,
+ "theoretical_loss": 3.3915082632412905,
+ "tokens_seen": 2551644160
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018341424272818457,
+ "loss": 1.2198,
+ "theoretical_loss": 3.3915014016351495,
+ "tokens_seen": 2551709696
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018339819458375127,
+ "loss": 1.3362,
+ "theoretical_loss": 3.3914945402545764,
+ "tokens_seen": 2551775232
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018338214643931797,
+ "loss": 1.1543,
+ "theoretical_loss": 3.3914876790995576,
+ "tokens_seen": 2551840768
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018336609829488468,
+ "loss": 1.1687,
+ "theoretical_loss": 3.3914808181700806,
+ "tokens_seen": 2551906304
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018335005015045138,
+ "loss": 1.3378,
+ "theoretical_loss": 3.391473957466132,
+ "tokens_seen": 2551971840
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018333400200601808,
+ "loss": 1.2873,
+ "theoretical_loss": 3.3914670969876983,
+ "tokens_seen": 2552037376
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018331795386158478,
+ "loss": 1.2233,
+ "theoretical_loss": 3.391460236734767,
+ "tokens_seen": 2552102912
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018330190571715148,
+ "loss": 1.1899,
+ "theoretical_loss": 3.3914533767073234,
+ "tokens_seen": 2552168448
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018328585757271818,
+ "loss": 1.2549,
+ "theoretical_loss": 3.391446516905356,
+ "tokens_seen": 2552233984
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018326980942828488,
+ "loss": 1.2398,
+ "theoretical_loss": 3.3914396573288506,
+ "tokens_seen": 2552299520
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018325376128385155,
+ "loss": 1.1484,
+ "theoretical_loss": 3.391432797977794,
+ "tokens_seen": 2552365056
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018323771313941825,
+ "loss": 1.1898,
+ "theoretical_loss": 3.3914259388521737,
+ "tokens_seen": 2552430592
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018322166499498495,
+ "loss": 1.2129,
+ "theoretical_loss": 3.3914190799519757,
+ "tokens_seen": 2552496128
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018320561685055165,
+ "loss": 1.2012,
+ "theoretical_loss": 3.3914122212771876,
+ "tokens_seen": 2552561664
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 852961,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0892187356948853,
+ "objective/train/theoretical_loss": 3.391405362827795,
+ "objective/train/tokens_used": 2573087200,
+ "theoretical_loss": 3.391405362827795,
+ "tokens_seen": 2552627200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018318956870611836,
+ "loss": 1.1892,
+ "theoretical_loss": 3.391405362827795,
+ "tokens_seen": 2552627200
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018317352056168506,
+ "loss": 1.2731,
+ "theoretical_loss": 3.391398504603786,
+ "tokens_seen": 2552692736
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018315747241725176,
+ "loss": 1.2785,
+ "theoretical_loss": 3.391391646605147,
+ "tokens_seen": 2552758272
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018314142427281846,
+ "loss": 1.1491,
+ "theoretical_loss": 3.3913847888318647,
+ "tokens_seen": 2552823808
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018312537612838516,
+ "loss": 1.2727,
+ "theoretical_loss": 3.391377931283926,
+ "tokens_seen": 2552889344
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018310932798395186,
+ "loss": 1.2745,
+ "theoretical_loss": 3.3913710739613174,
+ "tokens_seen": 2552954880
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018309327983951856,
+ "loss": 1.2209,
+ "theoretical_loss": 3.3913642168640257,
+ "tokens_seen": 2553020416
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018307723169508526,
+ "loss": 1.3988,
+ "theoretical_loss": 3.391357359992038,
+ "tokens_seen": 2553085952
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018306118355065196,
+ "loss": 1.1938,
+ "theoretical_loss": 3.3913505033453415,
+ "tokens_seen": 2553151488
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018304513540621866,
+ "loss": 1.2882,
+ "theoretical_loss": 3.391343646923922,
+ "tokens_seen": 2553217024
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018302908726178536,
+ "loss": 1.2226,
+ "theoretical_loss": 3.391336790727767,
+ "tokens_seen": 2553282560
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018301303911735206,
+ "loss": 1.2052,
+ "theoretical_loss": 3.3913299347568637,
+ "tokens_seen": 2553348096
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018299699097291876,
+ "loss": 1.2964,
+ "theoretical_loss": 3.391323079011198,
+ "tokens_seen": 2553413632
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018298094282848546,
+ "loss": 1.2836,
+ "theoretical_loss": 3.3913162234907572,
+ "tokens_seen": 2553479168
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018296489468405216,
+ "loss": 1.2031,
+ "theoretical_loss": 3.391309368195528,
+ "tokens_seen": 2553544704
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018294884653961886,
+ "loss": 1.2805,
+ "theoretical_loss": 3.3913025131254977,
+ "tokens_seen": 2553610240
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018293279839518557,
+ "loss": 1.1786,
+ "theoretical_loss": 3.391295658280652,
+ "tokens_seen": 2553675776
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018291675025075227,
+ "loss": 1.2469,
+ "theoretical_loss": 3.391288803660979,
+ "tokens_seen": 2553741312
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018290070210631897,
+ "loss": 1.1115,
+ "theoretical_loss": 3.3912819492664648,
+ "tokens_seen": 2553806848
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018288465396188567,
+ "loss": 1.2825,
+ "theoretical_loss": 3.3912750950970962,
+ "tokens_seen": 2553872384
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018286860581745237,
+ "loss": 1.1998,
+ "theoretical_loss": 3.3912682411528605,
+ "tokens_seen": 2553937920
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018285255767301907,
+ "loss": 1.2509,
+ "theoretical_loss": 3.3912613874337443,
+ "tokens_seen": 2554003456
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018283650952858577,
+ "loss": 1.1624,
+ "theoretical_loss": 3.391254533939734,
+ "tokens_seen": 2554068992
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018282046138415247,
+ "loss": 1.2273,
+ "theoretical_loss": 3.3912476806708174,
+ "tokens_seen": 2554134528
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018280441323971917,
+ "loss": 1.2205,
+ "theoretical_loss": 3.39124082762698,
+ "tokens_seen": 2554200064
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018278836509528587,
+ "loss": 1.1506,
+ "theoretical_loss": 3.39123397480821,
+ "tokens_seen": 2554265600
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018277231695085257,
+ "loss": 1.2619,
+ "theoretical_loss": 3.391227122214493,
+ "tokens_seen": 2554331136
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018275626880641927,
+ "loss": 1.2738,
+ "theoretical_loss": 3.3912202698458165,
+ "tokens_seen": 2554396672
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018274022066198597,
+ "loss": 1.2275,
+ "theoretical_loss": 3.3912134177021676,
+ "tokens_seen": 2554462208
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018272417251755267,
+ "loss": 1.2459,
+ "theoretical_loss": 3.391206565783533,
+ "tokens_seen": 2554527744
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018270812437311937,
+ "loss": 1.1933,
+ "theoretical_loss": 3.3911997140898986,
+ "tokens_seen": 2554593280
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018269207622868607,
+ "loss": 1.1933,
+ "theoretical_loss": 3.391192862621253,
+ "tokens_seen": 2554658816
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018267602808425278,
+ "loss": 1.1571,
+ "theoretical_loss": 3.391186011377581,
+ "tokens_seen": 2554724352
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018265997993981948,
+ "loss": 1.2193,
+ "theoretical_loss": 3.3911791603588712,
+ "tokens_seen": 2554789888
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018264393179538618,
+ "loss": 1.192,
+ "theoretical_loss": 3.391172309565109,
+ "tokens_seen": 2554855424
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018262788365095288,
+ "loss": 1.2918,
+ "theoretical_loss": 3.3911654589962827,
+ "tokens_seen": 2554920960
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018261183550651958,
+ "loss": 1.3042,
+ "theoretical_loss": 3.391158608652378,
+ "tokens_seen": 2554986496
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018259578736208628,
+ "loss": 1.1547,
+ "theoretical_loss": 3.391151758533382,
+ "tokens_seen": 2555052032
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018257973921765298,
+ "loss": 1.1846,
+ "theoretical_loss": 3.391144908639282,
+ "tokens_seen": 2555117568
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018256369107321968,
+ "loss": 1.2734,
+ "theoretical_loss": 3.3911380589700646,
+ "tokens_seen": 2555183104
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018254764292878635,
+ "loss": 1.2575,
+ "theoretical_loss": 3.3911312095257165,
+ "tokens_seen": 2555248640
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018253159478435305,
+ "loss": 1.3138,
+ "theoretical_loss": 3.3911243603062244,
+ "tokens_seen": 2555314176
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018251554663991975,
+ "loss": 1.185,
+ "theoretical_loss": 3.3911175113115757,
+ "tokens_seen": 2555379712
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018249949849548646,
+ "loss": 1.1856,
+ "theoretical_loss": 3.391110662541757,
+ "tokens_seen": 2555445248
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018248345035105316,
+ "loss": 1.2174,
+ "theoretical_loss": 3.3911038139967546,
+ "tokens_seen": 2555510784
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018246740220661986,
+ "loss": 1.2622,
+ "theoretical_loss": 3.391096965676556,
+ "tokens_seen": 2555576320
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018245135406218656,
+ "loss": 1.2002,
+ "theoretical_loss": 3.3910901175811485,
+ "tokens_seen": 2555641856
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018243530591775326,
+ "loss": 1.3218,
+ "theoretical_loss": 3.391083269710518,
+ "tokens_seen": 2555707392
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018241925777331996,
+ "loss": 1.1216,
+ "theoretical_loss": 3.3910764220646517,
+ "tokens_seen": 2555772928
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018240320962888666,
+ "loss": 1.2135,
+ "theoretical_loss": 3.391069574643536,
+ "tokens_seen": 2555838464
+ },
+ {
+ "epoch": 0.77,
+ "objective/train/docs_used": 854270,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.256941318511963,
+ "objective/train/theoretical_loss": 3.391062727447159,
+ "objective/train/tokens_used": 2576364000,
+ "theoretical_loss": 3.391062727447159,
+ "tokens_seen": 2555904000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018238716148445336,
+ "loss": 1.1547,
+ "theoretical_loss": 3.391062727447159,
+ "tokens_seen": 2555904000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018237111334002006,
+ "loss": 1.1315,
+ "theoretical_loss": 3.3910558804755064,
+ "tokens_seen": 2555969536
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018235506519558676,
+ "loss": 1.1719,
+ "theoretical_loss": 3.391049033728566,
+ "tokens_seen": 2556035072
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018233901705115346,
+ "loss": 1.19,
+ "theoretical_loss": 3.3910421872063234,
+ "tokens_seen": 2556100608
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018232296890672016,
+ "loss": 1.2329,
+ "theoretical_loss": 3.3910353409087666,
+ "tokens_seen": 2556166144
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018230692076228686,
+ "loss": 1.3025,
+ "theoretical_loss": 3.391028494835882,
+ "tokens_seen": 2556231680
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018229087261785356,
+ "loss": 1.2144,
+ "theoretical_loss": 3.391021648987657,
+ "tokens_seen": 2556297216
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001822748244734203,
+ "loss": 1.251,
+ "theoretical_loss": 3.3910148033640772,
+ "tokens_seen": 2556362752
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.000182258776328987,
+ "loss": 1.2475,
+ "theoretical_loss": 3.3910079579651304,
+ "tokens_seen": 2556428288
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.0001822427281845537,
+ "loss": 1.1583,
+ "theoretical_loss": 3.391001112790804,
+ "tokens_seen": 2556493824
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018222668004012037,
+ "loss": 1.252,
+ "theoretical_loss": 3.3909942678410836,
+ "tokens_seen": 2556559360
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018221063189568707,
+ "loss": 1.1881,
+ "theoretical_loss": 3.390987423115957,
+ "tokens_seen": 2556624896
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018219458375125377,
+ "loss": 1.3174,
+ "theoretical_loss": 3.3909805786154106,
+ "tokens_seen": 2556690432
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018217853560682047,
+ "loss": 1.2002,
+ "theoretical_loss": 3.3909737343394313,
+ "tokens_seen": 2556755968
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018216248746238717,
+ "loss": 1.1978,
+ "theoretical_loss": 3.390966890288006,
+ "tokens_seen": 2556821504
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018214643931795387,
+ "loss": 1.2715,
+ "theoretical_loss": 3.3909600464611223,
+ "tokens_seen": 2556887040
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018213039117352057,
+ "loss": 1.2949,
+ "theoretical_loss": 3.390953202858766,
+ "tokens_seen": 2556952576
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018211434302908727,
+ "loss": 1.3087,
+ "theoretical_loss": 3.3909463594809246,
+ "tokens_seen": 2557018112
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018209829488465397,
+ "loss": 1.2588,
+ "theoretical_loss": 3.3909395163275846,
+ "tokens_seen": 2557083648
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018208224674022067,
+ "loss": 1.1062,
+ "theoretical_loss": 3.3909326733987335,
+ "tokens_seen": 2557149184
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018206619859578737,
+ "loss": 1.2165,
+ "theoretical_loss": 3.3909258306943575,
+ "tokens_seen": 2557214720
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018205015045135407,
+ "loss": 1.1035,
+ "theoretical_loss": 3.3909189882144437,
+ "tokens_seen": 2557280256
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018203410230692077,
+ "loss": 1.0944,
+ "theoretical_loss": 3.3909121459589793,
+ "tokens_seen": 2557345792
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018201805416248747,
+ "loss": 1.2671,
+ "theoretical_loss": 3.390905303927951,
+ "tokens_seen": 2557411328
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 0.00018200200601805417,
+ "loss": 1.2414,
+ "theoretical_loss": 3.3908984621213456,
+ "tokens_seen": 2557476864
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018198595787362088,
+ "loss": 1.184,
+ "theoretical_loss": 3.3908916205391497,
+ "tokens_seen": 2557542400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018196990972918758,
+ "loss": 1.2357,
+ "theoretical_loss": 3.390884779181351,
+ "tokens_seen": 2557607936
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018195386158475428,
+ "loss": 1.272,
+ "theoretical_loss": 3.3908779380479355,
+ "tokens_seen": 2557673472
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018193781344032098,
+ "loss": 1.3094,
+ "theoretical_loss": 3.3908710971388905,
+ "tokens_seen": 2557739008
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018192176529588768,
+ "loss": 1.2334,
+ "theoretical_loss": 3.3908642564542033,
+ "tokens_seen": 2557804544
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018190571715145438,
+ "loss": 1.2013,
+ "theoretical_loss": 3.39085741599386,
+ "tokens_seen": 2557870080
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018188966900702108,
+ "loss": 1.2655,
+ "theoretical_loss": 3.390850575757848,
+ "tokens_seen": 2557935616
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018187362086258778,
+ "loss": 1.2648,
+ "theoretical_loss": 3.390843735746154,
+ "tokens_seen": 2558001152
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018185757271815448,
+ "loss": 1.1647,
+ "theoretical_loss": 3.3908368959587647,
+ "tokens_seen": 2558066688
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018184152457372115,
+ "loss": 1.1636,
+ "theoretical_loss": 3.390830056395668,
+ "tokens_seen": 2558132224
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018182547642928785,
+ "loss": 1.1806,
+ "theoretical_loss": 3.3908232170568495,
+ "tokens_seen": 2558197760
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018180942828485456,
+ "loss": 1.2388,
+ "theoretical_loss": 3.3908163779422966,
+ "tokens_seen": 2558263296
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018179338014042126,
+ "loss": 1.2125,
+ "theoretical_loss": 3.3908095390519963,
+ "tokens_seen": 2558328832
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018177733199598796,
+ "loss": 1.139,
+ "theoretical_loss": 3.390802700385936,
+ "tokens_seen": 2558394368
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018176128385155466,
+ "loss": 1.278,
+ "theoretical_loss": 3.3907958619441017,
+ "tokens_seen": 2558459904
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018174523570712136,
+ "loss": 1.2945,
+ "theoretical_loss": 3.39078902372648,
+ "tokens_seen": 2558525440
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018172918756268806,
+ "loss": 1.2162,
+ "theoretical_loss": 3.39078218573306,
+ "tokens_seen": 2558590976
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018171313941825476,
+ "loss": 1.1633,
+ "theoretical_loss": 3.390775347963826,
+ "tokens_seen": 2558656512
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018169709127382146,
+ "loss": 1.268,
+ "theoretical_loss": 3.3907685104187664,
+ "tokens_seen": 2558722048
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001816810431293882,
+ "loss": 1.2033,
+ "theoretical_loss": 3.390761673097867,
+ "tokens_seen": 2558787584
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001816649949849549,
+ "loss": 1.2641,
+ "theoretical_loss": 3.390754836001116,
+ "tokens_seen": 2558853120
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001816489468405216,
+ "loss": 1.2824,
+ "theoretical_loss": 3.3907479991284997,
+ "tokens_seen": 2558918656
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001816328986960883,
+ "loss": 1.216,
+ "theoretical_loss": 3.3907411624800052,
+ "tokens_seen": 2558984192
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000181616850551655,
+ "loss": 1.2461,
+ "theoretical_loss": 3.390734326055619,
+ "tokens_seen": 2559049728
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001816008024072217,
+ "loss": 1.2625,
+ "theoretical_loss": 3.3907274898553283,
+ "tokens_seen": 2559115264
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 855161,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.085707426071167,
+ "objective/train/theoretical_loss": 3.39072065387912,
+ "objective/train/tokens_used": 2579640800,
+ "theoretical_loss": 3.39072065387912,
+ "tokens_seen": 2559180800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001815847542627884,
+ "loss": 1.0913,
+ "theoretical_loss": 3.39072065387912,
+ "tokens_seen": 2559180800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001815687061183551,
+ "loss": 1.2379,
+ "theoretical_loss": 3.390713818126981,
+ "tokens_seen": 2559246336
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001815526579739218,
+ "loss": 1.265,
+ "theoretical_loss": 3.390706982598898,
+ "tokens_seen": 2559311872
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001815366098294885,
+ "loss": 1.2565,
+ "theoretical_loss": 3.390700147294859,
+ "tokens_seen": 2559377408
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018152056168505517,
+ "loss": 1.1769,
+ "theoretical_loss": 3.3906933122148493,
+ "tokens_seen": 2559442944
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018150451354062187,
+ "loss": 1.191,
+ "theoretical_loss": 3.3906864773588565,
+ "tokens_seen": 2559508480
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018148846539618857,
+ "loss": 1.3477,
+ "theoretical_loss": 3.3906796427268677,
+ "tokens_seen": 2559574016
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018147241725175527,
+ "loss": 1.2367,
+ "theoretical_loss": 3.39067280831887,
+ "tokens_seen": 2559639552
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018145636910732197,
+ "loss": 1.2564,
+ "theoretical_loss": 3.3906659741348495,
+ "tokens_seen": 2559705088
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018144032096288867,
+ "loss": 1.1895,
+ "theoretical_loss": 3.3906591401747943,
+ "tokens_seen": 2559770624
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018142427281845537,
+ "loss": 1.2772,
+ "theoretical_loss": 3.39065230643869,
+ "tokens_seen": 2559836160
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018140822467402207,
+ "loss": 1.1839,
+ "theoretical_loss": 3.3906454729265247,
+ "tokens_seen": 2559901696
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018139217652958877,
+ "loss": 1.1996,
+ "theoretical_loss": 3.390638639638285,
+ "tokens_seen": 2559967232
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018137612838515547,
+ "loss": 1.2065,
+ "theoretical_loss": 3.390631806573958,
+ "tokens_seen": 2560032768
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018136008024072217,
+ "loss": 1.2414,
+ "theoretical_loss": 3.3906249737335297,
+ "tokens_seen": 2560098304
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018134403209628887,
+ "loss": 1.2772,
+ "theoretical_loss": 3.3906181411169873,
+ "tokens_seen": 2560163840
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018132798395185557,
+ "loss": 1.2337,
+ "theoretical_loss": 3.390611308724319,
+ "tokens_seen": 2560229376
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018131193580742228,
+ "loss": 1.2546,
+ "theoretical_loss": 3.3906044765555103,
+ "tokens_seen": 2560294912
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018129588766298898,
+ "loss": 1.2422,
+ "theoretical_loss": 3.3905976446105486,
+ "tokens_seen": 2560360448
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018127983951855568,
+ "loss": 1.214,
+ "theoretical_loss": 3.390590812889421,
+ "tokens_seen": 2560425984
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018126379137412238,
+ "loss": 1.323,
+ "theoretical_loss": 3.3905839813921146,
+ "tokens_seen": 2560491520
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018124774322968908,
+ "loss": 1.2029,
+ "theoretical_loss": 3.3905771501186157,
+ "tokens_seen": 2560557056
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018123169508525578,
+ "loss": 1.2888,
+ "theoretical_loss": 3.3905703190689116,
+ "tokens_seen": 2560622592
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018121564694082248,
+ "loss": 1.2175,
+ "theoretical_loss": 3.3905634882429894,
+ "tokens_seen": 2560688128
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018119959879638918,
+ "loss": 1.1518,
+ "theoretical_loss": 3.3905566576408357,
+ "tokens_seen": 2560753664
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018118355065195588,
+ "loss": 1.1843,
+ "theoretical_loss": 3.390549827262438,
+ "tokens_seen": 2560819200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018116750250752258,
+ "loss": 1.232,
+ "theoretical_loss": 3.3905429971077825,
+ "tokens_seen": 2560884736
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018115145436308928,
+ "loss": 1.2461,
+ "theoretical_loss": 3.390536167176857,
+ "tokens_seen": 2560950272
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018113540621865595,
+ "loss": 1.1495,
+ "theoretical_loss": 3.3905293374696477,
+ "tokens_seen": 2561015808
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018111935807422266,
+ "loss": 1.2908,
+ "theoretical_loss": 3.3905225079861414,
+ "tokens_seen": 2561081344
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018110330992978936,
+ "loss": 1.2805,
+ "theoretical_loss": 3.390515678726326,
+ "tokens_seen": 2561146880
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018108726178535606,
+ "loss": 1.2426,
+ "theoretical_loss": 3.3905088496901876,
+ "tokens_seen": 2561212416
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018107121364092278,
+ "loss": 1.2379,
+ "theoretical_loss": 3.3905020208777135,
+ "tokens_seen": 2561277952
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018105516549648949,
+ "loss": 1.1045,
+ "theoretical_loss": 3.390495192288891,
+ "tokens_seen": 2561343488
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018103911735205619,
+ "loss": 1.2663,
+ "theoretical_loss": 3.3904883639237062,
+ "tokens_seen": 2561409024
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001810230692076229,
+ "loss": 1.2875,
+ "theoretical_loss": 3.390481535782147,
+ "tokens_seen": 2561474560
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001810070210631896,
+ "loss": 1.2584,
+ "theoretical_loss": 3.3904747078641995,
+ "tokens_seen": 2561540096
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001809909729187563,
+ "loss": 1.2855,
+ "theoretical_loss": 3.390467880169851,
+ "tokens_seen": 2561605632
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000180974924774323,
+ "loss": 1.2838,
+ "theoretical_loss": 3.390461052699089,
+ "tokens_seen": 2561671168
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001809588766298897,
+ "loss": 1.1751,
+ "theoretical_loss": 3.390454225451899,
+ "tokens_seen": 2561736704
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001809428284854564,
+ "loss": 1.1917,
+ "theoretical_loss": 3.39044739842827,
+ "tokens_seen": 2561802240
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001809267803410231,
+ "loss": 1.2807,
+ "theoretical_loss": 3.3904405716281873,
+ "tokens_seen": 2561867776
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001809107321965898,
+ "loss": 1.3434,
+ "theoretical_loss": 3.3904337450516384,
+ "tokens_seen": 2561933312
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001808946840521565,
+ "loss": 1.3187,
+ "theoretical_loss": 3.3904269186986102,
+ "tokens_seen": 2561998848
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001808786359077232,
+ "loss": 1.1852,
+ "theoretical_loss": 3.39042009256909,
+ "tokens_seen": 2562064384
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001808625877632899,
+ "loss": 1.2866,
+ "theoretical_loss": 3.3904132666630646,
+ "tokens_seen": 2562129920
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001808465396188566,
+ "loss": 1.1946,
+ "theoretical_loss": 3.3904064409805206,
+ "tokens_seen": 2562195456
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018083049147442327,
+ "loss": 1.2232,
+ "theoretical_loss": 3.3903996155214458,
+ "tokens_seen": 2562260992
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018081444332998997,
+ "loss": 1.2519,
+ "theoretical_loss": 3.390392790285826,
+ "tokens_seen": 2562326528
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018079839518555667,
+ "loss": 1.227,
+ "theoretical_loss": 3.3903859652736488,
+ "tokens_seen": 2562392064
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 856477,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3286690711975098,
+ "objective/train/theoretical_loss": 3.3903791404849013,
+ "objective/train/tokens_used": 2582917600,
+ "theoretical_loss": 3.3903791404849013,
+ "tokens_seen": 2562457600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018078234704112337,
+ "loss": 1.3219,
+ "theoretical_loss": 3.3903791404849013,
+ "tokens_seen": 2562457600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018076629889669007,
+ "loss": 1.2258,
+ "theoretical_loss": 3.3903723159195707,
+ "tokens_seen": 2562523136
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018075025075225677,
+ "loss": 1.1858,
+ "theoretical_loss": 3.3903654915776436,
+ "tokens_seen": 2562588672
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018073420260782347,
+ "loss": 1.2045,
+ "theoretical_loss": 3.3903586674591066,
+ "tokens_seen": 2562654208
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018071815446339017,
+ "loss": 1.2685,
+ "theoretical_loss": 3.390351843563947,
+ "tokens_seen": 2562719744
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018070210631895687,
+ "loss": 1.272,
+ "theoretical_loss": 3.390345019892152,
+ "tokens_seen": 2562785280
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018068605817452357,
+ "loss": 1.2652,
+ "theoretical_loss": 3.3903381964437083,
+ "tokens_seen": 2562850816
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018067001003009027,
+ "loss": 1.2318,
+ "theoretical_loss": 3.3903313732186033,
+ "tokens_seen": 2562916352
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018065396188565697,
+ "loss": 1.2112,
+ "theoretical_loss": 3.3903245502168238,
+ "tokens_seen": 2562981888
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018063791374122367,
+ "loss": 1.2324,
+ "theoretical_loss": 3.3903177274383562,
+ "tokens_seen": 2563047424
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018062186559679038,
+ "loss": 1.2,
+ "theoretical_loss": 3.390310904883188,
+ "tokens_seen": 2563112960
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018060581745235708,
+ "loss": 1.2099,
+ "theoretical_loss": 3.390304082551306,
+ "tokens_seen": 2563178496
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018058976930792378,
+ "loss": 1.2464,
+ "theoretical_loss": 3.390297260442698,
+ "tokens_seen": 2563244032
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018057372116349048,
+ "loss": 1.2781,
+ "theoretical_loss": 3.3902904385573494,
+ "tokens_seen": 2563309568
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018055767301905718,
+ "loss": 1.2602,
+ "theoretical_loss": 3.3902836168952484,
+ "tokens_seen": 2563375104
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018054162487462388,
+ "loss": 1.2748,
+ "theoretical_loss": 3.3902767954563817,
+ "tokens_seen": 2563440640
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018052557673019058,
+ "loss": 1.3444,
+ "theoretical_loss": 3.390269974240736,
+ "tokens_seen": 2563506176
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018050952858575728,
+ "loss": 1.1388,
+ "theoretical_loss": 3.390263153248299,
+ "tokens_seen": 2563571712
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018049348044132398,
+ "loss": 1.2426,
+ "theoretical_loss": 3.390256332479057,
+ "tokens_seen": 2563637248
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018047743229689068,
+ "loss": 1.233,
+ "theoretical_loss": 3.390249511932997,
+ "tokens_seen": 2563702784
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018046138415245738,
+ "loss": 1.2018,
+ "theoretical_loss": 3.3902426916101067,
+ "tokens_seen": 2563768320
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018044533600802408,
+ "loss": 1.2688,
+ "theoretical_loss": 3.390235871510372,
+ "tokens_seen": 2563833856
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018042928786359078,
+ "loss": 1.2421,
+ "theoretical_loss": 3.390229051633781,
+ "tokens_seen": 2563899392
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018041323971915748,
+ "loss": 1.1542,
+ "theoretical_loss": 3.39022223198032,
+ "tokens_seen": 2563964928
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018039719157472418,
+ "loss": 1.215,
+ "theoretical_loss": 3.390215412549976,
+ "tokens_seen": 2564030464
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018038114343029088,
+ "loss": 1.2216,
+ "theoretical_loss": 3.3902085933427366,
+ "tokens_seen": 2564096000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018036509528585759,
+ "loss": 1.2335,
+ "theoretical_loss": 3.390201774358588,
+ "tokens_seen": 2564161536
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018034904714142429,
+ "loss": 1.2288,
+ "theoretical_loss": 3.3901949555975177,
+ "tokens_seen": 2564227072
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000180332998996991,
+ "loss": 1.4146,
+ "theoretical_loss": 3.3901881370595124,
+ "tokens_seen": 2564292608
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001803169508525577,
+ "loss": 1.1692,
+ "theoretical_loss": 3.3901813187445597,
+ "tokens_seen": 2564358144
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001803009027081244,
+ "loss": 1.2303,
+ "theoretical_loss": 3.390174500652646,
+ "tokens_seen": 2564423680
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802848545636911,
+ "loss": 1.2277,
+ "theoretical_loss": 3.3901676827837584,
+ "tokens_seen": 2564489216
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802688064192578,
+ "loss": 1.2241,
+ "theoretical_loss": 3.3901608651378843,
+ "tokens_seen": 2564554752
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802527582748245,
+ "loss": 1.181,
+ "theoretical_loss": 3.39015404771501,
+ "tokens_seen": 2564620288
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802367101303912,
+ "loss": 1.2339,
+ "theoretical_loss": 3.3901472305151232,
+ "tokens_seen": 2564685824
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802206619859579,
+ "loss": 1.227,
+ "theoretical_loss": 3.3901404135382105,
+ "tokens_seen": 2564751360
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001802046138415246,
+ "loss": 1.221,
+ "theoretical_loss": 3.390133596784259,
+ "tokens_seen": 2564816896
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001801885656970913,
+ "loss": 1.2349,
+ "theoretical_loss": 3.3901267802532558,
+ "tokens_seen": 2564882432
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000180172517552658,
+ "loss": 1.2044,
+ "theoretical_loss": 3.3901199639451876,
+ "tokens_seen": 2564947968
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001801564694082247,
+ "loss": 1.1673,
+ "theoretical_loss": 3.390113147860042,
+ "tokens_seen": 2565013504
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001801404212637914,
+ "loss": 1.2772,
+ "theoretical_loss": 3.3901063319978055,
+ "tokens_seen": 2565079040
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018012437311935807,
+ "loss": 1.2857,
+ "theoretical_loss": 3.3900995163584655,
+ "tokens_seen": 2565144576
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018010832497492477,
+ "loss": 1.2228,
+ "theoretical_loss": 3.3900927009420085,
+ "tokens_seen": 2565210112
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018009227683049147,
+ "loss": 1.1032,
+ "theoretical_loss": 3.390085885748422,
+ "tokens_seen": 2565275648
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018007622868605817,
+ "loss": 1.2064,
+ "theoretical_loss": 3.390079070777693,
+ "tokens_seen": 2565341184
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018006018054162487,
+ "loss": 1.2457,
+ "theoretical_loss": 3.390072256029808,
+ "tokens_seen": 2565406720
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018004413239719157,
+ "loss": 1.1927,
+ "theoretical_loss": 3.390065441504755,
+ "tokens_seen": 2565472256
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018002808425275827,
+ "loss": 1.2141,
+ "theoretical_loss": 3.39005862720252,
+ "tokens_seen": 2565537792
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00018001203610832497,
+ "loss": 1.2668,
+ "theoretical_loss": 3.3900518131230903,
+ "tokens_seen": 2565603328
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017999598796389167,
+ "loss": 1.207,
+ "theoretical_loss": 3.390044999266453,
+ "tokens_seen": 2565668864
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 857286,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3080354928970337,
+ "objective/train/theoretical_loss": 3.3900381856325956,
+ "objective/train/tokens_used": 2586194400,
+ "theoretical_loss": 3.3900381856325956,
+ "tokens_seen": 2565734400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017997993981945837,
+ "loss": 1.2539,
+ "theoretical_loss": 3.3900381856325956,
+ "tokens_seen": 2565734400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017996389167502507,
+ "loss": 1.1962,
+ "theoretical_loss": 3.3900313722215043,
+ "tokens_seen": 2565799936
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017994784353059177,
+ "loss": 1.256,
+ "theoretical_loss": 3.390024559033167,
+ "tokens_seen": 2565865472
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017993179538615848,
+ "loss": 1.2064,
+ "theoretical_loss": 3.39001774606757,
+ "tokens_seen": 2565931008
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017991574724172518,
+ "loss": 1.2455,
+ "theoretical_loss": 3.3900109333247004,
+ "tokens_seen": 2565996544
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017989969909729188,
+ "loss": 1.159,
+ "theoretical_loss": 3.3900041208045457,
+ "tokens_seen": 2566062080
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017988365095285858,
+ "loss": 1.304,
+ "theoretical_loss": 3.3899973085070925,
+ "tokens_seen": 2566127616
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001798676028084253,
+ "loss": 1.2834,
+ "theoretical_loss": 3.389990496432328,
+ "tokens_seen": 2566193152
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000179851554663992,
+ "loss": 1.2513,
+ "theoretical_loss": 3.3899836845802396,
+ "tokens_seen": 2566258688
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001798355065195587,
+ "loss": 1.2488,
+ "theoretical_loss": 3.3899768729508133,
+ "tokens_seen": 2566324224
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001798194583751254,
+ "loss": 1.2606,
+ "theoretical_loss": 3.3899700615440373,
+ "tokens_seen": 2566389760
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017980341023069208,
+ "loss": 1.0601,
+ "theoretical_loss": 3.389963250359898,
+ "tokens_seen": 2566455296
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017978736208625878,
+ "loss": 1.181,
+ "theoretical_loss": 3.3899564393983823,
+ "tokens_seen": 2566520832
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017977131394182548,
+ "loss": 1.1751,
+ "theoretical_loss": 3.389949628659478,
+ "tokens_seen": 2566586368
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017975526579739218,
+ "loss": 1.2943,
+ "theoretical_loss": 3.389942818143171,
+ "tokens_seen": 2566651904
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017973921765295888,
+ "loss": 1.2742,
+ "theoretical_loss": 3.3899360078494496,
+ "tokens_seen": 2566717440
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017972316950852558,
+ "loss": 1.193,
+ "theoretical_loss": 3.3899291977783,
+ "tokens_seen": 2566782976
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017970712136409228,
+ "loss": 1.2308,
+ "theoretical_loss": 3.389922387929709,
+ "tokens_seen": 2566848512
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017969107321965898,
+ "loss": 1.2759,
+ "theoretical_loss": 3.3899155783036647,
+ "tokens_seen": 2566914048
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017967502507522569,
+ "loss": 1.2766,
+ "theoretical_loss": 3.3899087689001535,
+ "tokens_seen": 2566979584
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017965897693079239,
+ "loss": 1.331,
+ "theoretical_loss": 3.3899019597191624,
+ "tokens_seen": 2567045120
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001796429287863591,
+ "loss": 1.2048,
+ "theoretical_loss": 3.3898951507606787,
+ "tokens_seen": 2567110656
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001796268806419258,
+ "loss": 1.2453,
+ "theoretical_loss": 3.389888342024689,
+ "tokens_seen": 2567176192
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001796108324974925,
+ "loss": 1.1761,
+ "theoretical_loss": 3.389881533511181,
+ "tokens_seen": 2567241728
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001795947843530592,
+ "loss": 1.2994,
+ "theoretical_loss": 3.3898747252201415,
+ "tokens_seen": 2567307264
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001795787362086259,
+ "loss": 1.1707,
+ "theoretical_loss": 3.389867917151557,
+ "tokens_seen": 2567372800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001795626880641926,
+ "loss": 1.1435,
+ "theoretical_loss": 3.389861109305415,
+ "tokens_seen": 2567438336
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001795466399197593,
+ "loss": 1.2181,
+ "theoretical_loss": 3.3898543016817033,
+ "tokens_seen": 2567503872
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000179530591775326,
+ "loss": 1.206,
+ "theoretical_loss": 3.3898474942804073,
+ "tokens_seen": 2567569408
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001795145436308927,
+ "loss": 1.2691,
+ "theoretical_loss": 3.3898406871015156,
+ "tokens_seen": 2567634944
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001794984954864594,
+ "loss": 1.2661,
+ "theoretical_loss": 3.3898338801450145,
+ "tokens_seen": 2567700480
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001794824473420261,
+ "loss": 1.2389,
+ "theoretical_loss": 3.389827073410891,
+ "tokens_seen": 2567766016
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001794663991975928,
+ "loss": 1.1681,
+ "theoretical_loss": 3.3898202668991324,
+ "tokens_seen": 2567831552
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001794503510531595,
+ "loss": 1.1655,
+ "theoretical_loss": 3.389813460609726,
+ "tokens_seen": 2567897088
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001794343029087262,
+ "loss": 1.315,
+ "theoretical_loss": 3.3898066545426584,
+ "tokens_seen": 2567962624
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017941825476429287,
+ "loss": 1.1877,
+ "theoretical_loss": 3.389799848697917,
+ "tokens_seen": 2568028160
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017940220661985957,
+ "loss": 1.1842,
+ "theoretical_loss": 3.389793043075488,
+ "tokens_seen": 2568093696
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017938615847542627,
+ "loss": 1.2478,
+ "theoretical_loss": 3.3897862376753602,
+ "tokens_seen": 2568159232
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017937011033099297,
+ "loss": 1.2411,
+ "theoretical_loss": 3.389779432497519,
+ "tokens_seen": 2568224768
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017935406218655967,
+ "loss": 1.2754,
+ "theoretical_loss": 3.3897726275419524,
+ "tokens_seen": 2568290304
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017933801404212637,
+ "loss": 1.1493,
+ "theoretical_loss": 3.3897658228086467,
+ "tokens_seen": 2568355840
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017932196589769307,
+ "loss": 1.1881,
+ "theoretical_loss": 3.38975901829759,
+ "tokens_seen": 2568421376
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017930591775325977,
+ "loss": 1.2138,
+ "theoretical_loss": 3.3897522140087686,
+ "tokens_seen": 2568486912
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017928986960882647,
+ "loss": 1.1994,
+ "theoretical_loss": 3.3897454099421696,
+ "tokens_seen": 2568552448
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001792738214643932,
+ "loss": 1.1672,
+ "theoretical_loss": 3.3897386060977803,
+ "tokens_seen": 2568617984
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001792577733199599,
+ "loss": 1.2991,
+ "theoretical_loss": 3.389731802475588,
+ "tokens_seen": 2568683520
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001792417251755266,
+ "loss": 1.3044,
+ "theoretical_loss": 3.3897249990755793,
+ "tokens_seen": 2568749056
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001792256770310933,
+ "loss": 1.2355,
+ "theoretical_loss": 3.3897181958977414,
+ "tokens_seen": 2568814592
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017920962888666,
+ "loss": 1.1855,
+ "theoretical_loss": 3.3897113929420617,
+ "tokens_seen": 2568880128
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001791935807422267,
+ "loss": 1.2088,
+ "theoretical_loss": 3.3897045902085265,
+ "tokens_seen": 2568945664
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 858559,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2997121810913086,
+ "objective/train/theoretical_loss": 3.389697787697124,
+ "objective/train/tokens_used": 2589471200,
+ "theoretical_loss": 3.389697787697124,
+ "tokens_seen": 2569011200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001791775325977934,
+ "loss": 1.2811,
+ "theoretical_loss": 3.389697787697124,
+ "tokens_seen": 2569011200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001791614844533601,
+ "loss": 1.247,
+ "theoretical_loss": 3.3896909854078405,
+ "tokens_seen": 2569076736
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001791454363089268,
+ "loss": 1.2906,
+ "theoretical_loss": 3.3896841833406635,
+ "tokens_seen": 2569142272
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001791293881644935,
+ "loss": 1.1754,
+ "theoretical_loss": 3.3896773814955794,
+ "tokens_seen": 2569207808
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017911334002006018,
+ "loss": 1.2316,
+ "theoretical_loss": 3.3896705798725764,
+ "tokens_seen": 2569273344
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017909729187562688,
+ "loss": 1.2195,
+ "theoretical_loss": 3.38966377847164,
+ "tokens_seen": 2569338880
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017908124373119358,
+ "loss": 1.1101,
+ "theoretical_loss": 3.389656977292759,
+ "tokens_seen": 2569404416
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017906519558676028,
+ "loss": 1.2782,
+ "theoretical_loss": 3.3896501763359193,
+ "tokens_seen": 2569469952
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017904914744232698,
+ "loss": 1.238,
+ "theoretical_loss": 3.389643375601109,
+ "tokens_seen": 2569535488
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017903309929789368,
+ "loss": 1.1112,
+ "theoretical_loss": 3.3896365750883137,
+ "tokens_seen": 2569601024
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017901705115346038,
+ "loss": 1.2564,
+ "theoretical_loss": 3.389629774797522,
+ "tokens_seen": 2569666560
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017900100300902709,
+ "loss": 1.2645,
+ "theoretical_loss": 3.38962297472872,
+ "tokens_seen": 2569732096
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017898495486459379,
+ "loss": 1.2615,
+ "theoretical_loss": 3.389616174881895,
+ "tokens_seen": 2569797632
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001789689067201605,
+ "loss": 1.2755,
+ "theoretical_loss": 3.3896093752570344,
+ "tokens_seen": 2569863168
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001789528585757272,
+ "loss": 1.2705,
+ "theoretical_loss": 3.3896025758541253,
+ "tokens_seen": 2569928704
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001789368104312939,
+ "loss": 1.2235,
+ "theoretical_loss": 3.389595776673155,
+ "tokens_seen": 2569994240
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001789207622868606,
+ "loss": 1.1948,
+ "theoretical_loss": 3.3895889777141095,
+ "tokens_seen": 2570059776
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001789047141424273,
+ "loss": 1.1355,
+ "theoretical_loss": 3.389582178976977,
+ "tokens_seen": 2570125312
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000178888665997994,
+ "loss": 1.237,
+ "theoretical_loss": 3.389575380461744,
+ "tokens_seen": 2570190848
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001788726178535607,
+ "loss": 1.2993,
+ "theoretical_loss": 3.389568582168398,
+ "tokens_seen": 2570256384
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001788565697091274,
+ "loss": 1.1414,
+ "theoretical_loss": 3.389561784096926,
+ "tokens_seen": 2570321920
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001788405215646941,
+ "loss": 1.2056,
+ "theoretical_loss": 3.389554986247315,
+ "tokens_seen": 2570387456
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001788244734202608,
+ "loss": 1.1311,
+ "theoretical_loss": 3.389548188619552,
+ "tokens_seen": 2570452992
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001788084252758275,
+ "loss": 1.223,
+ "theoretical_loss": 3.3895413912136245,
+ "tokens_seen": 2570518528
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001787923771313942,
+ "loss": 1.1989,
+ "theoretical_loss": 3.389534594029519,
+ "tokens_seen": 2570584064
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001787763289869609,
+ "loss": 1.2353,
+ "theoretical_loss": 3.389527797067223,
+ "tokens_seen": 2570649600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001787602808425276,
+ "loss": 1.2517,
+ "theoretical_loss": 3.3895210003267238,
+ "tokens_seen": 2570715136
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001787442326980943,
+ "loss": 1.1913,
+ "theoretical_loss": 3.389514203808008,
+ "tokens_seen": 2570780672
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017872818455366097,
+ "loss": 1.1575,
+ "theoretical_loss": 3.389507407511063,
+ "tokens_seen": 2570846208
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017871213640922767,
+ "loss": 1.2418,
+ "theoretical_loss": 3.389500611435876,
+ "tokens_seen": 2570911744
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017869608826479437,
+ "loss": 1.2093,
+ "theoretical_loss": 3.389493815582434,
+ "tokens_seen": 2570977280
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017868004012036107,
+ "loss": 1.2663,
+ "theoretical_loss": 3.389487019950724,
+ "tokens_seen": 2571042816
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001786639919759278,
+ "loss": 1.2734,
+ "theoretical_loss": 3.3894802245407334,
+ "tokens_seen": 2571108352
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001786479438314945,
+ "loss": 1.1743,
+ "theoretical_loss": 3.389473429352449,
+ "tokens_seen": 2571173888
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001786318956870612,
+ "loss": 1.1462,
+ "theoretical_loss": 3.3894666343858577,
+ "tokens_seen": 2571239424
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001786158475426279,
+ "loss": 1.1905,
+ "theoretical_loss": 3.3894598396409474,
+ "tokens_seen": 2571304960
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785997993981946,
+ "loss": 1.1478,
+ "theoretical_loss": 3.3894530451177047,
+ "tokens_seen": 2571370496
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785837512537613,
+ "loss": 1.1724,
+ "theoretical_loss": 3.3894462508161167,
+ "tokens_seen": 2571436032
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000178567703109328,
+ "loss": 1.2492,
+ "theoretical_loss": 3.389439456736171,
+ "tokens_seen": 2571501568
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785516549648947,
+ "loss": 1.1418,
+ "theoretical_loss": 3.389432662877854,
+ "tokens_seen": 2571567104
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785356068204614,
+ "loss": 1.2501,
+ "theoretical_loss": 3.3894258692411534,
+ "tokens_seen": 2571632640
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785195586760281,
+ "loss": 1.2435,
+ "theoretical_loss": 3.389419075826056,
+ "tokens_seen": 2571698176
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001785035105315948,
+ "loss": 1.1872,
+ "theoretical_loss": 3.3894122826325486,
+ "tokens_seen": 2571763712
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001784874623871615,
+ "loss": 1.1577,
+ "theoretical_loss": 3.389405489660619,
+ "tokens_seen": 2571829248
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001784714142427282,
+ "loss": 1.218,
+ "theoretical_loss": 3.3893986969102543,
+ "tokens_seen": 2571894784
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001784553660982949,
+ "loss": 1.2067,
+ "theoretical_loss": 3.3893919043814416,
+ "tokens_seen": 2571960320
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001784393179538616,
+ "loss": 1.2239,
+ "theoretical_loss": 3.3893851120741676,
+ "tokens_seen": 2572025856
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001784232698094283,
+ "loss": 1.1898,
+ "theoretical_loss": 3.3893783199884195,
+ "tokens_seen": 2572091392
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017840722166499498,
+ "loss": 1.2695,
+ "theoretical_loss": 3.389371528124184,
+ "tokens_seen": 2572156928
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017839117352056168,
+ "loss": 1.306,
+ "theoretical_loss": 3.3893647364814496,
+ "tokens_seen": 2572222464
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 859589,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2575843334197998,
+ "objective/train/theoretical_loss": 3.3893579450602025,
+ "objective/train/tokens_used": 2592748000,
+ "theoretical_loss": 3.3893579450602025,
+ "tokens_seen": 2572288000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017837512537612838,
+ "loss": 1.2073,
+ "theoretical_loss": 3.3893579450602025,
+ "tokens_seen": 2572288000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017835907723169508,
+ "loss": 1.1739,
+ "theoretical_loss": 3.38935115386043,
+ "tokens_seen": 2572353536
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017834302908726178,
+ "loss": 1.2198,
+ "theoretical_loss": 3.3893443628821194,
+ "tokens_seen": 2572419072
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017832698094282848,
+ "loss": 1.1979,
+ "theoretical_loss": 3.3893375721252577,
+ "tokens_seen": 2572484608
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017831093279839519,
+ "loss": 1.1908,
+ "theoretical_loss": 3.3893307815898313,
+ "tokens_seen": 2572550144
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017829488465396189,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3893239912758286,
+ "tokens_seen": 2572615680
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001782788365095286,
+ "loss": 1.3042,
+ "theoretical_loss": 3.389317201183236,
+ "tokens_seen": 2572681216
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001782627883650953,
+ "loss": 1.1483,
+ "theoretical_loss": 3.389310411312041,
+ "tokens_seen": 2572746752
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000178246740220662,
+ "loss": 1.2285,
+ "theoretical_loss": 3.38930362166223,
+ "tokens_seen": 2572812288
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001782306920762287,
+ "loss": 1.11,
+ "theoretical_loss": 3.3892968322337915,
+ "tokens_seen": 2572877824
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001782146439317954,
+ "loss": 1.1701,
+ "theoretical_loss": 3.3892900430267114,
+ "tokens_seen": 2572943360
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781985957873621,
+ "loss": 1.2147,
+ "theoretical_loss": 3.3892832540409774,
+ "tokens_seen": 2573008896
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781825476429288,
+ "loss": 1.2397,
+ "theoretical_loss": 3.389276465276576,
+ "tokens_seen": 2573074432
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781664994984955,
+ "loss": 1.215,
+ "theoretical_loss": 3.3892696767334956,
+ "tokens_seen": 2573139968
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781504513540622,
+ "loss": 1.298,
+ "theoretical_loss": 3.3892628884117224,
+ "tokens_seen": 2573205504
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781344032096289,
+ "loss": 1.1692,
+ "theoretical_loss": 3.389256100311244,
+ "tokens_seen": 2573271040
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781183550651956,
+ "loss": 1.1638,
+ "theoretical_loss": 3.389249312432047,
+ "tokens_seen": 2573336576
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001781023069207623,
+ "loss": 1.2133,
+ "theoretical_loss": 3.3892425247741187,
+ "tokens_seen": 2573402112
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000178086258776329,
+ "loss": 1.1287,
+ "theoretical_loss": 3.389235737337447,
+ "tokens_seen": 2573467648
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001780702106318957,
+ "loss": 1.2206,
+ "theoretical_loss": 3.389228950122018,
+ "tokens_seen": 2573533184
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001780541624874624,
+ "loss": 1.2474,
+ "theoretical_loss": 3.38922216312782,
+ "tokens_seen": 2573598720
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001780381143430291,
+ "loss": 1.3237,
+ "theoretical_loss": 3.3892153763548385,
+ "tokens_seen": 2573664256
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001780220661985958,
+ "loss": 1.2068,
+ "theoretical_loss": 3.3892085898030624,
+ "tokens_seen": 2573729792
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001780060180541625,
+ "loss": 1.2293,
+ "theoretical_loss": 3.389201803472478,
+ "tokens_seen": 2573795328
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001779899699097292,
+ "loss": 1.2125,
+ "theoretical_loss": 3.3891950173630723,
+ "tokens_seen": 2573860864
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001779739217652959,
+ "loss": 1.2039,
+ "theoretical_loss": 3.389188231474833,
+ "tokens_seen": 2573926400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001779578736208626,
+ "loss": 1.1849,
+ "theoretical_loss": 3.3891814458077465,
+ "tokens_seen": 2573991936
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001779418254764293,
+ "loss": 1.2409,
+ "theoretical_loss": 3.389174660361801,
+ "tokens_seen": 2574057472
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000177925777331996,
+ "loss": 1.1624,
+ "theoretical_loss": 3.389167875136983,
+ "tokens_seen": 2574123008
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001779097291875627,
+ "loss": 1.2137,
+ "theoretical_loss": 3.3891610901332796,
+ "tokens_seen": 2574188544
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778936810431294,
+ "loss": 1.2594,
+ "theoretical_loss": 3.389154305350678,
+ "tokens_seen": 2574254080
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778776328986961,
+ "loss": 1.1259,
+ "theoretical_loss": 3.389147520789166,
+ "tokens_seen": 2574319616
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778615847542628,
+ "loss": 1.1884,
+ "theoretical_loss": 3.38914073644873,
+ "tokens_seen": 2574385152
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778455366098295,
+ "loss": 1.282,
+ "theoretical_loss": 3.3891339523293573,
+ "tokens_seen": 2574450688
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778294884653962,
+ "loss": 1.2009,
+ "theoretical_loss": 3.3891271684310356,
+ "tokens_seen": 2574516224
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001778134403209629,
+ "loss": 1.2479,
+ "theoretical_loss": 3.389120384753751,
+ "tokens_seen": 2574581760
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001777973921765296,
+ "loss": 1.1603,
+ "theoretical_loss": 3.389113601297492,
+ "tokens_seen": 2574647296
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001777813440320963,
+ "loss": 1.2148,
+ "theoretical_loss": 3.389106818062245,
+ "tokens_seen": 2574712832
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000177765295887663,
+ "loss": 1.295,
+ "theoretical_loss": 3.3891000350479974,
+ "tokens_seen": 2574778368
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001777492477432297,
+ "loss": 1.1549,
+ "theoretical_loss": 3.389093252254736,
+ "tokens_seen": 2574843904
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001777331995987964,
+ "loss": 1.2427,
+ "theoretical_loss": 3.3890864696824483,
+ "tokens_seen": 2574909440
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001777171514543631,
+ "loss": 1.2204,
+ "theoretical_loss": 3.3890796873311215,
+ "tokens_seen": 2574974976
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017770110330992978,
+ "loss": 1.2795,
+ "theoretical_loss": 3.389072905200743,
+ "tokens_seen": 2575040512
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017768505516549648,
+ "loss": 1.2987,
+ "theoretical_loss": 3.3890661232912995,
+ "tokens_seen": 2575106048
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017766900702106318,
+ "loss": 1.1839,
+ "theoretical_loss": 3.389059341602778,
+ "tokens_seen": 2575171584
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017765295887662988,
+ "loss": 1.3078,
+ "theoretical_loss": 3.3890525601351666,
+ "tokens_seen": 2575237120
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017763691073219658,
+ "loss": 1.1757,
+ "theoretical_loss": 3.389045778888452,
+ "tokens_seen": 2575302656
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017762086258776329,
+ "loss": 1.2358,
+ "theoretical_loss": 3.3890389978626208,
+ "tokens_seen": 2575368192
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017760481444332999,
+ "loss": 1.1783,
+ "theoretical_loss": 3.3890322170576606,
+ "tokens_seen": 2575433728
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775887662988967,
+ "loss": 1.2251,
+ "theoretical_loss": 3.389025436473559,
+ "tokens_seen": 2575499264
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 860863,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1932480335235596,
+ "objective/train/theoretical_loss": 3.389018656110303,
+ "objective/train/tokens_used": 2596024800,
+ "theoretical_loss": 3.389018656110303,
+ "tokens_seen": 2575564800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775727181544634,
+ "loss": 1.241,
+ "theoretical_loss": 3.389018656110303,
+ "tokens_seen": 2575564800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775566700100301,
+ "loss": 1.2484,
+ "theoretical_loss": 3.3890118759678796,
+ "tokens_seen": 2575630336
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775406218655968,
+ "loss": 1.2103,
+ "theoretical_loss": 3.389005096046276,
+ "tokens_seen": 2575695872
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775245737211635,
+ "loss": 1.3035,
+ "theoretical_loss": 3.388998316345479,
+ "tokens_seen": 2575761408
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001775085255767302,
+ "loss": 1.2884,
+ "theoretical_loss": 3.388991536865477,
+ "tokens_seen": 2575826944
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001774924774322969,
+ "loss": 1.1775,
+ "theoretical_loss": 3.3889847576062557,
+ "tokens_seen": 2575892480
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017747642928786362,
+ "loss": 1.3476,
+ "theoretical_loss": 3.388977978567804,
+ "tokens_seen": 2575958016
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017746038114343032,
+ "loss": 1.3121,
+ "theoretical_loss": 3.3889711997501073,
+ "tokens_seen": 2576023552
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017744433299899702,
+ "loss": 1.1326,
+ "theoretical_loss": 3.3889644211531538,
+ "tokens_seen": 2576089088
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017742828485456372,
+ "loss": 1.139,
+ "theoretical_loss": 3.388957642776931,
+ "tokens_seen": 2576154624
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017741223671013042,
+ "loss": 1.2352,
+ "theoretical_loss": 3.3889508646214246,
+ "tokens_seen": 2576220160
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773961885656971,
+ "loss": 1.1567,
+ "theoretical_loss": 3.388944086686623,
+ "tokens_seen": 2576285696
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773801404212638,
+ "loss": 1.1914,
+ "theoretical_loss": 3.3889373089725137,
+ "tokens_seen": 2576351232
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773640922768305,
+ "loss": 1.3076,
+ "theoretical_loss": 3.388930531479083,
+ "tokens_seen": 2576416768
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773480441323972,
+ "loss": 1.2488,
+ "theoretical_loss": 3.388923754206319,
+ "tokens_seen": 2576482304
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773319959879639,
+ "loss": 1.153,
+ "theoretical_loss": 3.3889169771542083,
+ "tokens_seen": 2576547840
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001773159478435306,
+ "loss": 1.1727,
+ "theoretical_loss": 3.388910200322738,
+ "tokens_seen": 2576613376
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772998996990973,
+ "loss": 1.1959,
+ "theoretical_loss": 3.388903423711896,
+ "tokens_seen": 2576678912
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000177283851554664,
+ "loss": 1.3102,
+ "theoretical_loss": 3.388896647321668,
+ "tokens_seen": 2576744448
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772678034102307,
+ "loss": 1.2785,
+ "theoretical_loss": 3.3888898711520428,
+ "tokens_seen": 2576809984
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772517552657974,
+ "loss": 1.1868,
+ "theoretical_loss": 3.388883095203007,
+ "tokens_seen": 2576875520
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772357071213641,
+ "loss": 1.2016,
+ "theoretical_loss": 3.388876319474548,
+ "tokens_seen": 2576941056
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772196589769308,
+ "loss": 1.1861,
+ "theoretical_loss": 3.3888695439666527,
+ "tokens_seen": 2577006592
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001772036108324975,
+ "loss": 1.1948,
+ "theoretical_loss": 3.3888627686793082,
+ "tokens_seen": 2577072128
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001771875626880642,
+ "loss": 1.2177,
+ "theoretical_loss": 3.3888559936125024,
+ "tokens_seen": 2577137664
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001771715145436309,
+ "loss": 1.2739,
+ "theoretical_loss": 3.388849218766222,
+ "tokens_seen": 2577203200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001771554663991976,
+ "loss": 1.2581,
+ "theoretical_loss": 3.388842444140454,
+ "tokens_seen": 2577268736
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001771394182547643,
+ "loss": 1.2876,
+ "theoretical_loss": 3.3888356697351867,
+ "tokens_seen": 2577334272
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000177123370110331,
+ "loss": 1.1863,
+ "theoretical_loss": 3.3888288955504056,
+ "tokens_seen": 2577399808
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001771073219658977,
+ "loss": 1.1972,
+ "theoretical_loss": 3.3888221215860996,
+ "tokens_seen": 2577465344
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001770912738214644,
+ "loss": 1.1687,
+ "theoretical_loss": 3.3888153478422547,
+ "tokens_seen": 2577530880
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001770752256770311,
+ "loss": 1.2341,
+ "theoretical_loss": 3.3888085743188587,
+ "tokens_seen": 2577596416
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001770591775325978,
+ "loss": 1.0885,
+ "theoretical_loss": 3.388801801015899,
+ "tokens_seen": 2577661952
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001770431293881645,
+ "loss": 1.1525,
+ "theoretical_loss": 3.3887950279333623,
+ "tokens_seen": 2577727488
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001770270812437312,
+ "loss": 1.201,
+ "theoretical_loss": 3.3887882550712356,
+ "tokens_seen": 2577793024
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017701103309929788,
+ "loss": 1.3,
+ "theoretical_loss": 3.3887814824295073,
+ "tokens_seen": 2577858560
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017699498495486458,
+ "loss": 1.111,
+ "theoretical_loss": 3.3887747100081635,
+ "tokens_seen": 2577924096
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017697893681043128,
+ "loss": 1.2109,
+ "theoretical_loss": 3.388767937807192,
+ "tokens_seen": 2577989632
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017696288866599798,
+ "loss": 1.2526,
+ "theoretical_loss": 3.3887611658265797,
+ "tokens_seen": 2578055168
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017694684052156469,
+ "loss": 1.2274,
+ "theoretical_loss": 3.3887543940663143,
+ "tokens_seen": 2578120704
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017693079237713139,
+ "loss": 1.2911,
+ "theoretical_loss": 3.3887476225263824,
+ "tokens_seen": 2578186240
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017691474423269809,
+ "loss": 1.3194,
+ "theoretical_loss": 3.388740851206772,
+ "tokens_seen": 2578251776
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001768986960882648,
+ "loss": 1.0683,
+ "theoretical_loss": 3.3887340801074695,
+ "tokens_seen": 2578317312
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001768826479438315,
+ "loss": 1.2085,
+ "theoretical_loss": 3.388727309228462,
+ "tokens_seen": 2578382848
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017686659979939822,
+ "loss": 1.2281,
+ "theoretical_loss": 3.3887205385697383,
+ "tokens_seen": 2578448384
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017685055165496492,
+ "loss": 1.2107,
+ "theoretical_loss": 3.388713768131284,
+ "tokens_seen": 2578513920
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017683450351053162,
+ "loss": 1.2917,
+ "theoretical_loss": 3.3887069979130873,
+ "tokens_seen": 2578579456
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017681845536609832,
+ "loss": 1.2247,
+ "theoretical_loss": 3.388700227915135,
+ "tokens_seen": 2578644992
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017680240722166502,
+ "loss": 1.2004,
+ "theoretical_loss": 3.3886934581374137,
+ "tokens_seen": 2578710528
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017678635907723172,
+ "loss": 1.2369,
+ "theoretical_loss": 3.388686688579912,
+ "tokens_seen": 2578776064
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 862089,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3208363056182861,
+ "objective/train/theoretical_loss": 3.388679919242616,
+ "objective/train/tokens_used": 2599301600,
+ "theoretical_loss": 3.388679919242616,
+ "tokens_seen": 2578841600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017677031093279842,
+ "loss": 1.2881,
+ "theoretical_loss": 3.388679919242616,
+ "tokens_seen": 2578841600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017675426278836512,
+ "loss": 1.1887,
+ "theoretical_loss": 3.3886731501255136,
+ "tokens_seen": 2578907136
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017673821464393182,
+ "loss": 1.2279,
+ "theoretical_loss": 3.3886663812285924,
+ "tokens_seen": 2578972672
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017672216649949852,
+ "loss": 1.2466,
+ "theoretical_loss": 3.3886596125518382,
+ "tokens_seen": 2579038208
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017670611835506522,
+ "loss": 1.2431,
+ "theoretical_loss": 3.38865284409524,
+ "tokens_seen": 2579103744
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001766900702106319,
+ "loss": 1.1764,
+ "theoretical_loss": 3.3886460758587837,
+ "tokens_seen": 2579169280
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001766740220661986,
+ "loss": 1.3048,
+ "theoretical_loss": 3.388639307842457,
+ "tokens_seen": 2579234816
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001766579739217653,
+ "loss": 1.2055,
+ "theoretical_loss": 3.3886325400462476,
+ "tokens_seen": 2579300352
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000176641925777332,
+ "loss": 1.1889,
+ "theoretical_loss": 3.388625772470142,
+ "tokens_seen": 2579365888
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001766258776328987,
+ "loss": 1.2055,
+ "theoretical_loss": 3.3886190051141276,
+ "tokens_seen": 2579431424
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001766098294884654,
+ "loss": 1.2845,
+ "theoretical_loss": 3.388612237978192,
+ "tokens_seen": 2579496960
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765937813440321,
+ "loss": 1.1668,
+ "theoretical_loss": 3.3886054710623226,
+ "tokens_seen": 2579562496
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765777331995988,
+ "loss": 1.3261,
+ "theoretical_loss": 3.3885987043665056,
+ "tokens_seen": 2579628032
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765616850551655,
+ "loss": 1.2671,
+ "theoretical_loss": 3.3885919378907294,
+ "tokens_seen": 2579693568
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765456369107322,
+ "loss": 1.2551,
+ "theoretical_loss": 3.3885851716349813,
+ "tokens_seen": 2579759104
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765295887662989,
+ "loss": 1.2194,
+ "theoretical_loss": 3.3885784055992474,
+ "tokens_seen": 2579824640
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001765135406218656,
+ "loss": 1.1647,
+ "theoretical_loss": 3.388571639783516,
+ "tokens_seen": 2579890176
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764974924774323,
+ "loss": 1.2953,
+ "theoretical_loss": 3.3885648741877743,
+ "tokens_seen": 2579955712
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000176481444332999,
+ "loss": 1.2153,
+ "theoretical_loss": 3.388558108812009,
+ "tokens_seen": 2580021248
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764653961885657,
+ "loss": 1.2077,
+ "theoretical_loss": 3.388551343656207,
+ "tokens_seen": 2580086784
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764493480441324,
+ "loss": 1.242,
+ "theoretical_loss": 3.3885445787203574,
+ "tokens_seen": 2580152320
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764332998996991,
+ "loss": 1.1567,
+ "theoretical_loss": 3.3885378140044455,
+ "tokens_seen": 2580217856
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764172517552658,
+ "loss": 1.2589,
+ "theoretical_loss": 3.3885310495084595,
+ "tokens_seen": 2580283392
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001764012036108325,
+ "loss": 1.2567,
+ "theoretical_loss": 3.3885242852323865,
+ "tokens_seen": 2580348928
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001763851554663992,
+ "loss": 1.2486,
+ "theoretical_loss": 3.3885175211762135,
+ "tokens_seen": 2580414464
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001763691073219659,
+ "loss": 1.1785,
+ "theoretical_loss": 3.3885107573399282,
+ "tokens_seen": 2580480000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001763530591775326,
+ "loss": 1.257,
+ "theoretical_loss": 3.3885039937235177,
+ "tokens_seen": 2580545536
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001763370110330993,
+ "loss": 1.2573,
+ "theoretical_loss": 3.388497230326969,
+ "tokens_seen": 2580611072
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000176320962888666,
+ "loss": 1.2117,
+ "theoretical_loss": 3.38849046715027,
+ "tokens_seen": 2580676608
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017630491474423268,
+ "loss": 1.2809,
+ "theoretical_loss": 3.3884837041934075,
+ "tokens_seen": 2580742144
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017628886659979938,
+ "loss": 1.2693,
+ "theoretical_loss": 3.388476941456369,
+ "tokens_seen": 2580807680
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001762728184553661,
+ "loss": 1.19,
+ "theoretical_loss": 3.3884701789391416,
+ "tokens_seen": 2580873216
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001762567703109328,
+ "loss": 1.293,
+ "theoretical_loss": 3.3884634166417125,
+ "tokens_seen": 2580938752
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001762407221664995,
+ "loss": 1.2969,
+ "theoretical_loss": 3.388456654564069,
+ "tokens_seen": 2581004288
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017622467402206621,
+ "loss": 1.2026,
+ "theoretical_loss": 3.3884498927061983,
+ "tokens_seen": 2581069824
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017620862587763291,
+ "loss": 1.232,
+ "theoretical_loss": 3.3884431310680885,
+ "tokens_seen": 2581135360
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017619257773319961,
+ "loss": 1.2468,
+ "theoretical_loss": 3.3884363696497255,
+ "tokens_seen": 2581200896
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017617652958876632,
+ "loss": 1.2059,
+ "theoretical_loss": 3.388429608451098,
+ "tokens_seen": 2581266432
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017616048144433302,
+ "loss": 1.2822,
+ "theoretical_loss": 3.3884228474721922,
+ "tokens_seen": 2581331968
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017614443329989972,
+ "loss": 1.2626,
+ "theoretical_loss": 3.3884160867129953,
+ "tokens_seen": 2581397504
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017612838515546642,
+ "loss": 1.3318,
+ "theoretical_loss": 3.388409326173496,
+ "tokens_seen": 2581463040
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017611233701103312,
+ "loss": 1.2436,
+ "theoretical_loss": 3.38840256585368,
+ "tokens_seen": 2581528576
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017609628886659982,
+ "loss": 1.3282,
+ "theoretical_loss": 3.3883958057535355,
+ "tokens_seen": 2581594112
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017608024072216652,
+ "loss": 1.2166,
+ "theoretical_loss": 3.3883890458730495,
+ "tokens_seen": 2581659648
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017606419257773322,
+ "loss": 1.3145,
+ "theoretical_loss": 3.3883822862122086,
+ "tokens_seen": 2581725184
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017604814443329992,
+ "loss": 1.2569,
+ "theoretical_loss": 3.388375526771002,
+ "tokens_seen": 2581790720
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017603209628886662,
+ "loss": 1.1943,
+ "theoretical_loss": 3.388368767549415,
+ "tokens_seen": 2581856256
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017601604814443332,
+ "loss": 1.2167,
+ "theoretical_loss": 3.3883620085474355,
+ "tokens_seen": 2581921792
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017600000000000002,
+ "loss": 1.2696,
+ "theoretical_loss": 3.388355249765051,
+ "tokens_seen": 2581987328
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759839518555667,
+ "loss": 1.2522,
+ "theoretical_loss": 3.3883484912022492,
+ "tokens_seen": 2582052864
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 863113,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.0789929628372192,
+ "objective/train/theoretical_loss": 3.3883417328590166,
+ "objective/train/tokens_used": 2602578400,
+ "theoretical_loss": 3.3883417328590166,
+ "tokens_seen": 2582118400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759679037111334,
+ "loss": 1.1272,
+ "theoretical_loss": 3.3883417328590166,
+ "tokens_seen": 2582118400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759518555667001,
+ "loss": 1.2293,
+ "theoretical_loss": 3.388334974735341,
+ "tokens_seen": 2582183936
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759358074222668,
+ "loss": 1.146,
+ "theoretical_loss": 3.388328216831209,
+ "tokens_seen": 2582249472
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759197592778335,
+ "loss": 1.1692,
+ "theoretical_loss": 3.388321459146609,
+ "tokens_seen": 2582315008
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001759037111334002,
+ "loss": 1.1112,
+ "theoretical_loss": 3.3883147016815274,
+ "tokens_seen": 2582380544
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001758876629889669,
+ "loss": 1.2691,
+ "theoretical_loss": 3.388307944435952,
+ "tokens_seen": 2582446080
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001758716148445336,
+ "loss": 1.1495,
+ "theoretical_loss": 3.3883011874098696,
+ "tokens_seen": 2582511616
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001758555667001003,
+ "loss": 1.1528,
+ "theoretical_loss": 3.3882944306032683,
+ "tokens_seen": 2582577152
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000175839518555667,
+ "loss": 1.2616,
+ "theoretical_loss": 3.3882876740161345,
+ "tokens_seen": 2582642688
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001758234704112337,
+ "loss": 1.2362,
+ "theoretical_loss": 3.3882809176484563,
+ "tokens_seen": 2582708224
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001758074222668004,
+ "loss": 1.2295,
+ "theoretical_loss": 3.3882741615002203,
+ "tokens_seen": 2582773760
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757913741223671,
+ "loss": 1.2835,
+ "theoretical_loss": 3.388267405571414,
+ "tokens_seen": 2582839296
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757753259779338,
+ "loss": 1.3089,
+ "theoretical_loss": 3.3882606498620254,
+ "tokens_seen": 2582904832
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757592778335005,
+ "loss": 1.2554,
+ "theoretical_loss": 3.3882538943720406,
+ "tokens_seen": 2582970368
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757432296890672,
+ "loss": 1.3243,
+ "theoretical_loss": 3.388247139101448,
+ "tokens_seen": 2583035904
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757271815446339,
+ "loss": 1.1819,
+ "theoretical_loss": 3.388240384050234,
+ "tokens_seen": 2583101440
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001757111334002006,
+ "loss": 1.1495,
+ "theoretical_loss": 3.3882336292183868,
+ "tokens_seen": 2583166976
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001756950852557673,
+ "loss": 1.2114,
+ "theoretical_loss": 3.388226874605893,
+ "tokens_seen": 2583232512
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000175679037111334,
+ "loss": 1.2126,
+ "theoretical_loss": 3.38822012021274,
+ "tokens_seen": 2583298048
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001756629889669007,
+ "loss": 1.2051,
+ "theoretical_loss": 3.388213366038916,
+ "tokens_seen": 2583363584
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001756469408224674,
+ "loss": 1.2365,
+ "theoretical_loss": 3.388206612084407,
+ "tokens_seen": 2583429120
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001756308926780341,
+ "loss": 1.1507,
+ "theoretical_loss": 3.3881998583492012,
+ "tokens_seen": 2583494656
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001756148445336008,
+ "loss": 1.1995,
+ "theoretical_loss": 3.3881931048332854,
+ "tokens_seen": 2583560192
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001755987963891675,
+ "loss": 1.1943,
+ "theoretical_loss": 3.3881863515366475,
+ "tokens_seen": 2583625728
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001755827482447342,
+ "loss": 1.2827,
+ "theoretical_loss": 3.388179598459274,
+ "tokens_seen": 2583691264
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001755667001003009,
+ "loss": 1.2566,
+ "theoretical_loss": 3.3881728456011535,
+ "tokens_seen": 2583756800
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001755506519558676,
+ "loss": 1.3004,
+ "theoretical_loss": 3.3881660929622717,
+ "tokens_seen": 2583822336
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017553460381143431,
+ "loss": 1.2872,
+ "theoretical_loss": 3.388159340542617,
+ "tokens_seen": 2583887872
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017551855566700101,
+ "loss": 1.2414,
+ "theoretical_loss": 3.3881525883421766,
+ "tokens_seen": 2583953408
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017550250752256772,
+ "loss": 1.2828,
+ "theoretical_loss": 3.3881458363609376,
+ "tokens_seen": 2584018944
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017548645937813442,
+ "loss": 1.1387,
+ "theoretical_loss": 3.3881390845988877,
+ "tokens_seen": 2584084480
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017547041123370112,
+ "loss": 1.2826,
+ "theoretical_loss": 3.3881323330560136,
+ "tokens_seen": 2584150016
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017545436308926782,
+ "loss": 1.3971,
+ "theoretical_loss": 3.388125581732303,
+ "tokens_seen": 2584215552
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017543831494483452,
+ "loss": 1.3271,
+ "theoretical_loss": 3.3881188306277434,
+ "tokens_seen": 2584281088
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017542226680040122,
+ "loss": 1.2073,
+ "theoretical_loss": 3.3881120797423216,
+ "tokens_seen": 2584346624
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017540621865596792,
+ "loss": 1.1866,
+ "theoretical_loss": 3.3881053290760255,
+ "tokens_seen": 2584412160
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017539017051153462,
+ "loss": 1.2537,
+ "theoretical_loss": 3.388098578628842,
+ "tokens_seen": 2584477696
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017537412236710132,
+ "loss": 1.1758,
+ "theoretical_loss": 3.388091828400759,
+ "tokens_seen": 2584543232
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017535807422266802,
+ "loss": 1.2253,
+ "theoretical_loss": 3.3880850783917627,
+ "tokens_seen": 2584608768
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017534202607823472,
+ "loss": 1.2839,
+ "theoretical_loss": 3.388078328601842,
+ "tokens_seen": 2584674304
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017532597793380142,
+ "loss": 1.175,
+ "theoretical_loss": 3.388071579030983,
+ "tokens_seen": 2584739840
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017530992978936812,
+ "loss": 1.3016,
+ "theoretical_loss": 3.3880648296791733,
+ "tokens_seen": 2584805376
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017529388164493482,
+ "loss": 1.2856,
+ "theoretical_loss": 3.3880580805464007,
+ "tokens_seen": 2584870912
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001752778335005015,
+ "loss": 1.0882,
+ "theoretical_loss": 3.388051331632652,
+ "tokens_seen": 2584936448
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001752617853560682,
+ "loss": 1.2157,
+ "theoretical_loss": 3.388044582937915,
+ "tokens_seen": 2585001984
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001752457372116349,
+ "loss": 1.1479,
+ "theoretical_loss": 3.388037834462177,
+ "tokens_seen": 2585067520
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001752296890672016,
+ "loss": 1.2277,
+ "theoretical_loss": 3.388031086205425,
+ "tokens_seen": 2585133056
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001752136409227683,
+ "loss": 1.1536,
+ "theoretical_loss": 3.388024338167646,
+ "tokens_seen": 2585198592
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000175197592778335,
+ "loss": 1.137,
+ "theoretical_loss": 3.388017590348828,
+ "tokens_seen": 2585264128
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751815446339017,
+ "loss": 1.2411,
+ "theoretical_loss": 3.3880108427489586,
+ "tokens_seen": 2585329664
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 864353,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1530442237854004,
+ "objective/train/theoretical_loss": 3.3880040953680246,
+ "objective/train/tokens_used": 2605855200,
+ "theoretical_loss": 3.3880040953680246,
+ "tokens_seen": 2585395200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751654964894684,
+ "loss": 1.146,
+ "theoretical_loss": 3.3880040953680246,
+ "tokens_seen": 2585395200
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751494483450351,
+ "loss": 1.2946,
+ "theoretical_loss": 3.3879973482060137,
+ "tokens_seen": 2585460736
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751334002006018,
+ "loss": 1.1847,
+ "theoretical_loss": 3.3879906012629126,
+ "tokens_seen": 2585526272
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751173520561685,
+ "loss": 1.2037,
+ "theoretical_loss": 3.387983854538709,
+ "tokens_seen": 2585591808
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001751013039117352,
+ "loss": 1.232,
+ "theoretical_loss": 3.3879771080333905,
+ "tokens_seen": 2585657344
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001750852557673019,
+ "loss": 1.2301,
+ "theoretical_loss": 3.3879703617469445,
+ "tokens_seen": 2585722880
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017506920762286863,
+ "loss": 1.1916,
+ "theoretical_loss": 3.3879636156793578,
+ "tokens_seen": 2585788416
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017505315947843533,
+ "loss": 1.1821,
+ "theoretical_loss": 3.387956869830618,
+ "tokens_seen": 2585853952
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017503711133400203,
+ "loss": 1.2764,
+ "theoretical_loss": 3.387950124200713,
+ "tokens_seen": 2585919488
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017502106318956873,
+ "loss": 1.1935,
+ "theoretical_loss": 3.3879433787896294,
+ "tokens_seen": 2585985024
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017500501504513543,
+ "loss": 1.2317,
+ "theoretical_loss": 3.387936633597355,
+ "tokens_seen": 2586050560
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017498896690070214,
+ "loss": 1.2129,
+ "theoretical_loss": 3.387929888623877,
+ "tokens_seen": 2586116096
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001749729187562688,
+ "loss": 1.2843,
+ "theoretical_loss": 3.387923143869182,
+ "tokens_seen": 2586181632
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001749568706118355,
+ "loss": 1.3108,
+ "theoretical_loss": 3.387916399333259,
+ "tokens_seen": 2586247168
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001749408224674022,
+ "loss": 1.0919,
+ "theoretical_loss": 3.3879096550160943,
+ "tokens_seen": 2586312704
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001749247743229689,
+ "loss": 1.2668,
+ "theoretical_loss": 3.3879029109176755,
+ "tokens_seen": 2586378240
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001749087261785356,
+ "loss": 1.2245,
+ "theoretical_loss": 3.3878961670379897,
+ "tokens_seen": 2586443776
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001748926780341023,
+ "loss": 1.2419,
+ "theoretical_loss": 3.3878894233770245,
+ "tokens_seen": 2586509312
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000174876629889669,
+ "loss": 1.2655,
+ "theoretical_loss": 3.3878826799347674,
+ "tokens_seen": 2586574848
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001748605817452357,
+ "loss": 1.1806,
+ "theoretical_loss": 3.3878759367112057,
+ "tokens_seen": 2586640384
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017484453360080241,
+ "loss": 1.1519,
+ "theoretical_loss": 3.3878691937063263,
+ "tokens_seen": 2586705920
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017482848545636911,
+ "loss": 1.1578,
+ "theoretical_loss": 3.387862450920117,
+ "tokens_seen": 2586771456
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017481243731193582,
+ "loss": 1.1833,
+ "theoretical_loss": 3.387855708352565,
+ "tokens_seen": 2586836992
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017479638916750252,
+ "loss": 1.2842,
+ "theoretical_loss": 3.3878489660036584,
+ "tokens_seen": 2586902528
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017478034102306922,
+ "loss": 1.2204,
+ "theoretical_loss": 3.3878422238733834,
+ "tokens_seen": 2586968064
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017476429287863592,
+ "loss": 1.307,
+ "theoretical_loss": 3.3878354819617282,
+ "tokens_seen": 2587033600
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017474824473420262,
+ "loss": 1.2557,
+ "theoretical_loss": 3.3878287402686795,
+ "tokens_seen": 2587099136
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017473219658976932,
+ "loss": 1.2095,
+ "theoretical_loss": 3.3878219987942253,
+ "tokens_seen": 2587164672
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017471614844533602,
+ "loss": 1.258,
+ "theoretical_loss": 3.3878152575383527,
+ "tokens_seen": 2587230208
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017470010030090272,
+ "loss": 1.329,
+ "theoretical_loss": 3.387808516501049,
+ "tokens_seen": 2587295744
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017468405215646942,
+ "loss": 1.2254,
+ "theoretical_loss": 3.387801775682302,
+ "tokens_seen": 2587361280
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017466800401203612,
+ "loss": 1.2582,
+ "theoretical_loss": 3.3877950350820987,
+ "tokens_seen": 2587426816
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017465195586760282,
+ "loss": 1.3002,
+ "theoretical_loss": 3.3877882947004263,
+ "tokens_seen": 2587492352
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017463590772316952,
+ "loss": 1.2101,
+ "theoretical_loss": 3.387781554537273,
+ "tokens_seen": 2587557888
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017461985957873622,
+ "loss": 1.3312,
+ "theoretical_loss": 3.3877748145926248,
+ "tokens_seen": 2587623424
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017460381143430292,
+ "loss": 1.1435,
+ "theoretical_loss": 3.3877680748664707,
+ "tokens_seen": 2587688960
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001745877632898696,
+ "loss": 1.2259,
+ "theoretical_loss": 3.387761335358797,
+ "tokens_seen": 2587754496
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001745717151454363,
+ "loss": 1.2851,
+ "theoretical_loss": 3.387754596069591,
+ "tokens_seen": 2587820032
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000174555667001003,
+ "loss": 1.2391,
+ "theoretical_loss": 3.387747856998841,
+ "tokens_seen": 2587885568
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001745396188565697,
+ "loss": 1.12,
+ "theoretical_loss": 3.3877411181465336,
+ "tokens_seen": 2587951104
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001745235707121364,
+ "loss": 1.2151,
+ "theoretical_loss": 3.387734379512656,
+ "tokens_seen": 2588016640
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001745075225677031,
+ "loss": 1.2787,
+ "theoretical_loss": 3.3877276410971966,
+ "tokens_seen": 2588082176
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001744914744232698,
+ "loss": 1.2199,
+ "theoretical_loss": 3.387720902900142,
+ "tokens_seen": 2588147712
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001744754262788365,
+ "loss": 1.1933,
+ "theoretical_loss": 3.3877141649214795,
+ "tokens_seen": 2588213248
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017445937813440323,
+ "loss": 1.2146,
+ "theoretical_loss": 3.387707427161197,
+ "tokens_seen": 2588278784
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017444332998996993,
+ "loss": 1.2288,
+ "theoretical_loss": 3.387700689619282,
+ "tokens_seen": 2588344320
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017442728184553663,
+ "loss": 1.3002,
+ "theoretical_loss": 3.387693952295721,
+ "tokens_seen": 2588409856
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017441123370110333,
+ "loss": 1.1622,
+ "theoretical_loss": 3.3876872151905024,
+ "tokens_seen": 2588475392
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017439518555667003,
+ "loss": 1.2999,
+ "theoretical_loss": 3.387680478303613,
+ "tokens_seen": 2588540928
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017437913741223673,
+ "loss": 1.1932,
+ "theoretical_loss": 3.387673741635041,
+ "tokens_seen": 2588606464
+ },
+ {
+ "epoch": 0.78,
+ "objective/train/docs_used": 865338,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2663666009902954,
+ "objective/train/theoretical_loss": 3.387667005184772,
+ "objective/train/tokens_used": 2609132000,
+ "theoretical_loss": 3.387667005184772,
+ "tokens_seen": 2588672000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017436308926780343,
+ "loss": 1.2422,
+ "theoretical_loss": 3.387667005184772,
+ "tokens_seen": 2588672000
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017434704112337013,
+ "loss": 1.2062,
+ "theoretical_loss": 3.3876602689527955,
+ "tokens_seen": 2588737536
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017433099297893683,
+ "loss": 1.2252,
+ "theoretical_loss": 3.3876535329390975,
+ "tokens_seen": 2588803072
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017431494483450353,
+ "loss": 1.2304,
+ "theoretical_loss": 3.387646797143666,
+ "tokens_seen": 2588868608
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017429889669007024,
+ "loss": 1.2406,
+ "theoretical_loss": 3.387640061566488,
+ "tokens_seen": 2588934144
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017428284854563694,
+ "loss": 1.21,
+ "theoretical_loss": 3.3876333262075518,
+ "tokens_seen": 2588999680
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001742668004012036,
+ "loss": 1.2748,
+ "theoretical_loss": 3.3876265910668435,
+ "tokens_seen": 2589065216
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001742507522567703,
+ "loss": 1.2559,
+ "theoretical_loss": 3.387619856144352,
+ "tokens_seen": 2589130752
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.000174234704112337,
+ "loss": 1.2898,
+ "theoretical_loss": 3.387613121440063,
+ "tokens_seen": 2589196288
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001742186559679037,
+ "loss": 1.3076,
+ "theoretical_loss": 3.3876063869539648,
+ "tokens_seen": 2589261824
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001742026078234704,
+ "loss": 1.1977,
+ "theoretical_loss": 3.3875996526860455,
+ "tokens_seen": 2589327360
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.0001741865596790371,
+ "loss": 1.2087,
+ "theoretical_loss": 3.387592918636291,
+ "tokens_seen": 2589392896
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017417051153460381,
+ "loss": 1.2353,
+ "theoretical_loss": 3.3875861848046904,
+ "tokens_seen": 2589458432
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017415446339017051,
+ "loss": 1.213,
+ "theoretical_loss": 3.38757945119123,
+ "tokens_seen": 2589523968
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017413841524573721,
+ "loss": 1.2511,
+ "theoretical_loss": 3.387572717795897,
+ "tokens_seen": 2589589504
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017412236710130392,
+ "loss": 1.2558,
+ "theoretical_loss": 3.3875659846186794,
+ "tokens_seen": 2589655040
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017410631895687062,
+ "loss": 1.1977,
+ "theoretical_loss": 3.3875592516595647,
+ "tokens_seen": 2589720576
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017409027081243732,
+ "loss": 1.1953,
+ "theoretical_loss": 3.38755251891854,
+ "tokens_seen": 2589786112
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017407422266800402,
+ "loss": 1.3006,
+ "theoretical_loss": 3.3875457863955925,
+ "tokens_seen": 2589851648
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017405817452357072,
+ "loss": 1.2918,
+ "theoretical_loss": 3.3875390540907104,
+ "tokens_seen": 2589917184
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017404212637913742,
+ "loss": 1.2164,
+ "theoretical_loss": 3.3875323220038807,
+ "tokens_seen": 2589982720
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017402607823470412,
+ "loss": 1.1891,
+ "theoretical_loss": 3.3875255901350902,
+ "tokens_seen": 2590048256
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017401003009027082,
+ "loss": 1.2296,
+ "theoretical_loss": 3.387518858484327,
+ "tokens_seen": 2590113792
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017399398194583752,
+ "loss": 1.1705,
+ "theoretical_loss": 3.387512127051579,
+ "tokens_seen": 2590179328
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017397793380140422,
+ "loss": 1.1944,
+ "theoretical_loss": 3.3875053958368326,
+ "tokens_seen": 2590244864
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017396188565697092,
+ "loss": 1.2131,
+ "theoretical_loss": 3.387498664840076,
+ "tokens_seen": 2590310400
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017394583751253762,
+ "loss": 1.2224,
+ "theoretical_loss": 3.3874919340612957,
+ "tokens_seen": 2590375936
+ },
+ {
+ "epoch": 0.78,
+ "learning_rate": 0.00017392978936810432,
+ "loss": 1.2181,
+ "theoretical_loss": 3.3874852035004803,
+ "tokens_seen": 2590441472
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017391374122367102,
+ "loss": 1.2253,
+ "theoretical_loss": 3.387478473157616,
+ "tokens_seen": 2590507008
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017389769307923772,
+ "loss": 1.2033,
+ "theoretical_loss": 3.387471743032691,
+ "tokens_seen": 2590572544
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001738816449348044,
+ "loss": 1.2208,
+ "theoretical_loss": 3.387465013125693,
+ "tokens_seen": 2590638080
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017386559679037113,
+ "loss": 1.1451,
+ "theoretical_loss": 3.387458283436609,
+ "tokens_seen": 2590703616
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017384954864593783,
+ "loss": 1.2017,
+ "theoretical_loss": 3.3874515539654264,
+ "tokens_seen": 2590769152
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017383350050150453,
+ "loss": 1.2549,
+ "theoretical_loss": 3.3874448247121327,
+ "tokens_seen": 2590834688
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017381745235707123,
+ "loss": 1.1609,
+ "theoretical_loss": 3.387438095676715,
+ "tokens_seen": 2590900224
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017380140421263793,
+ "loss": 1.2214,
+ "theoretical_loss": 3.387431366859161,
+ "tokens_seen": 2590965760
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017378535606820463,
+ "loss": 1.2006,
+ "theoretical_loss": 3.387424638259459,
+ "tokens_seen": 2591031296
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017376930792377133,
+ "loss": 1.2083,
+ "theoretical_loss": 3.3874179098775947,
+ "tokens_seen": 2591096832
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017375325977933803,
+ "loss": 1.3134,
+ "theoretical_loss": 3.3874111817135573,
+ "tokens_seen": 2591162368
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017373721163490473,
+ "loss": 1.2511,
+ "theoretical_loss": 3.3874044537673327,
+ "tokens_seen": 2591227904
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017372116349047143,
+ "loss": 1.2761,
+ "theoretical_loss": 3.3873977260389094,
+ "tokens_seen": 2591293440
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017370511534603813,
+ "loss": 1.1932,
+ "theoretical_loss": 3.387390998528274,
+ "tokens_seen": 2591358976
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017368906720160483,
+ "loss": 1.2793,
+ "theoretical_loss": 3.3873842712354154,
+ "tokens_seen": 2591424512
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017367301905717153,
+ "loss": 1.2113,
+ "theoretical_loss": 3.3873775441603193,
+ "tokens_seen": 2591490048
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017365697091273823,
+ "loss": 1.3129,
+ "theoretical_loss": 3.3873708173029744,
+ "tokens_seen": 2591555584
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017364092276830493,
+ "loss": 1.1931,
+ "theoretical_loss": 3.3873640906633673,
+ "tokens_seen": 2591621120
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017362487462387164,
+ "loss": 1.2458,
+ "theoretical_loss": 3.387357364241486,
+ "tokens_seen": 2591686656
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017360882647943834,
+ "loss": 1.235,
+ "theoretical_loss": 3.3873506380373177,
+ "tokens_seen": 2591752192
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017359277833500504,
+ "loss": 1.2025,
+ "theoretical_loss": 3.38734391205085,
+ "tokens_seen": 2591817728
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017357673019057174,
+ "loss": 1.2485,
+ "theoretical_loss": 3.3873371862820703,
+ "tokens_seen": 2591883264
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 866617,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.222945213317871,
+ "objective/train/theoretical_loss": 3.3873304607309658,
+ "objective/train/tokens_used": 2612408800,
+ "theoretical_loss": 3.3873304607309658,
+ "tokens_seen": 2591948800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001735606820461384,
+ "loss": 1.2603,
+ "theoretical_loss": 3.3873304607309658,
+ "tokens_seen": 2591948800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001735446339017051,
+ "loss": 1.2435,
+ "theoretical_loss": 3.387323735397524,
+ "tokens_seen": 2592014336
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001735285857572718,
+ "loss": 1.3102,
+ "theoretical_loss": 3.387317010281733,
+ "tokens_seen": 2592079872
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001735125376128385,
+ "loss": 1.2392,
+ "theoretical_loss": 3.3873102853835793,
+ "tokens_seen": 2592145408
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001734964894684052,
+ "loss": 1.2833,
+ "theoretical_loss": 3.3873035607030513,
+ "tokens_seen": 2592210944
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017348044132397191,
+ "loss": 1.1937,
+ "theoretical_loss": 3.3872968362401354,
+ "tokens_seen": 2592276480
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017346439317953861,
+ "loss": 1.2002,
+ "theoretical_loss": 3.38729011199482,
+ "tokens_seen": 2592342016
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017344834503510532,
+ "loss": 1.2345,
+ "theoretical_loss": 3.3872833879670923,
+ "tokens_seen": 2592407552
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017343229689067202,
+ "loss": 1.2141,
+ "theoretical_loss": 3.3872766641569396,
+ "tokens_seen": 2592473088
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017341624874623872,
+ "loss": 1.123,
+ "theoretical_loss": 3.387269940564349,
+ "tokens_seen": 2592538624
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017340020060180542,
+ "loss": 1.2176,
+ "theoretical_loss": 3.3872632171893087,
+ "tokens_seen": 2592604160
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017338415245737212,
+ "loss": 1.1603,
+ "theoretical_loss": 3.387256494031806,
+ "tokens_seen": 2592669696
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017336810431293882,
+ "loss": 1.2691,
+ "theoretical_loss": 3.387249771091828,
+ "tokens_seen": 2592735232
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017335205616850552,
+ "loss": 1.2367,
+ "theoretical_loss": 3.387243048369362,
+ "tokens_seen": 2592800768
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017333600802407222,
+ "loss": 1.1419,
+ "theoretical_loss": 3.3872363258643965,
+ "tokens_seen": 2592866304
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017331995987963892,
+ "loss": 1.2271,
+ "theoretical_loss": 3.387229603576918,
+ "tokens_seen": 2592931840
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017330391173520562,
+ "loss": 1.1657,
+ "theoretical_loss": 3.387222881506914,
+ "tokens_seen": 2592997376
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017328786359077232,
+ "loss": 1.2565,
+ "theoretical_loss": 3.3872161596543724,
+ "tokens_seen": 2593062912
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017327181544633905,
+ "loss": 1.1884,
+ "theoretical_loss": 3.3872094380192808,
+ "tokens_seen": 2593128448
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017325576730190572,
+ "loss": 1.2188,
+ "theoretical_loss": 3.3872027166016263,
+ "tokens_seen": 2593193984
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017323971915747242,
+ "loss": 1.1616,
+ "theoretical_loss": 3.387195995401396,
+ "tokens_seen": 2593259520
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017322367101303912,
+ "loss": 1.3137,
+ "theoretical_loss": 3.387189274418578,
+ "tokens_seen": 2593325056
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017320762286860582,
+ "loss": 1.1413,
+ "theoretical_loss": 3.38718255365316,
+ "tokens_seen": 2593390592
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017319157472417253,
+ "loss": 1.1535,
+ "theoretical_loss": 3.3871758331051285,
+ "tokens_seen": 2593456128
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017317552657973923,
+ "loss": 1.2339,
+ "theoretical_loss": 3.3871691127744716,
+ "tokens_seen": 2593521664
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017315947843530593,
+ "loss": 1.2153,
+ "theoretical_loss": 3.387162392661177,
+ "tokens_seen": 2593587200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017314343029087263,
+ "loss": 1.1137,
+ "theoretical_loss": 3.387155672765232,
+ "tokens_seen": 2593652736
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017312738214643933,
+ "loss": 1.223,
+ "theoretical_loss": 3.3871489530866232,
+ "tokens_seen": 2593718272
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017311133400200603,
+ "loss": 1.2706,
+ "theoretical_loss": 3.3871422336253394,
+ "tokens_seen": 2593783808
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017309528585757273,
+ "loss": 1.2182,
+ "theoretical_loss": 3.3871355143813675,
+ "tokens_seen": 2593849344
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017309528585757273,
+ "loss": 1.1527,
+ "theoretical_loss": 3.387128795354695,
+ "tokens_seen": 2593914880
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017307923771313943,
+ "loss": 1.2436,
+ "theoretical_loss": 3.3871220765453094,
+ "tokens_seen": 2593980416
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017306318956870613,
+ "loss": 1.2908,
+ "theoretical_loss": 3.3871153579531983,
+ "tokens_seen": 2594045952
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017304714142427283,
+ "loss": 1.4248,
+ "theoretical_loss": 3.3871086395783485,
+ "tokens_seen": 2594111488
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017303109327983953,
+ "loss": 1.2007,
+ "theoretical_loss": 3.3871019214207485,
+ "tokens_seen": 2594177024
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017301504513540623,
+ "loss": 1.2614,
+ "theoretical_loss": 3.3870952034803854,
+ "tokens_seen": 2594242560
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017299899699097293,
+ "loss": 1.2362,
+ "theoretical_loss": 3.3870884857572463,
+ "tokens_seen": 2594308096
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017298294884653963,
+ "loss": 1.2607,
+ "theoretical_loss": 3.3870817682513192,
+ "tokens_seen": 2594373632
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017296690070210633,
+ "loss": 1.1894,
+ "theoretical_loss": 3.3870750509625918,
+ "tokens_seen": 2594439168
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017295085255767303,
+ "loss": 1.1638,
+ "theoretical_loss": 3.3870683338910506,
+ "tokens_seen": 2594504704
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017293480441323974,
+ "loss": 1.212,
+ "theoretical_loss": 3.387061617036684,
+ "tokens_seen": 2594570240
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017291875626880644,
+ "loss": 1.2823,
+ "theoretical_loss": 3.3870549003994785,
+ "tokens_seen": 2594635776
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017290270812437314,
+ "loss": 1.1624,
+ "theoretical_loss": 3.387048183979423,
+ "tokens_seen": 2594701312
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017288665997993984,
+ "loss": 1.1834,
+ "theoretical_loss": 3.387041467776504,
+ "tokens_seen": 2594766848
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001728706118355065,
+ "loss": 1.309,
+ "theoretical_loss": 3.3870347517907096,
+ "tokens_seen": 2594832384
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001728545636910732,
+ "loss": 1.2446,
+ "theoretical_loss": 3.3870280360220266,
+ "tokens_seen": 2594897920
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001728385155466399,
+ "loss": 1.255,
+ "theoretical_loss": 3.387021320470443,
+ "tokens_seen": 2594963456
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001728224674022066,
+ "loss": 1.3284,
+ "theoretical_loss": 3.387014605135946,
+ "tokens_seen": 2595028992
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001728064192577733,
+ "loss": 1.2686,
+ "theoretical_loss": 3.3870078900185234,
+ "tokens_seen": 2595094528
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017279037111334001,
+ "loss": 1.2053,
+ "theoretical_loss": 3.3870011751181623,
+ "tokens_seen": 2595160064
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 867500,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2935428619384766,
+ "objective/train/theoretical_loss": 3.3869944604348508,
+ "objective/train/tokens_used": 2615685600,
+ "theoretical_loss": 3.3869944604348508,
+ "tokens_seen": 2595225600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017277432296890671,
+ "loss": 1.2201,
+ "theoretical_loss": 3.3869944604348508,
+ "tokens_seen": 2595225600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017275827482447342,
+ "loss": 1.3862,
+ "theoretical_loss": 3.3869877459685758,
+ "tokens_seen": 2595291136
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017274222668004012,
+ "loss": 1.2416,
+ "theoretical_loss": 3.3869810317193254,
+ "tokens_seen": 2595356672
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017272617853560682,
+ "loss": 1.1229,
+ "theoretical_loss": 3.3869743176870863,
+ "tokens_seen": 2595422208
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017271013039117352,
+ "loss": 1.2682,
+ "theoretical_loss": 3.386967603871847,
+ "tokens_seen": 2595487744
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017269408224674022,
+ "loss": 1.1596,
+ "theoretical_loss": 3.386960890273594,
+ "tokens_seen": 2595553280
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017267803410230692,
+ "loss": 1.3222,
+ "theoretical_loss": 3.3869541768923153,
+ "tokens_seen": 2595618816
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017266198595787365,
+ "loss": 1.16,
+ "theoretical_loss": 3.386947463727999,
+ "tokens_seen": 2595684352
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017264593781344035,
+ "loss": 1.2224,
+ "theoretical_loss": 3.3869407507806315,
+ "tokens_seen": 2595749888
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017262988966900705,
+ "loss": 1.3072,
+ "theoretical_loss": 3.386934038050201,
+ "tokens_seen": 2595815424
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017261384152457375,
+ "loss": 1.1697,
+ "theoretical_loss": 3.3869273255366945,
+ "tokens_seen": 2595880960
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017259779338014045,
+ "loss": 1.1951,
+ "theoretical_loss": 3.3869206132401004,
+ "tokens_seen": 2595946496
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017258174523570715,
+ "loss": 1.2609,
+ "theoretical_loss": 3.3869139011604052,
+ "tokens_seen": 2596012032
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017256569709127385,
+ "loss": 1.3575,
+ "theoretical_loss": 3.386907189297597,
+ "tokens_seen": 2596077568
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017254964894684052,
+ "loss": 1.3009,
+ "theoretical_loss": 3.3869004776516634,
+ "tokens_seen": 2596143104
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017253360080240722,
+ "loss": 1.2669,
+ "theoretical_loss": 3.3868937662225918,
+ "tokens_seen": 2596208640
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017251755265797392,
+ "loss": 1.3154,
+ "theoretical_loss": 3.3868870550103694,
+ "tokens_seen": 2596274176
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017250150451354063,
+ "loss": 1.2121,
+ "theoretical_loss": 3.3868803440149837,
+ "tokens_seen": 2596339712
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017248545636910733,
+ "loss": 1.2473,
+ "theoretical_loss": 3.386873633236423,
+ "tokens_seen": 2596405248
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017246940822467403,
+ "loss": 1.3068,
+ "theoretical_loss": 3.386866922674674,
+ "tokens_seen": 2596470784
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017245336008024073,
+ "loss": 1.25,
+ "theoretical_loss": 3.3868602123297245,
+ "tokens_seen": 2596536320
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017243731193580743,
+ "loss": 1.2173,
+ "theoretical_loss": 3.3868535022015624,
+ "tokens_seen": 2596601856
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017242126379137413,
+ "loss": 1.2819,
+ "theoretical_loss": 3.3868467922901746,
+ "tokens_seen": 2596667392
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017240521564694083,
+ "loss": 1.2966,
+ "theoretical_loss": 3.386840082595549,
+ "tokens_seen": 2596732928
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017238916750250753,
+ "loss": 1.2212,
+ "theoretical_loss": 3.386833373117673,
+ "tokens_seen": 2596798464
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017237311935807423,
+ "loss": 1.2407,
+ "theoretical_loss": 3.386826663856534,
+ "tokens_seen": 2596864000
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017235707121364093,
+ "loss": 1.2206,
+ "theoretical_loss": 3.3868199548121196,
+ "tokens_seen": 2596929536
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017234102306920763,
+ "loss": 1.2696,
+ "theoretical_loss": 3.386813245984418,
+ "tokens_seen": 2596995072
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017232497492477433,
+ "loss": 1.2916,
+ "theoretical_loss": 3.386806537373415,
+ "tokens_seen": 2597060608
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017230892678034103,
+ "loss": 1.3016,
+ "theoretical_loss": 3.3867998289791004,
+ "tokens_seen": 2597126144
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017229287863590773,
+ "loss": 1.251,
+ "theoretical_loss": 3.3867931208014603,
+ "tokens_seen": 2597191680
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017227683049147443,
+ "loss": 1.1992,
+ "theoretical_loss": 3.3867864128404825,
+ "tokens_seen": 2597257216
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017226078234704113,
+ "loss": 1.1278,
+ "theoretical_loss": 3.386779705096154,
+ "tokens_seen": 2597322752
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017224473420260784,
+ "loss": 1.1856,
+ "theoretical_loss": 3.386772997568464,
+ "tokens_seen": 2597388288
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017222868605817454,
+ "loss": 1.3132,
+ "theoretical_loss": 3.3867662902573983,
+ "tokens_seen": 2597453824
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017221263791374124,
+ "loss": 1.3362,
+ "theoretical_loss": 3.386759583162945,
+ "tokens_seen": 2597519360
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017219658976930794,
+ "loss": 1.2204,
+ "theoretical_loss": 3.386752876285092,
+ "tokens_seen": 2597584896
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017218054162487464,
+ "loss": 1.3016,
+ "theoretical_loss": 3.3867461696238266,
+ "tokens_seen": 2597650432
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001721644934804413,
+ "loss": 1.1612,
+ "theoretical_loss": 3.3867394631791363,
+ "tokens_seen": 2597715968
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.000172148445336008,
+ "loss": 1.2632,
+ "theoretical_loss": 3.3867327569510084,
+ "tokens_seen": 2597781504
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001721323971915747,
+ "loss": 1.3071,
+ "theoretical_loss": 3.386726050939431,
+ "tokens_seen": 2597847040
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.0001721163490471414,
+ "loss": 1.1998,
+ "theoretical_loss": 3.386719345144391,
+ "tokens_seen": 2597912576
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017210030090270811,
+ "loss": 1.1001,
+ "theoretical_loss": 3.386712639565877,
+ "tokens_seen": 2597978112
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017208425275827481,
+ "loss": 1.1156,
+ "theoretical_loss": 3.3867059342038752,
+ "tokens_seen": 2598043648
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017206820461384154,
+ "loss": 1.1023,
+ "theoretical_loss": 3.3866992290583737,
+ "tokens_seen": 2598109184
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017205215646940824,
+ "loss": 1.225,
+ "theoretical_loss": 3.3866925241293604,
+ "tokens_seen": 2598174720
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017203610832497494,
+ "loss": 1.1384,
+ "theoretical_loss": 3.386685819416823,
+ "tokens_seen": 2598240256
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017202006018054164,
+ "loss": 1.1971,
+ "theoretical_loss": 3.3866791149207476,
+ "tokens_seen": 2598305792
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017200401203610835,
+ "loss": 1.2702,
+ "theoretical_loss": 3.3866724106411237,
+ "tokens_seen": 2598371328
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017198796389167505,
+ "loss": 1.2759,
+ "theoretical_loss": 3.3866657065779373,
+ "tokens_seen": 2598436864
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 868777,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1186769008636475,
+ "objective/train/theoretical_loss": 3.386659002731177,
+ "objective/train/tokens_used": 2618962400,
+ "theoretical_loss": 3.386659002731177,
+ "tokens_seen": 2598502400
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017197191574724175,
+ "loss": 1.2016,
+ "theoretical_loss": 3.386659002731177,
+ "tokens_seen": 2598502400
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017195586760280845,
+ "loss": 1.142,
+ "theoretical_loss": 3.38665229910083,
+ "tokens_seen": 2598567936
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017193981945837515,
+ "loss": 1.3139,
+ "theoretical_loss": 3.3866455956868835,
+ "tokens_seen": 2598633472
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017192377131394185,
+ "loss": 1.2091,
+ "theoretical_loss": 3.3866388924893256,
+ "tokens_seen": 2598699008
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017190772316950855,
+ "loss": 1.0186,
+ "theoretical_loss": 3.386632189508144,
+ "tokens_seen": 2598764544
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017189167502507525,
+ "loss": 1.2268,
+ "theoretical_loss": 3.386625486743325,
+ "tokens_seen": 2598830080
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017187562688064195,
+ "loss": 1.1567,
+ "theoretical_loss": 3.386618784194858,
+ "tokens_seen": 2598895616
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017185957873620865,
+ "loss": 1.2143,
+ "theoretical_loss": 3.3866120818627286,
+ "tokens_seen": 2598961152
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017184353059177532,
+ "loss": 1.1921,
+ "theoretical_loss": 3.3866053797469258,
+ "tokens_seen": 2599026688
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017182748244734202,
+ "loss": 1.2387,
+ "theoretical_loss": 3.386598677847437,
+ "tokens_seen": 2599092224
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017181143430290873,
+ "loss": 1.1933,
+ "theoretical_loss": 3.386591976164249,
+ "tokens_seen": 2599157760
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017179538615847543,
+ "loss": 1.2019,
+ "theoretical_loss": 3.3865852746973504,
+ "tokens_seen": 2599223296
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017177933801404213,
+ "loss": 1.1098,
+ "theoretical_loss": 3.386578573446728,
+ "tokens_seen": 2599288832
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017176328986960883,
+ "loss": 1.2159,
+ "theoretical_loss": 3.3865718724123695,
+ "tokens_seen": 2599354368
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017174724172517553,
+ "loss": 1.229,
+ "theoretical_loss": 3.3865651715942624,
+ "tokens_seen": 2599419904
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017173119358074223,
+ "loss": 1.2518,
+ "theoretical_loss": 3.386558470992395,
+ "tokens_seen": 2599485440
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017171514543630893,
+ "loss": 1.2574,
+ "theoretical_loss": 3.3865517706067534,
+ "tokens_seen": 2599550976
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017169909729187563,
+ "loss": 1.2573,
+ "theoretical_loss": 3.386545070437327,
+ "tokens_seen": 2599616512
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017168304914744233,
+ "loss": 1.2931,
+ "theoretical_loss": 3.386538370484102,
+ "tokens_seen": 2599682048
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017166700100300903,
+ "loss": 1.2482,
+ "theoretical_loss": 3.3865316707470665,
+ "tokens_seen": 2599747584
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017165095285857573,
+ "loss": 1.2237,
+ "theoretical_loss": 3.3865249712262075,
+ "tokens_seen": 2599813120
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017163490471414243,
+ "loss": 1.292,
+ "theoretical_loss": 3.386518271921514,
+ "tokens_seen": 2599878656
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017161885656970913,
+ "loss": 1.2264,
+ "theoretical_loss": 3.3865115728329718,
+ "tokens_seen": 2599944192
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017160280842527583,
+ "loss": 1.2392,
+ "theoretical_loss": 3.38650487396057,
+ "tokens_seen": 2600009728
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017158676028084253,
+ "loss": 1.222,
+ "theoretical_loss": 3.386498175304295,
+ "tokens_seen": 2600075264
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017157071213640924,
+ "loss": 1.0486,
+ "theoretical_loss": 3.386491476864135,
+ "tokens_seen": 2600140800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017155466399197594,
+ "loss": 1.254,
+ "theoretical_loss": 3.3864847786400776,
+ "tokens_seen": 2600206336
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017153861584754264,
+ "loss": 1.1932,
+ "theoretical_loss": 3.38647808063211,
+ "tokens_seen": 2600271872
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017152256770310934,
+ "loss": 1.3205,
+ "theoretical_loss": 3.38647138284022,
+ "tokens_seen": 2600337408
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017150651955867604,
+ "loss": 1.2231,
+ "theoretical_loss": 3.3864646852643956,
+ "tokens_seen": 2600402944
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017149047141424274,
+ "loss": 1.2535,
+ "theoretical_loss": 3.386457987904624,
+ "tokens_seen": 2600468480
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017147442326980944,
+ "loss": 1.1954,
+ "theoretical_loss": 3.3864512907608924,
+ "tokens_seen": 2600534016
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017145837512537614,
+ "loss": 1.2332,
+ "theoretical_loss": 3.386444593833189,
+ "tokens_seen": 2600599552
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017144232698094284,
+ "loss": 1.2781,
+ "theoretical_loss": 3.386437897121501,
+ "tokens_seen": 2600665088
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017142627883650954,
+ "loss": 1.2763,
+ "theoretical_loss": 3.3864312006258164,
+ "tokens_seen": 2600730624
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017141023069207624,
+ "loss": 1.0783,
+ "theoretical_loss": 3.386424504346122,
+ "tokens_seen": 2600796160
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017139418254764294,
+ "loss": 1.2935,
+ "theoretical_loss": 3.386417808282406,
+ "tokens_seen": 2600861696
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017137813440320964,
+ "loss": 1.2007,
+ "theoretical_loss": 3.3864111124346565,
+ "tokens_seen": 2600927232
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017136208625877634,
+ "loss": 1.2695,
+ "theoretical_loss": 3.3864044168028604,
+ "tokens_seen": 2600992768
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017134603811434304,
+ "loss": 1.2083,
+ "theoretical_loss": 3.3863977213870053,
+ "tokens_seen": 2601058304
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017132998996990974,
+ "loss": 1.2228,
+ "theoretical_loss": 3.386391026187079,
+ "tokens_seen": 2601123840
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017131394182547645,
+ "loss": 1.1878,
+ "theoretical_loss": 3.3863843312030686,
+ "tokens_seen": 2601189376
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017129789368104315,
+ "loss": 1.2929,
+ "theoretical_loss": 3.386377636434962,
+ "tokens_seen": 2601254912
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017128184553660985,
+ "loss": 1.2214,
+ "theoretical_loss": 3.3863709418827472,
+ "tokens_seen": 2601320448
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017126579739217655,
+ "loss": 1.2248,
+ "theoretical_loss": 3.3863642475464117,
+ "tokens_seen": 2601385984
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017124974924774325,
+ "loss": 1.1397,
+ "theoretical_loss": 3.3863575534259427,
+ "tokens_seen": 2601451520
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017123370110330995,
+ "loss": 1.2759,
+ "theoretical_loss": 3.3863508595213276,
+ "tokens_seen": 2601517056
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017121765295887665,
+ "loss": 1.2822,
+ "theoretical_loss": 3.386344165832555,
+ "tokens_seen": 2601582592
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017120160481444335,
+ "loss": 1.2372,
+ "theoretical_loss": 3.3863374723596116,
+ "tokens_seen": 2601648128
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017118555667001005,
+ "loss": 1.1435,
+ "theoretical_loss": 3.3863307791024853,
+ "tokens_seen": 2601713664
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 869720,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.024566650390625,
+ "objective/train/theoretical_loss": 3.3863240860611636,
+ "objective/train/tokens_used": 2622239200,
+ "theoretical_loss": 3.3863240860611636,
+ "tokens_seen": 2601779200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017116950852557675,
+ "loss": 1.1325,
+ "theoretical_loss": 3.3863240860611636,
+ "tokens_seen": 2601779200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017115346038114342,
+ "loss": 1.1851,
+ "theoretical_loss": 3.3863173932356343,
+ "tokens_seen": 2601844736
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017113741223671013,
+ "loss": 1.2064,
+ "theoretical_loss": 3.3863107006258852,
+ "tokens_seen": 2601910272
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017112136409227683,
+ "loss": 1.2823,
+ "theoretical_loss": 3.386304008231903,
+ "tokens_seen": 2601975808
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017110531594784353,
+ "loss": 1.2141,
+ "theoretical_loss": 3.3862973160536765,
+ "tokens_seen": 2602041344
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017108926780341023,
+ "loss": 1.1668,
+ "theoretical_loss": 3.3862906240911927,
+ "tokens_seen": 2602106880
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017107321965897693,
+ "loss": 1.17,
+ "theoretical_loss": 3.386283932344439,
+ "tokens_seen": 2602172416
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017105717151454363,
+ "loss": 1.1668,
+ "theoretical_loss": 3.3862772408134036,
+ "tokens_seen": 2602237952
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017104112337011033,
+ "loss": 1.2318,
+ "theoretical_loss": 3.386270549498074,
+ "tokens_seen": 2602303488
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017102507522567703,
+ "loss": 1.2274,
+ "theoretical_loss": 3.386263858398437,
+ "tokens_seen": 2602369024
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017100902708124373,
+ "loss": 1.1591,
+ "theoretical_loss": 3.386257167514481,
+ "tokens_seen": 2602434560
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017099297893681043,
+ "loss": 1.1516,
+ "theoretical_loss": 3.386250476846193,
+ "tokens_seen": 2602500096
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017097693079237713,
+ "loss": 1.1849,
+ "theoretical_loss": 3.3862437863935617,
+ "tokens_seen": 2602565632
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017096088264794383,
+ "loss": 1.2019,
+ "theoretical_loss": 3.3862370961565738,
+ "tokens_seen": 2602631168
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017094483450351053,
+ "loss": 1.3373,
+ "theoretical_loss": 3.3862304061352173,
+ "tokens_seen": 2602696704
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017092878635907723,
+ "loss": 1.2018,
+ "theoretical_loss": 3.38622371632948,
+ "tokens_seen": 2602762240
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017091273821464393,
+ "loss": 1.238,
+ "theoretical_loss": 3.3862170267393488,
+ "tokens_seen": 2602827776
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017089669007021063,
+ "loss": 1.1757,
+ "theoretical_loss": 3.386210337364812,
+ "tokens_seen": 2602893312
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017088064192577734,
+ "loss": 1.2018,
+ "theoretical_loss": 3.3862036482058566,
+ "tokens_seen": 2602958848
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017086459378134406,
+ "loss": 1.256,
+ "theoretical_loss": 3.3861969592624708,
+ "tokens_seen": 2603024384
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017084854563691076,
+ "loss": 1.2102,
+ "theoretical_loss": 3.386190270534642,
+ "tokens_seen": 2603089920
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017083249749247744,
+ "loss": 1.3176,
+ "theoretical_loss": 3.3861835820223583,
+ "tokens_seen": 2603155456
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017081644934804414,
+ "loss": 1.2257,
+ "theoretical_loss": 3.3861768937256063,
+ "tokens_seen": 2603220992
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017080040120361084,
+ "loss": 1.2117,
+ "theoretical_loss": 3.3861702056443743,
+ "tokens_seen": 2603286528
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017078435305917754,
+ "loss": 1.2162,
+ "theoretical_loss": 3.38616351777865,
+ "tokens_seen": 2603352064
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017076830491474424,
+ "loss": 1.2384,
+ "theoretical_loss": 3.386156830128421,
+ "tokens_seen": 2603417600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017075225677031094,
+ "loss": 1.1341,
+ "theoretical_loss": 3.3861501426936744,
+ "tokens_seen": 2603483136
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017073620862587764,
+ "loss": 1.1867,
+ "theoretical_loss": 3.3861434554743983,
+ "tokens_seen": 2603548672
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017072016048144434,
+ "loss": 1.2525,
+ "theoretical_loss": 3.3861367684705805,
+ "tokens_seen": 2603614208
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017070411233701104,
+ "loss": 1.2272,
+ "theoretical_loss": 3.386130081682208,
+ "tokens_seen": 2603679744
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017068806419257774,
+ "loss": 1.1098,
+ "theoretical_loss": 3.3861233951092693,
+ "tokens_seen": 2603745280
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017067201604814444,
+ "loss": 1.241,
+ "theoretical_loss": 3.386116708751752,
+ "tokens_seen": 2603810816
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017065596790371114,
+ "loss": 1.2437,
+ "theoretical_loss": 3.3861100226096426,
+ "tokens_seen": 2603876352
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017063991975927784,
+ "loss": 1.2221,
+ "theoretical_loss": 3.3861033366829294,
+ "tokens_seen": 2603941888
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017062387161484455,
+ "loss": 1.3139,
+ "theoretical_loss": 3.3860966509716004,
+ "tokens_seen": 2604007424
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017060782347041125,
+ "loss": 1.2106,
+ "theoretical_loss": 3.3860899654756427,
+ "tokens_seen": 2604072960
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017059177532597795,
+ "loss": 1.2153,
+ "theoretical_loss": 3.3860832801950447,
+ "tokens_seen": 2604138496
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017057572718154465,
+ "loss": 1.1526,
+ "theoretical_loss": 3.386076595129793,
+ "tokens_seen": 2604204032
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017055967903711135,
+ "loss": 1.3029,
+ "theoretical_loss": 3.386069910279876,
+ "tokens_seen": 2604269568
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017054363089267805,
+ "loss": 1.1406,
+ "theoretical_loss": 3.386063225645281,
+ "tokens_seen": 2604335104
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017052758274824475,
+ "loss": 1.2035,
+ "theoretical_loss": 3.3860565412259955,
+ "tokens_seen": 2604400640
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017051153460381145,
+ "loss": 1.1899,
+ "theoretical_loss": 3.386049857022008,
+ "tokens_seen": 2604466176
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017049548645937815,
+ "loss": 1.2937,
+ "theoretical_loss": 3.386043173033305,
+ "tokens_seen": 2604531712
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017047943831494485,
+ "loss": 1.2311,
+ "theoretical_loss": 3.386036489259875,
+ "tokens_seen": 2604597248
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017046339017051155,
+ "loss": 1.2336,
+ "theoretical_loss": 3.3860298057017055,
+ "tokens_seen": 2604662784
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017044734202607823,
+ "loss": 1.2547,
+ "theoretical_loss": 3.3860231223587838,
+ "tokens_seen": 2604728320
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017043129388164493,
+ "loss": 1.1884,
+ "theoretical_loss": 3.386016439231098,
+ "tokens_seen": 2604793856
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017041524573721163,
+ "loss": 1.233,
+ "theoretical_loss": 3.386009756318635,
+ "tokens_seen": 2604859392
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017039919759277833,
+ "loss": 1.2576,
+ "theoretical_loss": 3.3860030736213833,
+ "tokens_seen": 2604924928
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017038314944834503,
+ "loss": 1.2657,
+ "theoretical_loss": 3.3859963911393303,
+ "tokens_seen": 2604990464
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 870967,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2595572471618652,
+ "objective/train/theoretical_loss": 3.3859897088724633,
+ "objective/train/tokens_used": 2625516000,
+ "theoretical_loss": 3.3859897088724633,
+ "tokens_seen": 2605056000
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017036710130391173,
+ "loss": 1.1543,
+ "theoretical_loss": 3.3859897088724633,
+ "tokens_seen": 2605056000
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017035105315947843,
+ "loss": 1.2483,
+ "theoretical_loss": 3.3859830268207705,
+ "tokens_seen": 2605121536
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017033500501504513,
+ "loss": 1.1958,
+ "theoretical_loss": 3.3859763449842393,
+ "tokens_seen": 2605187072
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017031895687061183,
+ "loss": 1.1906,
+ "theoretical_loss": 3.3859696633628573,
+ "tokens_seen": 2605252608
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017030290872617853,
+ "loss": 1.3546,
+ "theoretical_loss": 3.3859629819566117,
+ "tokens_seen": 2605318144
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017028686058174523,
+ "loss": 1.2204,
+ "theoretical_loss": 3.3859563007654914,
+ "tokens_seen": 2605383680
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017027081243731193,
+ "loss": 1.2398,
+ "theoretical_loss": 3.385949619789483,
+ "tokens_seen": 2605449216
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017025476429287866,
+ "loss": 1.255,
+ "theoretical_loss": 3.3859429390285745,
+ "tokens_seen": 2605514752
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017023871614844536,
+ "loss": 1.2037,
+ "theoretical_loss": 3.3859362584827535,
+ "tokens_seen": 2605580288
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017022266800401206,
+ "loss": 1.2931,
+ "theoretical_loss": 3.385929578152008,
+ "tokens_seen": 2605645824
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017020661985957876,
+ "loss": 1.2856,
+ "theoretical_loss": 3.385922898036325,
+ "tokens_seen": 2605711360
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017019057171514546,
+ "loss": 1.2999,
+ "theoretical_loss": 3.385916218135693,
+ "tokens_seen": 2605776896
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017017452357071216,
+ "loss": 1.2362,
+ "theoretical_loss": 3.385909538450099,
+ "tokens_seen": 2605842432
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017015847542627886,
+ "loss": 1.2724,
+ "theoretical_loss": 3.385902858979531,
+ "tokens_seen": 2605907968
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017014242728184556,
+ "loss": 1.3852,
+ "theoretical_loss": 3.3858961797239764,
+ "tokens_seen": 2605973504
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017012637913741224,
+ "loss": 1.1625,
+ "theoretical_loss": 3.3858895006834233,
+ "tokens_seen": 2606039040
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017011033099297894,
+ "loss": 1.2253,
+ "theoretical_loss": 3.3858828218578587,
+ "tokens_seen": 2606104576
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017009428284854564,
+ "loss": 1.3155,
+ "theoretical_loss": 3.385876143247271,
+ "tokens_seen": 2606170112
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017007823470411234,
+ "loss": 1.3072,
+ "theoretical_loss": 3.385869464851648,
+ "tokens_seen": 2606235648
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017006218655967904,
+ "loss": 1.1748,
+ "theoretical_loss": 3.3858627866709763,
+ "tokens_seen": 2606301184
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017004613841524574,
+ "loss": 1.2022,
+ "theoretical_loss": 3.3858561087052443,
+ "tokens_seen": 2606366720
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017003009027081244,
+ "loss": 1.3129,
+ "theoretical_loss": 3.3858494309544396,
+ "tokens_seen": 2606432256
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00017001404212637914,
+ "loss": 1.2566,
+ "theoretical_loss": 3.38584275341855,
+ "tokens_seen": 2606497792
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016999799398194584,
+ "loss": 1.1721,
+ "theoretical_loss": 3.385836076097563,
+ "tokens_seen": 2606563328
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016998194583751254,
+ "loss": 1.2103,
+ "theoretical_loss": 3.3858293989914663,
+ "tokens_seen": 2606628864
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016996589769307924,
+ "loss": 1.2351,
+ "theoretical_loss": 3.3858227221002477,
+ "tokens_seen": 2606694400
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016994984954864594,
+ "loss": 1.2447,
+ "theoretical_loss": 3.385816045423895,
+ "tokens_seen": 2606759936
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016993380140421265,
+ "loss": 1.2751,
+ "theoretical_loss": 3.3858093689623954,
+ "tokens_seen": 2606825472
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016991775325977935,
+ "loss": 1.1853,
+ "theoretical_loss": 3.3858026927157368,
+ "tokens_seen": 2606891008
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016990170511534605,
+ "loss": 1.1921,
+ "theoretical_loss": 3.385796016683907,
+ "tokens_seen": 2606956544
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016988565697091275,
+ "loss": 1.2133,
+ "theoretical_loss": 3.3857893408668938,
+ "tokens_seen": 2607022080
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016986960882647945,
+ "loss": 1.1407,
+ "theoretical_loss": 3.385782665264685,
+ "tokens_seen": 2607087616
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016985356068204615,
+ "loss": 1.3111,
+ "theoretical_loss": 3.3857759898772675,
+ "tokens_seen": 2607153152
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016983751253761285,
+ "loss": 1.1399,
+ "theoretical_loss": 3.3857693147046297,
+ "tokens_seen": 2607218688
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016982146439317955,
+ "loss": 1.292,
+ "theoretical_loss": 3.3857626397467593,
+ "tokens_seen": 2607284224
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016980541624874625,
+ "loss": 1.2486,
+ "theoretical_loss": 3.3857559650036433,
+ "tokens_seen": 2607349760
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016978936810431295,
+ "loss": 1.2484,
+ "theoretical_loss": 3.38574929047527,
+ "tokens_seen": 2607415296
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016977331995987965,
+ "loss": 1.2124,
+ "theoretical_loss": 3.385742616161627,
+ "tokens_seen": 2607480832
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016975727181544635,
+ "loss": 1.3056,
+ "theoretical_loss": 3.3857359420627025,
+ "tokens_seen": 2607546368
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016974122367101303,
+ "loss": 1.18,
+ "theoretical_loss": 3.3857292681784834,
+ "tokens_seen": 2607611904
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016972517552657973,
+ "loss": 1.1419,
+ "theoretical_loss": 3.3857225945089575,
+ "tokens_seen": 2607677440
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016970912738214643,
+ "loss": 1.2472,
+ "theoretical_loss": 3.385715921054113,
+ "tokens_seen": 2607742976
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016969307923771313,
+ "loss": 1.1902,
+ "theoretical_loss": 3.385709247813937,
+ "tokens_seen": 2607808512
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016967703109327983,
+ "loss": 1.2354,
+ "theoretical_loss": 3.385702574788417,
+ "tokens_seen": 2607874048
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016966098294884656,
+ "loss": 1.0969,
+ "theoretical_loss": 3.3856959019775417,
+ "tokens_seen": 2607939584
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016964493480441326,
+ "loss": 1.2418,
+ "theoretical_loss": 3.385689229381298,
+ "tokens_seen": 2608005120
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016962888665997996,
+ "loss": 1.2278,
+ "theoretical_loss": 3.3856825569996745,
+ "tokens_seen": 2608070656
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016961283851554666,
+ "loss": 1.1736,
+ "theoretical_loss": 3.385675884832658,
+ "tokens_seen": 2608136192
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016959679037111336,
+ "loss": 1.2711,
+ "theoretical_loss": 3.385669212880236,
+ "tokens_seen": 2608201728
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016958074222668006,
+ "loss": 1.228,
+ "theoretical_loss": 3.385662541142397,
+ "tokens_seen": 2608267264
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 872150,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2419285774230957,
+ "objective/train/theoretical_loss": 3.3856558696191286,
+ "objective/train/tokens_used": 2628792800,
+ "theoretical_loss": 3.3856558696191286,
+ "tokens_seen": 2608332800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016956469408224676,
+ "loss": 1.1865,
+ "theoretical_loss": 3.3856558696191286,
+ "tokens_seen": 2608332800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016954864593781346,
+ "loss": 1.2419,
+ "theoretical_loss": 3.3856491983104178,
+ "tokens_seen": 2608398336
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016953259779338016,
+ "loss": 1.3424,
+ "theoretical_loss": 3.385642527216253,
+ "tokens_seen": 2608463872
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016951654964894686,
+ "loss": 1.1517,
+ "theoretical_loss": 3.385635856336622,
+ "tokens_seen": 2608529408
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016950050150451356,
+ "loss": 1.3027,
+ "theoretical_loss": 3.385629185671512,
+ "tokens_seen": 2608594944
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016948445336008026,
+ "loss": 1.2372,
+ "theoretical_loss": 3.385622515220911,
+ "tokens_seen": 2608660480
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016946840521564696,
+ "loss": 1.2367,
+ "theoretical_loss": 3.3856158449848066,
+ "tokens_seen": 2608726016
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016945235707121366,
+ "loss": 1.2451,
+ "theoretical_loss": 3.3856091749631867,
+ "tokens_seen": 2608791552
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016943630892678034,
+ "loss": 1.1902,
+ "theoretical_loss": 3.3856025051560388,
+ "tokens_seen": 2608857088
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016942026078234704,
+ "loss": 1.3121,
+ "theoretical_loss": 3.385595835563351,
+ "tokens_seen": 2608922624
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016940421263791374,
+ "loss": 1.2943,
+ "theoretical_loss": 3.38558916618511,
+ "tokens_seen": 2608988160
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016938816449348044,
+ "loss": 1.3143,
+ "theoretical_loss": 3.385582497021305,
+ "tokens_seen": 2609053696
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016937211634904714,
+ "loss": 1.1525,
+ "theoretical_loss": 3.3855758280719224,
+ "tokens_seen": 2609119232
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016935606820461384,
+ "loss": 1.1841,
+ "theoretical_loss": 3.385569159336951,
+ "tokens_seen": 2609184768
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016934002006018054,
+ "loss": 1.1959,
+ "theoretical_loss": 3.385562490816378,
+ "tokens_seen": 2609250304
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016932397191574724,
+ "loss": 1.2736,
+ "theoretical_loss": 3.3855558225101907,
+ "tokens_seen": 2609315840
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016930792377131394,
+ "loss": 1.2516,
+ "theoretical_loss": 3.3855491544183773,
+ "tokens_seen": 2609381376
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016929187562688064,
+ "loss": 1.1596,
+ "theoretical_loss": 3.3855424865409254,
+ "tokens_seen": 2609446912
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016927582748244734,
+ "loss": 1.2844,
+ "theoretical_loss": 3.3855358188778233,
+ "tokens_seen": 2609512448
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016925977933801405,
+ "loss": 1.1782,
+ "theoretical_loss": 3.3855291514290577,
+ "tokens_seen": 2609577984
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016924373119358075,
+ "loss": 1.2169,
+ "theoretical_loss": 3.385522484194617,
+ "tokens_seen": 2609643520
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016922768304914745,
+ "loss": 1.176,
+ "theoretical_loss": 3.385515817174489,
+ "tokens_seen": 2609709056
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016921163490471415,
+ "loss": 1.232,
+ "theoretical_loss": 3.385509150368661,
+ "tokens_seen": 2609774592
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016919558676028085,
+ "loss": 1.2884,
+ "theoretical_loss": 3.385502483777121,
+ "tokens_seen": 2609840128
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016917953861584755,
+ "loss": 1.0545,
+ "theoretical_loss": 3.385495817399857,
+ "tokens_seen": 2609905664
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016916349047141425,
+ "loss": 1.277,
+ "theoretical_loss": 3.3854891512368557,
+ "tokens_seen": 2609971200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016914744232698095,
+ "loss": 1.2624,
+ "theoretical_loss": 3.385482485288106,
+ "tokens_seen": 2610036736
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016913139418254765,
+ "loss": 1.2994,
+ "theoretical_loss": 3.385475819553595,
+ "tokens_seen": 2610102272
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016911534603811435,
+ "loss": 1.2652,
+ "theoretical_loss": 3.3854691540333106,
+ "tokens_seen": 2610167808
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016909929789368105,
+ "loss": 1.2449,
+ "theoretical_loss": 3.3854624887272404,
+ "tokens_seen": 2610233344
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016908324974924775,
+ "loss": 1.2374,
+ "theoretical_loss": 3.3854558236353727,
+ "tokens_seen": 2610298880
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016906720160481445,
+ "loss": 1.2267,
+ "theoretical_loss": 3.3854491587576945,
+ "tokens_seen": 2610364416
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016905115346038115,
+ "loss": 1.2022,
+ "theoretical_loss": 3.385442494094194,
+ "tokens_seen": 2610429952
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016903510531594785,
+ "loss": 1.1957,
+ "theoretical_loss": 3.385435829644859,
+ "tokens_seen": 2610495488
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016901905717151455,
+ "loss": 1.3015,
+ "theoretical_loss": 3.3854291654096764,
+ "tokens_seen": 2610561024
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016900300902708126,
+ "loss": 1.3187,
+ "theoretical_loss": 3.385422501388635,
+ "tokens_seen": 2610626560
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016898696088264796,
+ "loss": 1.2453,
+ "theoretical_loss": 3.3854158375817223,
+ "tokens_seen": 2610692096
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016897091273821466,
+ "loss": 1.0755,
+ "theoretical_loss": 3.3854091739889256,
+ "tokens_seen": 2610757632
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016895486459378136,
+ "loss": 1.2581,
+ "theoretical_loss": 3.3854025106102332,
+ "tokens_seen": 2610823168
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016893881644934806,
+ "loss": 1.2159,
+ "theoretical_loss": 3.3853958474456323,
+ "tokens_seen": 2610888704
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016892276830491476,
+ "loss": 1.2971,
+ "theoretical_loss": 3.3853891844951107,
+ "tokens_seen": 2610954240
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016890672016048146,
+ "loss": 1.2983,
+ "theoretical_loss": 3.3853825217586566,
+ "tokens_seen": 2611019776
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016889067201604816,
+ "loss": 1.2521,
+ "theoretical_loss": 3.385375859236258,
+ "tokens_seen": 2611085312
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016887462387161486,
+ "loss": 1.2069,
+ "theoretical_loss": 3.3853691969279014,
+ "tokens_seen": 2611150848
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016885857572718156,
+ "loss": 1.2595,
+ "theoretical_loss": 3.385362534833576,
+ "tokens_seen": 2611216384
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016884252758274826,
+ "loss": 1.2228,
+ "theoretical_loss": 3.3853558729532685,
+ "tokens_seen": 2611281920
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016882647943831496,
+ "loss": 1.1859,
+ "theoretical_loss": 3.3853492112869668,
+ "tokens_seen": 2611347456
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016881043129388166,
+ "loss": 1.3231,
+ "theoretical_loss": 3.385342549834659,
+ "tokens_seen": 2611412992
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016879438314944836,
+ "loss": 1.189,
+ "theoretical_loss": 3.385335888596333,
+ "tokens_seen": 2611478528
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016877833500501506,
+ "loss": 1.3082,
+ "theoretical_loss": 3.385329227571976,
+ "tokens_seen": 2611544064
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 873055,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.1384434700012207,
+ "objective/train/theoretical_loss": 3.3853225667615763,
+ "objective/train/tokens_used": 2632069600,
+ "theoretical_loss": 3.3853225667615763,
+ "tokens_seen": 2611609600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016876228686058176,
+ "loss": 1.215,
+ "theoretical_loss": 3.3853225667615763,
+ "tokens_seen": 2611609600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016874623871614847,
+ "loss": 1.2868,
+ "theoretical_loss": 3.385315906165121,
+ "tokens_seen": 2611675136
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016873019057171514,
+ "loss": 1.2178,
+ "theoretical_loss": 3.3853092457825986,
+ "tokens_seen": 2611740672
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016871414242728184,
+ "loss": 1.288,
+ "theoretical_loss": 3.385302585613996,
+ "tokens_seen": 2611806208
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016869809428284854,
+ "loss": 1.2948,
+ "theoretical_loss": 3.3852959256593023,
+ "tokens_seen": 2611871744
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016868204613841524,
+ "loss": 1.2146,
+ "theoretical_loss": 3.3852892659185043,
+ "tokens_seen": 2611937280
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016866599799398194,
+ "loss": 1.26,
+ "theoretical_loss": 3.3852826063915895,
+ "tokens_seen": 2612002816
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016864994984954864,
+ "loss": 1.2703,
+ "theoretical_loss": 3.3852759470785463,
+ "tokens_seen": 2612068352
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016863390170511534,
+ "loss": 1.3293,
+ "theoretical_loss": 3.385269287979362,
+ "tokens_seen": 2612133888
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016861785356068204,
+ "loss": 1.153,
+ "theoretical_loss": 3.385262629094025,
+ "tokens_seen": 2612199424
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016860180541624874,
+ "loss": 1.0797,
+ "theoretical_loss": 3.3852559704225227,
+ "tokens_seen": 2612264960
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016858575727181544,
+ "loss": 1.1908,
+ "theoretical_loss": 3.3852493119648424,
+ "tokens_seen": 2612330496
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016856970912738215,
+ "loss": 1.2822,
+ "theoretical_loss": 3.3852426537209723,
+ "tokens_seen": 2612396032
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016855366098294885,
+ "loss": 1.2198,
+ "theoretical_loss": 3.3852359956909006,
+ "tokens_seen": 2612461568
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016853761283851555,
+ "loss": 1.2771,
+ "theoretical_loss": 3.3852293378746143,
+ "tokens_seen": 2612527104
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016852156469408225,
+ "loss": 1.2405,
+ "theoretical_loss": 3.385222680272102,
+ "tokens_seen": 2612592640
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016850551654964895,
+ "loss": 1.2426,
+ "theoretical_loss": 3.385216022883351,
+ "tokens_seen": 2612658176
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016848946840521565,
+ "loss": 1.1061,
+ "theoretical_loss": 3.3852093657083486,
+ "tokens_seen": 2612723712
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016847342026078235,
+ "loss": 1.2411,
+ "theoretical_loss": 3.3852027087470833,
+ "tokens_seen": 2612789248
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016845737211634908,
+ "loss": 1.3229,
+ "theoretical_loss": 3.3851960519995425,
+ "tokens_seen": 2612854784
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016844132397191578,
+ "loss": 1.3325,
+ "theoretical_loss": 3.385189395465714,
+ "tokens_seen": 2612920320
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016842527582748248,
+ "loss": 1.2822,
+ "theoretical_loss": 3.385182739145586,
+ "tokens_seen": 2612985856
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016840922768304915,
+ "loss": 1.284,
+ "theoretical_loss": 3.3851760830391457,
+ "tokens_seen": 2613051392
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016839317953861585,
+ "loss": 1.1827,
+ "theoretical_loss": 3.3851694271463817,
+ "tokens_seen": 2613116928
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016837713139418255,
+ "loss": 1.3158,
+ "theoretical_loss": 3.3851627714672805,
+ "tokens_seen": 2613182464
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016836108324974925,
+ "loss": 1.2324,
+ "theoretical_loss": 3.385156116001831,
+ "tokens_seen": 2613248000
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016834503510531595,
+ "loss": 1.2266,
+ "theoretical_loss": 3.3851494607500205,
+ "tokens_seen": 2613313536
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016832898696088265,
+ "loss": 1.178,
+ "theoretical_loss": 3.385142805711837,
+ "tokens_seen": 2613379072
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016831293881644936,
+ "loss": 1.1424,
+ "theoretical_loss": 3.385136150887268,
+ "tokens_seen": 2613444608
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016829689067201606,
+ "loss": 1.2393,
+ "theoretical_loss": 3.385129496276302,
+ "tokens_seen": 2613510144
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016828084252758276,
+ "loss": 1.1723,
+ "theoretical_loss": 3.3851228418789256,
+ "tokens_seen": 2613575680
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016826479438314946,
+ "loss": 1.2367,
+ "theoretical_loss": 3.3851161876951275,
+ "tokens_seen": 2613641216
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016824874623871616,
+ "loss": 1.3545,
+ "theoretical_loss": 3.3851095337248953,
+ "tokens_seen": 2613706752
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016823269809428286,
+ "loss": 1.1597,
+ "theoretical_loss": 3.3851028799682163,
+ "tokens_seen": 2613772288
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016821664994984956,
+ "loss": 1.2352,
+ "theoretical_loss": 3.385096226425079,
+ "tokens_seen": 2613837824
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016820060180541626,
+ "loss": 1.1962,
+ "theoretical_loss": 3.385089573095471,
+ "tokens_seen": 2613903360
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016818455366098296,
+ "loss": 1.1827,
+ "theoretical_loss": 3.38508291997938,
+ "tokens_seen": 2613968896
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016816850551654966,
+ "loss": 1.1724,
+ "theoretical_loss": 3.3850762670767933,
+ "tokens_seen": 2614034432
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016815245737211636,
+ "loss": 1.3049,
+ "theoretical_loss": 3.3850696143876995,
+ "tokens_seen": 2614099968
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016813640922768306,
+ "loss": 1.2785,
+ "theoretical_loss": 3.3850629619120864,
+ "tokens_seen": 2614165504
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016812036108324976,
+ "loss": 1.2684,
+ "theoretical_loss": 3.385056309649941,
+ "tokens_seen": 2614231040
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016810431293881646,
+ "loss": 1.2728,
+ "theoretical_loss": 3.385049657601252,
+ "tokens_seen": 2614296576
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016808826479438316,
+ "loss": 1.1985,
+ "theoretical_loss": 3.3850430057660064,
+ "tokens_seen": 2614362112
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016807221664994986,
+ "loss": 1.2302,
+ "theoretical_loss": 3.3850363541441926,
+ "tokens_seen": 2614427648
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016805616850551657,
+ "loss": 1.2629,
+ "theoretical_loss": 3.3850297027357983,
+ "tokens_seen": 2614493184
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016804012036108327,
+ "loss": 1.2341,
+ "theoretical_loss": 3.385023051540811,
+ "tokens_seen": 2614558720
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016802407221664994,
+ "loss": 1.3219,
+ "theoretical_loss": 3.3850164005592185,
+ "tokens_seen": 2614624256
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016800802407221664,
+ "loss": 1.2591,
+ "theoretical_loss": 3.385009749791009,
+ "tokens_seen": 2614689792
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016799197592778334,
+ "loss": 1.2642,
+ "theoretical_loss": 3.38500309923617,
+ "tokens_seen": 2614755328
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016797592778335004,
+ "loss": 1.2432,
+ "theoretical_loss": 3.3849964488946895,
+ "tokens_seen": 2614820864
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 874224,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.2787208557128906,
+ "objective/train/theoretical_loss": 3.384989798766555,
+ "objective/train/tokens_used": 2635346400,
+ "theoretical_loss": 3.384989798766555,
+ "tokens_seen": 2614886400
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016795987963891674,
+ "loss": 1.2356,
+ "theoretical_loss": 3.384989798766555,
+ "tokens_seen": 2614886400
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016794383149448344,
+ "loss": 1.2377,
+ "theoretical_loss": 3.384983148851755,
+ "tokens_seen": 2614951936
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016792778335005014,
+ "loss": 1.2531,
+ "theoretical_loss": 3.3849764991502767,
+ "tokens_seen": 2615017472
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016791173520561684,
+ "loss": 1.2361,
+ "theoretical_loss": 3.3849698496621077,
+ "tokens_seen": 2615083008
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016789568706118354,
+ "loss": 1.1571,
+ "theoretical_loss": 3.3849632003872365,
+ "tokens_seen": 2615148544
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016787963891675025,
+ "loss": 1.1759,
+ "theoretical_loss": 3.3849565513256503,
+ "tokens_seen": 2615214080
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016786359077231697,
+ "loss": 1.2038,
+ "theoretical_loss": 3.384949902477337,
+ "tokens_seen": 2615279616
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016784754262788367,
+ "loss": 1.2945,
+ "theoretical_loss": 3.3849432538422852,
+ "tokens_seen": 2615345152
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016783149448345037,
+ "loss": 1.1668,
+ "theoretical_loss": 3.384936605420482,
+ "tokens_seen": 2615410688
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016781544633901708,
+ "loss": 1.1838,
+ "theoretical_loss": 3.3849299572119147,
+ "tokens_seen": 2615476224
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016779939819458378,
+ "loss": 1.3118,
+ "theoretical_loss": 3.3849233092165725,
+ "tokens_seen": 2615541760
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016778335005015048,
+ "loss": 1.2835,
+ "theoretical_loss": 3.384916661434442,
+ "tokens_seen": 2615607296
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016776730190571718,
+ "loss": 1.3089,
+ "theoretical_loss": 3.384910013865512,
+ "tokens_seen": 2615672832
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016775125376128388,
+ "loss": 1.2713,
+ "theoretical_loss": 3.384903366509769,
+ "tokens_seen": 2615738368
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016773520561685058,
+ "loss": 1.2629,
+ "theoretical_loss": 3.384896719367202,
+ "tokens_seen": 2615803904
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016771915747241728,
+ "loss": 1.3956,
+ "theoretical_loss": 3.384890072437799,
+ "tokens_seen": 2615869440
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016770310932798395,
+ "loss": 1.2264,
+ "theoretical_loss": 3.3848834257215463,
+ "tokens_seen": 2615934976
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016768706118355065,
+ "loss": 1.2027,
+ "theoretical_loss": 3.3848767792184336,
+ "tokens_seen": 2616000512
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016767101303911735,
+ "loss": 1.2908,
+ "theoretical_loss": 3.384870132928447,
+ "tokens_seen": 2616066048
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016765496489468405,
+ "loss": 1.2497,
+ "theoretical_loss": 3.384863486851576,
+ "tokens_seen": 2616131584
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016763891675025075,
+ "loss": 1.2624,
+ "theoretical_loss": 3.384856840987807,
+ "tokens_seen": 2616197120
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016762286860581746,
+ "loss": 1.0917,
+ "theoretical_loss": 3.384850195337129,
+ "tokens_seen": 2616262656
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016760682046138416,
+ "loss": 1.2987,
+ "theoretical_loss": 3.3848435498995286,
+ "tokens_seen": 2616328192
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016759077231695086,
+ "loss": 1.1984,
+ "theoretical_loss": 3.3848369046749944,
+ "tokens_seen": 2616393728
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016757472417251756,
+ "loss": 1.2002,
+ "theoretical_loss": 3.3848302596635143,
+ "tokens_seen": 2616459264
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016755867602808426,
+ "loss": 1.2797,
+ "theoretical_loss": 3.3848236148650757,
+ "tokens_seen": 2616524800
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016754262788365096,
+ "loss": 1.2631,
+ "theoretical_loss": 3.3848169702796667,
+ "tokens_seen": 2616590336
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016752657973921766,
+ "loss": 1.2049,
+ "theoretical_loss": 3.3848103259072753,
+ "tokens_seen": 2616655872
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016751053159478436,
+ "loss": 1.2776,
+ "theoretical_loss": 3.384803681747889,
+ "tokens_seen": 2616721408
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016749448345035106,
+ "loss": 1.3258,
+ "theoretical_loss": 3.384797037801496,
+ "tokens_seen": 2616786944
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016747843530591776,
+ "loss": 1.276,
+ "theoretical_loss": 3.3847903940680837,
+ "tokens_seen": 2616852480
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016746238716148446,
+ "loss": 1.1697,
+ "theoretical_loss": 3.38478375054764,
+ "tokens_seen": 2616918016
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016744633901705116,
+ "loss": 1.2393,
+ "theoretical_loss": 3.3847771072401533,
+ "tokens_seen": 2616983552
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016743029087261786,
+ "loss": 1.3329,
+ "theoretical_loss": 3.384770464145611,
+ "tokens_seen": 2617049088
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016741424272818456,
+ "loss": 1.2222,
+ "theoretical_loss": 3.3847638212640008,
+ "tokens_seen": 2617114624
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016739819458375126,
+ "loss": 1.2212,
+ "theoretical_loss": 3.3847571785953106,
+ "tokens_seen": 2617180160
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016738214643931797,
+ "loss": 1.2713,
+ "theoretical_loss": 3.3847505361395287,
+ "tokens_seen": 2617245696
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016736609829488467,
+ "loss": 1.2496,
+ "theoretical_loss": 3.3847438938966423,
+ "tokens_seen": 2617311232
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016735005015045137,
+ "loss": 1.1258,
+ "theoretical_loss": 3.38473725186664,
+ "tokens_seen": 2617376768
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016733400200601807,
+ "loss": 1.3258,
+ "theoretical_loss": 3.384730610049509,
+ "tokens_seen": 2617442304
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016731795386158474,
+ "loss": 1.2922,
+ "theoretical_loss": 3.384723968445237,
+ "tokens_seen": 2617507840
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016730190571715144,
+ "loss": 1.1316,
+ "theoretical_loss": 3.3847173270538127,
+ "tokens_seen": 2617573376
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016728585757271814,
+ "loss": 1.3189,
+ "theoretical_loss": 3.384710685875223,
+ "tokens_seen": 2617638912
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016726980942828484,
+ "loss": 1.3079,
+ "theoretical_loss": 3.3847040449094563,
+ "tokens_seen": 2617704448
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016725376128385157,
+ "loss": 1.2491,
+ "theoretical_loss": 3.3846974041565003,
+ "tokens_seen": 2617769984
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016723771313941827,
+ "loss": 1.2274,
+ "theoretical_loss": 3.3846907636163435,
+ "tokens_seen": 2617835520
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016722166499498497,
+ "loss": 1.26,
+ "theoretical_loss": 3.3846841232889724,
+ "tokens_seen": 2617901056
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016720561685055167,
+ "loss": 1.3491,
+ "theoretical_loss": 3.384677483174376,
+ "tokens_seen": 2617966592
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016718956870611837,
+ "loss": 1.2777,
+ "theoretical_loss": 3.384670843272542,
+ "tokens_seen": 2618032128
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016717352056168507,
+ "loss": 1.2068,
+ "theoretical_loss": 3.3846642035834575,
+ "tokens_seen": 2618097664
+ },
+ {
+ "epoch": 0.79,
+ "objective/train/docs_used": 875113,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.3086519241333008,
+ "objective/train/theoretical_loss": 3.384657564107111,
+ "objective/train/tokens_used": 2638623200,
+ "theoretical_loss": 3.384657564107111,
+ "tokens_seen": 2618163200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016715747241725177,
+ "loss": 1.2691,
+ "theoretical_loss": 3.384657564107111,
+ "tokens_seen": 2618163200
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016714142427281847,
+ "loss": 1.2629,
+ "theoretical_loss": 3.3846509248434904,
+ "tokens_seen": 2618228736
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016712537612838518,
+ "loss": 1.1998,
+ "theoretical_loss": 3.3846442857925836,
+ "tokens_seen": 2618294272
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016710932798395188,
+ "loss": 1.3553,
+ "theoretical_loss": 3.384637646954378,
+ "tokens_seen": 2618359808
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016709327983951858,
+ "loss": 1.2328,
+ "theoretical_loss": 3.3846310083288618,
+ "tokens_seen": 2618425344
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016707723169508528,
+ "loss": 1.1874,
+ "theoretical_loss": 3.384624369916023,
+ "tokens_seen": 2618490880
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016706118355065198,
+ "loss": 1.2423,
+ "theoretical_loss": 3.384617731715849,
+ "tokens_seen": 2618556416
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016704513540621868,
+ "loss": 1.2853,
+ "theoretical_loss": 3.384611093728328,
+ "tokens_seen": 2618621952
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016702908726178538,
+ "loss": 1.042,
+ "theoretical_loss": 3.384604455953448,
+ "tokens_seen": 2618687488
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016701303911735205,
+ "loss": 1.2206,
+ "theoretical_loss": 3.384597818391196,
+ "tokens_seen": 2618753024
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016699699097291875,
+ "loss": 1.2023,
+ "theoretical_loss": 3.3845911810415608,
+ "tokens_seen": 2618818560
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016698094282848545,
+ "loss": 1.2691,
+ "theoretical_loss": 3.38458454390453,
+ "tokens_seen": 2618884096
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016696489468405215,
+ "loss": 1.3746,
+ "theoretical_loss": 3.3845779069800916,
+ "tokens_seen": 2618949632
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016694884653961886,
+ "loss": 1.2727,
+ "theoretical_loss": 3.3845712702682333,
+ "tokens_seen": 2619015168
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016693279839518556,
+ "loss": 1.1702,
+ "theoretical_loss": 3.3845646337689432,
+ "tokens_seen": 2619080704
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016691675025075226,
+ "loss": 1.2712,
+ "theoretical_loss": 3.384557997482209,
+ "tokens_seen": 2619146240
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016690070210631896,
+ "loss": 1.3124,
+ "theoretical_loss": 3.3845513614080183,
+ "tokens_seen": 2619211776
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016688465396188566,
+ "loss": 1.3187,
+ "theoretical_loss": 3.384544725546359,
+ "tokens_seen": 2619277312
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016686860581745236,
+ "loss": 1.1336,
+ "theoretical_loss": 3.3845380898972195,
+ "tokens_seen": 2619342848
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016685255767301906,
+ "loss": 1.2219,
+ "theoretical_loss": 3.3845314544605873,
+ "tokens_seen": 2619408384
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016683650952858576,
+ "loss": 1.2748,
+ "theoretical_loss": 3.3845248192364505,
+ "tokens_seen": 2619473920
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016682046138415246,
+ "loss": 1.2611,
+ "theoretical_loss": 3.3845181842247967,
+ "tokens_seen": 2619539456
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016680441323971916,
+ "loss": 1.39,
+ "theoretical_loss": 3.384511549425614,
+ "tokens_seen": 2619604992
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016678836509528586,
+ "loss": 1.1171,
+ "theoretical_loss": 3.38450491483889,
+ "tokens_seen": 2619670528
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016677231695085256,
+ "loss": 1.178,
+ "theoretical_loss": 3.384498280464613,
+ "tokens_seen": 2619736064
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016675626880641926,
+ "loss": 1.3206,
+ "theoretical_loss": 3.3844916463027706,
+ "tokens_seen": 2619801600
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016674022066198596,
+ "loss": 1.1987,
+ "theoretical_loss": 3.384485012353351,
+ "tokens_seen": 2619867136
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016672417251755266,
+ "loss": 1.1255,
+ "theoretical_loss": 3.3844783786163415,
+ "tokens_seen": 2619932672
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016670812437311936,
+ "loss": 1.2536,
+ "theoretical_loss": 3.38447174509173,
+ "tokens_seen": 2619998208
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016669207622868607,
+ "loss": 1.2508,
+ "theoretical_loss": 3.3844651117795053,
+ "tokens_seen": 2620063744
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016667602808425277,
+ "loss": 1.2442,
+ "theoretical_loss": 3.3844584786796545,
+ "tokens_seen": 2620129280
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016665997993981947,
+ "loss": 1.2818,
+ "theoretical_loss": 3.3844518457921655,
+ "tokens_seen": 2620194816
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016664393179538617,
+ "loss": 1.4008,
+ "theoretical_loss": 3.3844452131170266,
+ "tokens_seen": 2620260352
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016662788365095287,
+ "loss": 1.2616,
+ "theoretical_loss": 3.3844385806542254,
+ "tokens_seen": 2620325888
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016661183550651957,
+ "loss": 1.2534,
+ "theoretical_loss": 3.38443194840375,
+ "tokens_seen": 2620391424
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016659578736208627,
+ "loss": 1.2473,
+ "theoretical_loss": 3.3844253163655877,
+ "tokens_seen": 2620456960
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016657973921765297,
+ "loss": 1.1846,
+ "theoretical_loss": 3.3844186845397273,
+ "tokens_seen": 2620522496
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016656369107321967,
+ "loss": 1.2849,
+ "theoretical_loss": 3.384412052926156,
+ "tokens_seen": 2620588032
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016654764292878637,
+ "loss": 1.1846,
+ "theoretical_loss": 3.3844054215248622,
+ "tokens_seen": 2620653568
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016653159478435307,
+ "loss": 1.2914,
+ "theoretical_loss": 3.3843987903358332,
+ "tokens_seen": 2620719104
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016651554663991977,
+ "loss": 1.2257,
+ "theoretical_loss": 3.384392159359057,
+ "tokens_seen": 2620784640
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016649949849548647,
+ "loss": 1.2509,
+ "theoretical_loss": 3.3843855285945224,
+ "tokens_seen": 2620850176
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016648345035105317,
+ "loss": 1.1446,
+ "theoretical_loss": 3.384378898042216,
+ "tokens_seen": 2620915712
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016646740220661987,
+ "loss": 1.1978,
+ "theoretical_loss": 3.3843722677021266,
+ "tokens_seen": 2620981248
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016645135406218657,
+ "loss": 1.3028,
+ "theoretical_loss": 3.3843656375742417,
+ "tokens_seen": 2621046784
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016643530591775328,
+ "loss": 1.2063,
+ "theoretical_loss": 3.3843590076585492,
+ "tokens_seen": 2621112320
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016641925777331998,
+ "loss": 1.2767,
+ "theoretical_loss": 3.3843523779550377,
+ "tokens_seen": 2621177856
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016640320962888668,
+ "loss": 1.1978,
+ "theoretical_loss": 3.3843457484636943,
+ "tokens_seen": 2621243392
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016638716148445338,
+ "loss": 1.3784,
+ "theoretical_loss": 3.384339119184507,
+ "tokens_seen": 2621308928
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016637111334002008,
+ "loss": 1.1895,
+ "theoretical_loss": 3.3843324901174636,
+ "tokens_seen": 2621374464
+ },
+ {
+ "debugging/Compilability": 0.0,
+ "debugging/distinct-1-grams": 0.646233227762004,
+ "debugging/entropy-1-grams": 4.872327732208451,
+ "debugging/length": 579.1111111111111,
+ "debugging/num_segments": 9,
+ "debugging/score": 0.0698136403192916,
+ "debugging/score_std": 0.015129296720156065,
+ "epoch": 0.79,
+ "objective/train/docs_used": 876316,
+ "objective/train/instantaneous_batch_size": 32,
+ "objective/train/instantaneous_microbatch_size": 32768,
+ "objective/train/original_loss": 1.4646857976913452,
+ "objective/train/theoretical_loss": 3.384325861262553,
+ "objective/train/tokens_used": 2641900000,
+ "theoretical_loss": 3.384325861262553,
+ "tokens_seen": 2621440000
+ },
+ {
+ "epoch": 0.79,
+ "learning_rate": 0.00016635506519558678,
+ "loss": 1.2757,
+ "theoretical_loss": 3.384325861262553,
+ "tokens_seen": 2621440000
+ }
+ ],
+ "max_steps": 50354,
+ "num_train_epochs": 9223372036854775807,
+ "total_flos": 1.33781520384e+18,
+ "trial_name": null,
+ "trial_params": null
+}